From: 윤현식/On-Device Lab(SR)/Principal Engineer/삼성전자 Date: Tue, 5 Nov 2019 06:17:38 +0000 (+0900) Subject: [tf2tflite-dredd-pbtxt-test] Adding CMakeLists.txt and runner.sh (#8744) X-Git-Tag: submit/tizen/20191205.083104~411 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=92035cdb7fa8993592e3b9fa7e4b18e62fc50e9d;p=platform%2Fcore%2Fml%2Fnnfw.git [tf2tflite-dredd-pbtxt-test] Adding CMakeLists.txt and runner.sh (#8744) * [tf2tflite-dredd-pbtxt-test] Adding CMakeLists.txt and runner.sh CMakeLists.txt and runner.sh files were added. Signed-off-by: Hyun Sik Yoon * modify comment * rules -> rule * TARGET_FILENAME -> TARGET_TESTNAME --- diff --git a/compiler/tf2tflite-dredd-pbtxt-test/CMakeLists.txt b/compiler/tf2tflite-dredd-pbtxt-test/CMakeLists.txt index b8bd383..e30afb0 100644 --- a/compiler/tf2tflite-dredd-pbtxt-test/CMakeLists.txt +++ b/compiler/tf2tflite-dredd-pbtxt-test/CMakeLists.txt @@ -1 +1,176 @@ -#TODO Write this +nnas_include(TargetRequire) + +unset(REQUIRED_TARGETS) +list(APPEND REQUIRED_TARGETS tfl-inspect) +list(APPEND REQUIRED_TARGETS tf2tflite) +list(APPEND REQUIRED_TARGETS tfkit) +TargetRequire_Return(${REQUIRED_TARGETS}) + +nncc_find_resource(TensorFlowTests) + +set(MODEL_REPO "${TensorFlowTests_DIR}") # Where to find text models to test + +unset(KEYS) +unset(DEPS) + +# +# processing models in test.lst and test.local.lst +# +# Example) +# +# Add(NET_0025 RULE test.rule) +# -> Read test.pbtxt file under res/TensorFlowTests/NET_0025 and create "NET_0025.tflite" +# Then the tflite is tested against rules in test.rule file. +# +macro(Add MODEL_DIR) + + set(ARG_OPTION) + set(ARG_ONE_VALUE RULE) # rule file name + set(ARG_MULTI_VALUE) + cmake_parse_arguments(ARG "${ARG_OPTION}" "${ARG_ONE_VALUE}" "${ARG_MULTI_VALUE}" ${ARGN}) + + if(NOT ARG_RULE) + message( FATAL_ERROR "RULE is mandadatory arg" ) + endif() + + set(RULE_FILENAME ${ARG_RULE}) + + set(TARGET_TESTNAME "${MODEL_DIR}") + + set(MODEL_SOURCE_DIR "${MODEL_REPO}/${MODEL_DIR}") + + set(TXT_SOURCE_PBTXT_PATH "${MODEL_SOURCE_DIR}/test.pbtxt") + set(TXT_SOURCE_INFO_PATH "${MODEL_SOURCE_DIR}/test.info") + set(TXT_SOURCE_RULE_PATH "${MODEL_SOURCE_DIR}/${RULE_FILENAME}") + + set(TXT_TARGET_PB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.pb") + set(TXT_TARGET_PBTXT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.pbtxt") + set(TXT_TARGET_INFO_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.info") + set(TXT_TARGET_RULE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.rule") + + if(NOT EXISTS "${TXT_SOURCE_PBTXT_PATH}") + message(FATAL_ERROR "${TXT_SOURCE_PBTXT_PATH} - pbtxt file does not exist") + endif(NOT EXISTS "${TXT_SOURCE_PBTXT_PATH}") + + if(NOT EXISTS "${TXT_SOURCE_INFO_PATH}") + message(FATAL_ERROR "${TXT_SOURCE_INFO_PATH} - info file does not exist") + endif(NOT EXISTS "${TXT_SOURCE_INFO_PATH}") + + if(NOT EXISTS "${TXT_SOURCE_RULE_PATH}") + message(FATAL_ERROR "${TXT_SOURCE_RULE_PATH} - rule file does not exist") + endif(NOT EXISTS "${TXT_SOURCE_RULE_PATH}") + + # Copy .pbtxt + add_custom_command(OUTPUT ${TXT_TARGET_PBTXT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_PBTXT_PATH}" "${TXT_TARGET_PBTXT_PATH}" + DEPENDS ${TXT_SOURCE_PBTXT_PATH} + COMMENT "Generate ${TXT_TARGET_PBTXT_PATH}" + ) + + # Copy .info + add_custom_command(OUTPUT ${TXT_TARGET_INFO_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_INFO_PATH}" "${TXT_TARGET_INFO_PATH}" + DEPENDS ${TXT_SOURCE_INFO_PATH} + COMMENT "Generate ${TXT_TARGET_INFO_PATH}" + ) + + # Copy .rule + add_custom_command(OUTPUT ${TXT_TARGET_RULE_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_RULE_PATH}" "${TXT_TARGET_RULE_PATH}" + DEPENDS ${TXT_SOURCE_RULE_PATH} + COMMENT "Generate ${TXT_TARGET_RULE_PATH}" + ) + + # Generate .pb from .pbtxt + add_custom_command(OUTPUT ${TXT_TARGET_PB_PATH} + COMMAND $ encode ${TXT_TARGET_PBTXT_PATH} ${TXT_TARGET_PB_PATH} + DEPENDS ${TXT_TARGET_PBTXT_PATH} + COMMENT "Generate ${TXT_TARGET_PB_PATH}" + ) + + # Generate .test file which declares path of target pb, info, rule files + # this file is used inside runner.sh + set(TEST_CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.test") + + add_custom_command( + OUTPUT ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_PB_PATH="${TXT_TARGET_PB_PATH}"' >> ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_INFO_PATH="${TXT_TARGET_INFO_PATH}"' >> ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_RULE_PATH="${TXT_TARGET_RULE_PATH}"' >> ${TEST_CONFIG_FILE} + DEPENDS + ${TXT_TARGET_PB_PATH} + ${TXT_TARGET_INFO_PATH} + ${TXT_TARGET_RULE_PATH} + COMMENT "Generate ${TARGET_TESTNAME} configuration for TXT" + ) + + list(APPEND DEPS "${TEST_CONFIG_FILE}") + list(APPEND KEYS "${TARGET_TESTNAME}") + +endmacro(Add) + +# Read "test.lst" +include("test.lst") +# Read "test.local.lst" if exists +include("test.local.lst" OPTIONAL) + +# +# Generate toolchain.config +# +set(TOOLCHAIN_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/toolchain.config") + +add_custom_command( + OUTPUT ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E remove -f ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'TFL_INSPECT_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'TF2TFLITE_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + # add more if new excutable file is needed in runner.sh and rule-lib.sh + DEPENDS + tfl-inspect + tf2tflite + COMMENT "Generate toolchin configuration" +) + +list(APPEND DEPS "${TOOLCHAIN_CONFIG}") + +# +# copy runner.sh +# +set(SOURCE_RUNNER "${CMAKE_CURRENT_SOURCE_DIR}/runner.sh") +set(TARGET_RUNNER "${CMAKE_CURRENT_BINARY_DIR}/runner.sh") + +add_custom_command( + OUTPUT ${TARGET_RUNNER} + COMMAND ${CMAKE_COMMAND} -E copy "${SOURCE_RUNNER}" "${TARGET_RUNNER}" + DEPENDS ${SOURCE_RUNNER} + COMMENT "Generate test runner" +) + +list(APPEND DEPS "${TARGET_RUNNER}") + +# +# copy rule-lib.sh (a library of shell script functions) +# +set(SOURCE_RULE_LIB "${CMAKE_CURRENT_SOURCE_DIR}/rule-lib.sh") +set(TARGET_RULE_LIB "${CMAKE_CURRENT_BINARY_DIR}/rule-lib.sh") + +add_custom_command( + OUTPUT ${TARGET_RULE_LIB} + COMMAND ${CMAKE_COMMAND} -E copy "${SOURCE_RULE_LIB}" "${TARGET_RULE_LIB}" + DEPENDS ${SOURCE_RULE_LIB} + COMMENT "Generate rule lib" +) + +list(APPEND DEPS "${TARGET_RULE_LIB}") + +# Generate dependencies +add_custom_target(tf2tflite_dredd_pbtxt_deps ALL DEPENDS ${DEPS}) + +add_test( + NAME tf2tflite_dredd_pbtxt_test + COMMAND + "${TARGET_RUNNER}" + "${TOOLCHAIN_CONFIG}" + ${KEYS} +) diff --git a/compiler/tf2tflite-dredd-pbtxt-test/requires.cmake b/compiler/tf2tflite-dredd-pbtxt-test/requires.cmake index 9fa4bf9..c077b66 100644 --- a/compiler/tf2tflite-dredd-pbtxt-test/requires.cmake +++ b/compiler/tf2tflite-dredd-pbtxt-test/requires.cmake @@ -1 +1,3 @@ -# TODO write this +require("tfkit") +require("tf2tflite") +require("tfl-inspect") diff --git a/compiler/tf2tflite-dredd-pbtxt-test/rule-lib.sh b/compiler/tf2tflite-dredd-pbtxt-test/rule-lib.sh new file mode 100755 index 0000000..9fa4bf9 --- /dev/null +++ b/compiler/tf2tflite-dredd-pbtxt-test/rule-lib.sh @@ -0,0 +1 @@ +# TODO write this diff --git a/compiler/tf2tflite-dredd-pbtxt-test/runner.sh b/compiler/tf2tflite-dredd-pbtxt-test/runner.sh new file mode 100755 index 0000000..f84137e --- /dev/null +++ b/compiler/tf2tflite-dredd-pbtxt-test/runner.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# This script checks tflite file generated by tf2tflite + +WORKDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Need at least toolchain.config +if [[ $# -lt 1 ]]; then + echo "USAGE: $0 ..." + echo + echo "ARGUMENTS:" + echo " [toolchain.config path]" + echo " [Prefix1]" + echo " [Prefix2]" + echo " ..." + exit 255 +fi + +CONFIG_PATH="$1"; shift + +source "${CONFIG_PATH}" + +echo "-- Found tfl-inspect: ${TFL_INSPECT_PATH}" +echo "-- Found tf2tflite: ${TF2TFLITE_PATH}" +echo "-- Found workdir: ${WORKDIR}" + +TESTED=() +PASSED=() +FAILED=() + +pushd "${WORKDIR}" + +# loading RULE and functions that checks tflite file +source rule-lib.sh + +while [[ $# -ne 0 ]]; do + PREFIX="$1"; shift + + echo "[ RUN ] ${PREFIX}" + + TESTED+=("${PREFIX}") + + PASSED_TAG="${PREFIX}.passed" + + rm -f "${PASSED_TAG}" + + cat > "${PREFIX}.log" <( + exec 2>&1 + + source "${PREFIX}.test" + + echo "-- Use '${MODEL_PB_PATH}', '${MODEL_INFO_PATH}', and '${MODEL_RULE_PATH}'" + + # Exit immediately if any command fails + set -e + # Show commands + set -x + + # Generate tflite + "${TF2TFLITE_PATH}" \ + "${MODEL_INFO_PATH}" \ + "${MODEL_PB_PATH}" \ + "${WORKDIR}/${PREFIX}.tflite" + + # + # Run rule prepared to check tflite file + # + + # set vars needed by rule file + TFLITE_PATH="${WORKDIR}/${PREFIX}.tflite" + + # Note: turn off 'command printing'. Otherwise printing will be so messy + set +x + source "${MODEL_RULE_PATH}" + set -x + + if [[ $? -eq 0 ]]; then + touch "${PASSED_TAG}" + fi + ) + + if [[ -f "${PASSED_TAG}" ]]; then + echo "[ OK ] ${PREFIX}" + PASSED+=("$PREFIX") + else + echo "[ FAIL] ${PREFIX}" + FAILED+=("$PREFIX") + fi +done +popd + +if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then + echo "FAILED" + for TEST in "${FAILED[@]}" + do + echo "- ${TEST}" + done + exit 255 +fi + +echo "PASSED" +exit 0 diff --git a/compiler/tf2tflite-dredd-pbtxt-test/test.lst b/compiler/tf2tflite-dredd-pbtxt-test/test.lst new file mode 100644 index 0000000..5699a2c --- /dev/null +++ b/compiler/tf2tflite-dredd-pbtxt-test/test.lst @@ -0,0 +1,2 @@ +# TODO Add NET_0025 like the following +# Add(NET_0025 RULE test.rule) # CBR