From ce56e84a9eb795428a9745ee48bafb039fc9e05c Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=9C=A4=ED=98=84=EC=8B=9D/On-Device=20Lab=28SR=29/Princip?= =?utf8?q?al=20Engineer/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Mon, 11 Nov 2019 15:11:00 +0900 Subject: [PATCH] [tf2circle-dredd-pbtxt-test] Introducing tf2circle-dredd-pbtxt-test (#8882) * [tf2circle-dredd-pbtxt-test] Introducing tf2circle-dredd-pbtxt-test tf2circle-dredd-pbtxt-test is introduced, which is rule base test for circle files generated from pbtxt files under `res/TensorFlowTests/`. Signed-off-by: Hyun Sik Yoon * Update compiler/tf2circle-dredd-pbtxt-test/runner.sh thanks, bot. Co-Authored-By: hyunsik-yoon --- compiler/tf2circle-dredd-pbtxt-test/.gitignore | 1 + compiler/tf2circle-dredd-pbtxt-test/CMakeLists.txt | 184 +++++++++++++++++++++ compiler/tf2circle-dredd-pbtxt-test/README.md | 3 + compiler/tf2circle-dredd-pbtxt-test/requires.cmake | 5 + compiler/tf2circle-dredd-pbtxt-test/runner.sh | 110 ++++++++++++ compiler/tf2circle-dredd-pbtxt-test/test.lst | 2 + 6 files changed, 305 insertions(+) create mode 100644 compiler/tf2circle-dredd-pbtxt-test/.gitignore create mode 100644 compiler/tf2circle-dredd-pbtxt-test/CMakeLists.txt create mode 100644 compiler/tf2circle-dredd-pbtxt-test/README.md create mode 100644 compiler/tf2circle-dredd-pbtxt-test/requires.cmake create mode 100755 compiler/tf2circle-dredd-pbtxt-test/runner.sh create mode 100644 compiler/tf2circle-dredd-pbtxt-test/test.lst diff --git a/compiler/tf2circle-dredd-pbtxt-test/.gitignore b/compiler/tf2circle-dredd-pbtxt-test/.gitignore new file mode 100644 index 0000000..8dbfa90 --- /dev/null +++ b/compiler/tf2circle-dredd-pbtxt-test/.gitignore @@ -0,0 +1 @@ +/test.local.lst diff --git a/compiler/tf2circle-dredd-pbtxt-test/CMakeLists.txt b/compiler/tf2circle-dredd-pbtxt-test/CMakeLists.txt new file mode 100644 index 0000000..789e585 --- /dev/null +++ b/compiler/tf2circle-dredd-pbtxt-test/CMakeLists.txt @@ -0,0 +1,184 @@ +nnas_include(TargetRequire) + +unset(REQUIRED_TARGETS) +list(APPEND REQUIRED_TARGETS circle-inspect) +list(APPEND REQUIRED_TARGETS circle-verify) +list(APPEND REQUIRED_TARGETS tf2circle) +list(APPEND REQUIRED_TARGETS tfkit) +list(APPEND REQUIRED_TARGETS dredd_rule_lib) +TargetRequire_Return(${REQUIRED_TARGETS}) + +nncc_find_resource(TensorFlowTests) + +set(MODEL_REPO "${TensorFlowTests_DIR}") # Where to find text models to test + +unset(KEYS) +unset(DEPS) + +# +# processing models in test.lst and test.local.lst +# +# Example) +# +# Add(NET_0025 RULE test.rule) +# -> Read test.pbtxt file under res/TensorFlowTests/NET_0025 and create "NET_0025.circle" +# Then the circle is tested against rules in test.rule file. +# +macro(Add MODEL_DIR) + + set(ARG_OPTION) + set(ARG_ONE_VALUE RULE) # rule file name + set(ARG_MULTI_VALUE) + cmake_parse_arguments(ARG "${ARG_OPTION}" "${ARG_ONE_VALUE}" "${ARG_MULTI_VALUE}" ${ARGN}) + + if(NOT ARG_RULE) + message( FATAL_ERROR "RULE is mandadatory arg" ) + endif() + + set(RULE_FILENAME ${ARG_RULE}) + + set(TARGET_TESTNAME "${MODEL_DIR}") + + set(MODEL_SOURCE_DIR "${MODEL_REPO}/${MODEL_DIR}") + + set(TXT_SOURCE_PBTXT_PATH "${MODEL_SOURCE_DIR}/test.pbtxt") + set(TXT_SOURCE_INFO_PATH "${MODEL_SOURCE_DIR}/test.info") + set(TXT_SOURCE_RULE_PATH "${MODEL_SOURCE_DIR}/${RULE_FILENAME}") + + set(TXT_TARGET_PB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.pb") + set(TXT_TARGET_PBTXT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.pbtxt") + set(TXT_TARGET_INFO_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.info") + set(TXT_TARGET_RULE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.rule") + + if(NOT EXISTS "${TXT_SOURCE_PBTXT_PATH}") + message(FATAL_ERROR "${TXT_SOURCE_PBTXT_PATH} - pbtxt file does not exist") + endif(NOT EXISTS "${TXT_SOURCE_PBTXT_PATH}") + + if(NOT EXISTS "${TXT_SOURCE_INFO_PATH}") + message(FATAL_ERROR "${TXT_SOURCE_INFO_PATH} - info file does not exist") + endif(NOT EXISTS "${TXT_SOURCE_INFO_PATH}") + + if(NOT EXISTS "${TXT_SOURCE_RULE_PATH}") + message(FATAL_ERROR "${TXT_SOURCE_RULE_PATH} - rule file does not exist") + endif(NOT EXISTS "${TXT_SOURCE_RULE_PATH}") + + # Copy .pbtxt + add_custom_command(OUTPUT ${TXT_TARGET_PBTXT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_PBTXT_PATH}" "${TXT_TARGET_PBTXT_PATH}" + DEPENDS ${TXT_SOURCE_PBTXT_PATH} + COMMENT "Generate ${TXT_TARGET_PBTXT_PATH}" + ) + + # Copy .info + add_custom_command(OUTPUT ${TXT_TARGET_INFO_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_INFO_PATH}" "${TXT_TARGET_INFO_PATH}" + DEPENDS ${TXT_SOURCE_INFO_PATH} + COMMENT "Generate ${TXT_TARGET_INFO_PATH}" + ) + + # Copy .rule + add_custom_command(OUTPUT ${TXT_TARGET_RULE_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_RULE_PATH}" "${TXT_TARGET_RULE_PATH}" + DEPENDS ${TXT_SOURCE_RULE_PATH} + COMMENT "Generate ${TXT_TARGET_RULE_PATH}" + ) + + # Generate .pb from .pbtxt + add_custom_command(OUTPUT ${TXT_TARGET_PB_PATH} + COMMAND $ encode ${TXT_TARGET_PBTXT_PATH} ${TXT_TARGET_PB_PATH} + DEPENDS ${TXT_TARGET_PBTXT_PATH} + COMMENT "Generate ${TXT_TARGET_PB_PATH}" + ) + + # Generate .test file which declares path of target pb, info, rule files + # this file is used inside runner.sh + set(TEST_CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.test") + + add_custom_command( + OUTPUT ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_PB_PATH="${TXT_TARGET_PB_PATH}"' >> ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_INFO_PATH="${TXT_TARGET_INFO_PATH}"' >> ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_RULE_PATH="${TXT_TARGET_RULE_PATH}"' >> ${TEST_CONFIG_FILE} + DEPENDS + ${TXT_TARGET_PB_PATH} + ${TXT_TARGET_INFO_PATH} + ${TXT_TARGET_RULE_PATH} + COMMENT "Generate ${TARGET_TESTNAME} configuration for TXT" + ) + + list(APPEND DEPS "${TEST_CONFIG_FILE}") + list(APPEND KEYS "${TARGET_TESTNAME}") + +endmacro(Add) + +# Read "test.lst" +include("test.lst") +# Read "test.local.lst" if exists +include("test.local.lst" OPTIONAL) + +# +# Generate toolchain.config +# +set(TOOLCHAIN_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/toolchain.config") + +add_custom_command( + OUTPUT ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E remove -f ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_INSPECT_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_VERIFY_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'TF2CIRCLE_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + # add more if new excutable file is needed in runner.sh and rule-lib.sh + DEPENDS + circle-inspect + circle-verify + tf2circle + COMMENT "Generate toolchin configuration" +) + +list(APPEND DEPS "${TOOLCHAIN_CONFIG}") + +# +# copy runner.sh +# +set(SOURCE_RUNNER "${CMAKE_CURRENT_SOURCE_DIR}/runner.sh") +set(TARGET_RUNNER "${CMAKE_CURRENT_BINARY_DIR}/runner.sh") + +add_custom_command( + OUTPUT ${TARGET_RUNNER} + COMMAND ${CMAKE_COMMAND} -E copy "${SOURCE_RUNNER}" "${TARGET_RUNNER}" + DEPENDS ${SOURCE_RUNNER} + COMMENT "Generate test runner" +) + +list(APPEND DEPS "${TARGET_RUNNER}") + +# +# copy rule-lib.sh (a library of shell script functions) +# + +# getting path for rule-lib.sh in dredd-rule-lib +get_target_property(DREDD_RULE_LIB_DIR dredd_rule_lib BINARY_DIR) + +set(SOURCE_RULE_LIB "${DREDD_RULE_LIB_DIR}/rule-lib.sh") +set(TARGET_RULE_LIB "${CMAKE_CURRENT_BINARY_DIR}/rule-lib.sh") + +add_custom_command( + OUTPUT ${TARGET_RULE_LIB} + COMMAND ${CMAKE_COMMAND} -E copy "${SOURCE_RULE_LIB}" "${TARGET_RULE_LIB}" + DEPENDS ${SOURCE_RULE_LIB} + COMMENT "Generate rule lib" +) + +list(APPEND DEPS "${TARGET_RULE_LIB}") + +# Generate dependencies +add_custom_target(tf2circle_dredd_pbtxt_deps ALL DEPENDS ${DEPS}) + +add_test( + NAME tf2circle_dredd_pbtxt_test + COMMAND + "${TARGET_RUNNER}" + "${TOOLCHAIN_CONFIG}" + ${KEYS} +) diff --git a/compiler/tf2circle-dredd-pbtxt-test/README.md b/compiler/tf2circle-dredd-pbtxt-test/README.md new file mode 100644 index 0000000..8eb9061 --- /dev/null +++ b/compiler/tf2circle-dredd-pbtxt-test/README.md @@ -0,0 +1,3 @@ +# tf2circle-dredd-pbtxt-test + +TODO write content. diff --git a/compiler/tf2circle-dredd-pbtxt-test/requires.cmake b/compiler/tf2circle-dredd-pbtxt-test/requires.cmake new file mode 100644 index 0000000..747d8ab --- /dev/null +++ b/compiler/tf2circle-dredd-pbtxt-test/requires.cmake @@ -0,0 +1,5 @@ +require("tfkit") +require("tf2circle") +require("circle-inspect") +require("circle-verify") +require("dredd-rule-lib") diff --git a/compiler/tf2circle-dredd-pbtxt-test/runner.sh b/compiler/tf2circle-dredd-pbtxt-test/runner.sh new file mode 100755 index 0000000..5745db1 --- /dev/null +++ b/compiler/tf2circle-dredd-pbtxt-test/runner.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# This script checks circle file generated by tf2circle + +WORKDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Need at least toolchain.config +if [[ $# -lt 1 ]]; then + echo "USAGE: $0 ..." + echo + echo "ARGUMENTS:" + echo " [toolchain.config path]" + echo " [Prefix1]" + echo " [Prefix2]" + echo " ..." + exit 255 +fi + +CONFIG_PATH="$1"; shift + +source "${CONFIG_PATH}" + +echo "-- Found circle-inspect: ${CIRCLE_INSPECT_PATH}" +echo "-- Found circle-verify: ${CIRCLE_VERIFY_PATH}" +echo "-- Found tf2circle: ${TF2CIRCLE_PATH}" +echo "-- Found workdir: ${WORKDIR}" + +TESTED=() +PASSED=() +FAILED=() + +pushd "${WORKDIR}" + +# running each rule file + +while [[ $# -ne 0 ]]; do + PREFIX="$1"; shift + + echo "[ RUN ] ${PREFIX}" + + TESTED+=("${PREFIX}") + + PASSED_TAG="${PREFIX}.passed" + + rm -f "${PASSED_TAG}" + + cat > "${PREFIX}.log" <( + exec 2>&1 + + source "${PREFIX}.test" + + echo "-- Use '${MODEL_PB_PATH}', '${MODEL_INFO_PATH}', and '${MODEL_RULE_PATH}'" + + # Exit immediately if any command fails + set -e + # Show commands + set -x + + # Generate circle + "${TF2CIRCLE_PATH}" \ + "${MODEL_INFO_PATH}" \ + "${MODEL_PB_PATH}" \ + "${WORKDIR}/${PREFIX}.circle" + + # + # Run rule prepared to check circle file + # + + # set vars needed by rule file + CIRCLE_PATH="${WORKDIR}/${PREFIX}.circle" + + # Note: turn off 'command printing'. Otherwise printing will be so messy + set +x + + # set vars required by rule-lib.sh and rule file + COMPILED_FILE=${CIRCLE_PATH} + INSPECT_PROG_PATH=${CIRCLE_INSPECT_PATH} + VERIFY_PROG_PATH=${CIRCLE_VERIFY_PATH} + + source rule-lib.sh + source "${MODEL_RULE_PATH}" + + set -x + + if [[ $? -eq 0 ]]; then + touch "${PASSED_TAG}" + fi + ) + + if [[ -f "${PASSED_TAG}" ]]; then + echo "[ OK ] ${PREFIX}" + PASSED+=("$PREFIX") + else + echo "[ FAIL] ${PREFIX}" + FAILED+=("$PREFIX") + fi +done +popd + +if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then + echo "FAILED" + for TEST in "${FAILED[@]}" + do + echo "- ${TEST}" + done + exit 255 +fi + +echo "PASSED" +exit 0 diff --git a/compiler/tf2circle-dredd-pbtxt-test/test.lst b/compiler/tf2circle-dredd-pbtxt-test/test.lst new file mode 100644 index 0000000..185b3fc --- /dev/null +++ b/compiler/tf2circle-dredd-pbtxt-test/test.lst @@ -0,0 +1,2 @@ +# TODO add tests like the following: +# Add(NET_0030 RULE circle_1.0_rel_requirement.rule) # Concat -- 2.7.4