From 333f1564aca614faabc71dd6bcfa8257f7bfd7ec Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EB=B0=95=EC=B2=9C=EA=B5=90/On-Device=20Lab=28SR=29/Enginee?= =?utf8?q?r/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Tue, 5 Nov 2019 09:18:41 +0900 Subject: [PATCH] Introduce tf2tflite-value-pb-test (#8735) This commit introduces tf2tflite value test for pb input model. It copies existing tf2tflite-model-test only with renaming. Signed-off-by: Cheongyo Bahk --- compiler/tf2tflite-value-pb-test/.gitignore | 1 + compiler/tf2tflite-value-pb-test/CMakeLists.txt | 131 +++++++++++++++++++++ compiler/tf2tflite-value-pb-test/README.md | 1 + .../tf2tflite-value-pb-test/contrib/.gitignore | 3 + compiler/tf2tflite-value-pb-test/requires.cmake | 6 + compiler/tf2tflite-value-pb-test/runner.sh | 112 ++++++++++++++++++ 6 files changed, 254 insertions(+) create mode 100644 compiler/tf2tflite-value-pb-test/.gitignore create mode 100644 compiler/tf2tflite-value-pb-test/CMakeLists.txt create mode 100644 compiler/tf2tflite-value-pb-test/README.md create mode 100644 compiler/tf2tflite-value-pb-test/contrib/.gitignore create mode 100644 compiler/tf2tflite-value-pb-test/requires.cmake create mode 100755 compiler/tf2tflite-value-pb-test/runner.sh diff --git a/compiler/tf2tflite-value-pb-test/.gitignore b/compiler/tf2tflite-value-pb-test/.gitignore new file mode 100644 index 0000000..23c7c1b --- /dev/null +++ b/compiler/tf2tflite-value-pb-test/.gitignore @@ -0,0 +1 @@ +/contrib.lst diff --git a/compiler/tf2tflite-value-pb-test/CMakeLists.txt b/compiler/tf2tflite-value-pb-test/CMakeLists.txt new file mode 100644 index 0000000..41974f7 --- /dev/null +++ b/compiler/tf2tflite-value-pb-test/CMakeLists.txt @@ -0,0 +1,131 @@ +nnas_include(TargetRequire) + +unset(REQUIRED_TARGETS) +list(APPEND REQUIRED_TARGETS tf2tflite) +list(APPEND REQUIRED_TARGETS tfkit) +list(APPEND REQUIRED_TARGETS nnkit-run) +list(APPEND REQUIRED_TARGETS nnkit_tf_backend) +list(APPEND REQUIRED_TARGETS nnkit_tflite_backend) +list(APPEND REQUIRED_TARGETS nnkit_randomize_action) +list(APPEND REQUIRED_TARGETS nnkit_HDF5_export_action) +list(APPEND REQUIRED_TARGETS nnkit_HDF5_import_action) +list(APPEND REQUIRED_TARGETS i5diff) +TargetRequire_Return(${REQUIRED_TARGETS}) + +unset(KEYS) +unset(DEPS) + +### +### Add "Contrib" tests +### +macro(Add PREFIX) + # Let's use CONTRIB prefix to avoid name conflicts with official models + set(TEST_KEY "CONTRIB.${PREFIX}") + + set(PACKAGE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/contrib/${PREFIX}") + + set(MODEL_DOWNLOAD_SCRIPT "${PACKAGE_DIR}/model.download") + set(MODEL_PB_FILE "${PACKAGE_DIR}/model.pb") + set(MODEL_INFO_FILE "${PACKAGE_DIR}/model.info") + set(MODEL_MD5SUM_FILE "${PACKAGE_DIR}/model.md5sum") + + # Try to download a model if it is missing + if(NOT EXISTS "${MODEL_PB_FILE}") + # TODO Extract this routine as a helper function + if(NOT EXISTS "${MODEL_DOWNLOAD_SCRIPT}") + message(FATAL_ERROR "${TEST_KEY} - Download script is missing") + endif(NOT EXISTS "${MODEL_DOWNLOAD_SCRIPT}") + + execute_process( + COMMAND ${CMAKE_COMMAND} -D OUTPUT_PATH=${MODEL_PB_FILE} -P "${MODEL_DOWNLOAD_SCRIPT}" + RESULT_VARIABLE EXITCODE + ) + + if(NOT EXITCODE EQUAL 0) + message(FATAL_ERROR "${TEST_KEY} - Download fails") + endif(NOT EXITCODE EQUAL 0) + endif() + + if(EXISTS "${MODEL_MD5SUM_FILE}") + # TODO Extract this routine as a helper function + file(STRINGS "${MODEL_MD5SUM_FILE}" EXPECTED_MD5SUM) + file(MD5 "${MODEL_PB_FILE}" OBTAINED_MD5SUM) + + if(NOT "${EXPECTED_MD5SUM}" STREQUAL "${OBTAINED_MD5SUM}") + message(FATAL_ERROR "${TEST_KEY} - Checksum mismatches") + endif() + endif() + + # Generate .test file which declares MODEL_PB_PATH and MODEL_INFO_PATH + set(TEST_CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TEST_KEY}.test") + + add_custom_command( + OUTPUT ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_PB_PATH="${MODEL_PB_FILE}"' >> ${TEST_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_INFO_PATH="${MODEL_INFO_FILE}"' >> ${TEST_CONFIG_FILE} + COMMENT "Generate ${TEST_KEY} configuration" + ) + + list(APPEND KEYS "${TEST_KEY}") + list(APPEND DEPS "${TEST_CONFIG_FILE}") +endmacro(Add) + +include(contrib.lst OPTIONAL) + +### +### Generate toolchain.config +### +set(TOOLCHAIN_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/toolchain.config") + +add_custom_command( + OUTPUT ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E remove -f ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'NNKIT_RUN_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'TF_BACKEND_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'TFLITE_BACKEND_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'TF2TFLITE_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'RANDOMIZE_ACTION_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'HDF5_EXPORT_ACTION_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'HDF5_IMPORT_ACTION_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'I5DIFF_PATH=\"$\"' >> ${TOOLCHAIN_CONFIG} + DEPENDS + nnkit-run + nnkit_tf_backend + nnkit_tflite_backend + tf2tflite + nnkit_randomize_action + nnkit_HDF5_export_action + nnkit_HDF5_import_action + i5diff + COMMENT "Generate toolchin configuration" +) + +list(APPEND DEPS "${TOOLCHAIN_CONFIG}") + +## +## Generate test runner +## +set(TEST_RUNNER_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/runner.sh") +set(TEST_RUNNER_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/runner.sh") + +add_custom_command( + OUTPUT ${TEST_RUNNER_SCRIPT} + COMMAND ${CMAKE_COMMAND} -E copy "${TEST_RUNNER_SOURCE}" "${TEST_RUNNER_SCRIPT}" + DEPENDS ${TEST_RUNNER_SOURCE} + COMMENT "Generate test runner" +) + +list(APPEND DEPS "${TEST_RUNNER_SCRIPT}") + +### Generate dependencies +add_custom_target(tf2tflite_value_pb_test_deps ALL DEPENDS ${DEPS}) + +# NOTE This target is not built by default +add_test( + NAME tf2tflite_value_pb_test + COMMAND + "${TEST_RUNNER_SCRIPT}" + "${TOOLCHAIN_CONFIG}" + ${KEYS} +) diff --git a/compiler/tf2tflite-value-pb-test/README.md b/compiler/tf2tflite-value-pb-test/README.md new file mode 100644 index 0000000..6c4a68c --- /dev/null +++ b/compiler/tf2tflite-value-pb-test/README.md @@ -0,0 +1 @@ +# tf2tflite-value-pb-test diff --git a/compiler/tf2tflite-value-pb-test/contrib/.gitignore b/compiler/tf2tflite-value-pb-test/contrib/.gitignore new file mode 100644 index 0000000..968c345 --- /dev/null +++ b/compiler/tf2tflite-value-pb-test/contrib/.gitignore @@ -0,0 +1,3 @@ +/* +# Exclude all except below +!.gitignore diff --git a/compiler/tf2tflite-value-pb-test/requires.cmake b/compiler/tf2tflite-value-pb-test/requires.cmake new file mode 100644 index 0000000..471025e --- /dev/null +++ b/compiler/tf2tflite-value-pb-test/requires.cmake @@ -0,0 +1,6 @@ +require("i5diff") +require("nnkit-tf") +require("nnkit-tflite") +require("nnkit") +require("tf2tflite") +require("tfkit") diff --git a/compiler/tf2tflite-value-pb-test/runner.sh b/compiler/tf2tflite-value-pb-test/runner.sh new file mode 100755 index 0000000..faec125 --- /dev/null +++ b/compiler/tf2tflite-value-pb-test/runner.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +WORKDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Need at least toolchain.config +if [[ $# -lt 1 ]]; then + echo "USAGE: $0 ..." + echo + echo "ARGUMENTS:" + echo " [toolchain.config path]" + echo " [Prefix1]" + echo " [Prefix2]" + echo " ..." + exit 255 +fi + +CONFIG_PATH="$1"; shift + +source "${CONFIG_PATH}" + +echo "-- Found nnkit-run: ${NNKIT_RUN_PATH}" +echo "-- Found TF backend: ${TF_BACKEND_PATH}" +echo "-- Found TFLITE backend: ${TFLITE_BACKEND_PATH}" +echo "-- Found TF2TFLITE: ${TF2TFLITE_PATH}" +echo "-- Found randomize action: ${RANDOMIZE_ACTION_PATH}" +echo "-- Found HDF5 export action: ${HDF5_EXPORT_ACTION_PATH}" +echo "-- Found HDF5 import action: ${HDF5_IMPORT_ACTION_PATH}" +echo "-- Found i5diff: ${I5DIFF_PATH}" +echo "-- Found workdir: ${WORKDIR}" + +TESTED=() +PASSED=() +FAILED=() + +pushd "${WORKDIR}" +while [[ $# -ne 0 ]]; do + PREFIX="$1"; shift + + echo "[ RUN ] ${PREFIX}" + + TESTED+=("${PREFIX}") + + PASSED_TAG="${PREFIX}.passed" + + rm -f "${PASSED_TAG}" + + cat > "${PREFIX}.log" <( + exec 2>&1 + + source "${PREFIX}.test" + + echo "-- Use '${MODEL_PB_PATH}' and '${MODEL_INFO_PATH}'" + + # Exit immediately if any command fails + set -e + # Show commands + set -x + + # Generate tflite + "${TF2TFLITE_PATH}" \ + "${MODEL_INFO_PATH}" \ + "${MODEL_PB_PATH}" \ + "${WORKDIR}/${PREFIX}.tflite" + + # Run TensorFlow + "${NNKIT_RUN_PATH}" \ + --backend "${TF_BACKEND_PATH}" \ + --backend-arg "${MODEL_PB_PATH}" \ + --backend-arg "${MODEL_INFO_PATH}" \ + --pre "${RANDOMIZE_ACTION_PATH}" \ + --pre "${HDF5_EXPORT_ACTION_PATH}" \ + --pre-arg "${WORKDIR}/${PREFIX}.input.h5" \ + --post "${HDF5_EXPORT_ACTION_PATH}" \ + --post-arg "${WORKDIR}/${PREFIX}.expected.h5" + + # Run TensorFlow Lite + "${NNKIT_RUN_PATH}" \ + --backend "${TFLITE_BACKEND_PATH}" \ + --backend-arg "${WORKDIR}/${PREFIX}.tflite" \ + --pre "${HDF5_IMPORT_ACTION_PATH}" \ + --pre-arg "${WORKDIR}/${PREFIX}.input.h5" \ + --post "${HDF5_EXPORT_ACTION_PATH}" \ + --post-arg "${WORKDIR}/${PREFIX}.obtained.h5" + + "${I5DIFF_PATH}" -d 0.001 "${PREFIX}.expected.h5" "${PREFIX}.obtained.h5" + + if [[ $? -eq 0 ]]; then + touch "${PASSED_TAG}" + fi + ) + + if [[ -f "${PASSED_TAG}" ]]; then + echo "[ OK ] ${PREFIX}" + PASSED+=("$PREFIX") + else + echo "[ FAIL] ${PREFIX}" + FAILED+=("$PREFIX") + fi +done +popd + +if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then + echo "FAILED" + for TEST in "${FAILED[@]}" + do + echo "- ${TEST}" + done + exit 255 +fi + +echo "PASSED" +exit 0 -- 2.7.4