Introduce tf2tflite-value-pb-test (#8735)

This commit introduces tf2tflite value test for pb input model. It copies
existing tf2tflite-model-test only with renaming.

Signed-off-by: Cheongyo Bahk <ch.bahk@samsung.com>

diff --git a/compiler/tf2tflite-value-pb-test/.gitignore b/compiler/tf2tflite-value-pb-test/.gitignore
new file mode 100644 (file)
index 0000000..23c7c1b
--- /dev/null
+++ b/compiler/tf2tflite-value-pb-test/.gitignore
@@ -0,0 +1 @@
+/contrib.lst

diff --git a/compiler/tf2tflite-value-pb-test/CMakeLists.txt b/compiler/tf2tflite-value-pb-test/CMakeLists.txt
new file mode 100644 (file)
index 0000000..41974f7
--- /dev/null
+++ b/compiler/tf2tflite-value-pb-test/CMakeLists.txt
@@ -0,0 +1,131 @@
+nnas_include(TargetRequire)
+
+unset(REQUIRED_TARGETS)
+list(APPEND REQUIRED_TARGETS tf2tflite)
+list(APPEND REQUIRED_TARGETS tfkit)
+list(APPEND REQUIRED_TARGETS nnkit-run)
+list(APPEND REQUIRED_TARGETS nnkit_tf_backend)
+list(APPEND REQUIRED_TARGETS nnkit_tflite_backend)
+list(APPEND REQUIRED_TARGETS nnkit_randomize_action)
+list(APPEND REQUIRED_TARGETS nnkit_HDF5_export_action)
+list(APPEND REQUIRED_TARGETS nnkit_HDF5_import_action)
+list(APPEND REQUIRED_TARGETS i5diff)
+TargetRequire_Return(${REQUIRED_TARGETS})
+
+unset(KEYS)
+unset(DEPS)
+
+###
+### Add "Contrib" tests
+###
+macro(Add PREFIX)
+  # Let's use CONTRIB prefix to avoid name conflicts with official models
+  set(TEST_KEY "CONTRIB.${PREFIX}")
+
+  set(PACKAGE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/contrib/${PREFIX}")
+
+  set(MODEL_DOWNLOAD_SCRIPT "${PACKAGE_DIR}/model.download")
+  set(MODEL_PB_FILE "${PACKAGE_DIR}/model.pb")
+  set(MODEL_INFO_FILE "${PACKAGE_DIR}/model.info")
+  set(MODEL_MD5SUM_FILE "${PACKAGE_DIR}/model.md5sum")
+
+  # Try to download a model if it is missing
+  if(NOT EXISTS "${MODEL_PB_FILE}")
+    # TODO Extract this routine as a helper function
+    if(NOT EXISTS "${MODEL_DOWNLOAD_SCRIPT}")
+      message(FATAL_ERROR "${TEST_KEY} - Download script is missing")
+    endif(NOT EXISTS "${MODEL_DOWNLOAD_SCRIPT}")
+
+    execute_process(
+      COMMAND ${CMAKE_COMMAND} -D OUTPUT_PATH=${MODEL_PB_FILE} -P "${MODEL_DOWNLOAD_SCRIPT}"
+      RESULT_VARIABLE EXITCODE
+    )
+
+    if(NOT EXITCODE EQUAL 0)
+      message(FATAL_ERROR "${TEST_KEY} - Download fails")
+    endif(NOT EXITCODE EQUAL 0)
+  endif()
+
+  if(EXISTS "${MODEL_MD5SUM_FILE}")
+    # TODO Extract this routine as a helper function
+    file(STRINGS "${MODEL_MD5SUM_FILE}" EXPECTED_MD5SUM)
+    file(MD5 "${MODEL_PB_FILE}" OBTAINED_MD5SUM)
+
+    if(NOT "${EXPECTED_MD5SUM}" STREQUAL "${OBTAINED_MD5SUM}")
+      message(FATAL_ERROR "${TEST_KEY} - Checksum mismatches")
+    endif()
+  endif()
+
+  # Generate .test file which declares MODEL_PB_PATH and MODEL_INFO_PATH
+  set(TEST_CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TEST_KEY}.test")
+
+  add_custom_command(
+    OUTPUT ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_PB_PATH="${MODEL_PB_FILE}"' >> ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_INFO_PATH="${MODEL_INFO_FILE}"' >> ${TEST_CONFIG_FILE}
+    COMMENT "Generate ${TEST_KEY} configuration"
+  )
+
+  list(APPEND KEYS "${TEST_KEY}")
+  list(APPEND DEPS "${TEST_CONFIG_FILE}")
+endmacro(Add)
+
+include(contrib.lst OPTIONAL)
+
+###
+### Generate toolchain.config
+###
+set(TOOLCHAIN_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/toolchain.config")
+
+add_custom_command(
+  OUTPUT ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E remove -f ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'NNKIT_RUN_PATH=\"$<TARGET_FILE:nnkit-run>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TF_BACKEND_PATH=\"$<TARGET_FILE:nnkit_tf_backend>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TFLITE_BACKEND_PATH=\"$<TARGET_FILE:nnkit_tflite_backend>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TF2TFLITE_PATH=\"$<TARGET_FILE:tf2tflite>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'RANDOMIZE_ACTION_PATH=\"$<TARGET_FILE:nnkit_randomize_action>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'HDF5_EXPORT_ACTION_PATH=\"$<TARGET_FILE:nnkit_HDF5_export_action>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'HDF5_IMPORT_ACTION_PATH=\"$<TARGET_FILE:nnkit_HDF5_import_action>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'I5DIFF_PATH=\"$<TARGET_FILE:i5diff>\"' >> ${TOOLCHAIN_CONFIG}
+  DEPENDS
+    nnkit-run
+    nnkit_tf_backend
+    nnkit_tflite_backend
+    tf2tflite
+    nnkit_randomize_action
+    nnkit_HDF5_export_action
+    nnkit_HDF5_import_action
+    i5diff
+  COMMENT "Generate toolchin configuration"
+)
+
+list(APPEND DEPS "${TOOLCHAIN_CONFIG}")
+
+##
+## Generate test runner
+##
+set(TEST_RUNNER_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/runner.sh")
+set(TEST_RUNNER_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/runner.sh")
+
+add_custom_command(
+  OUTPUT ${TEST_RUNNER_SCRIPT}
+  COMMAND ${CMAKE_COMMAND} -E copy "${TEST_RUNNER_SOURCE}" "${TEST_RUNNER_SCRIPT}"
+  DEPENDS ${TEST_RUNNER_SOURCE}
+  COMMENT "Generate test runner"
+)
+
+list(APPEND DEPS "${TEST_RUNNER_SCRIPT}")
+
+### Generate dependencies
+add_custom_target(tf2tflite_value_pb_test_deps ALL DEPENDS ${DEPS})
+
+# NOTE This target is not built by default
+add_test(
+  NAME tf2tflite_value_pb_test
+  COMMAND
+    "${TEST_RUNNER_SCRIPT}"
+    "${TOOLCHAIN_CONFIG}"
+    ${KEYS}
+)

diff --git a/compiler/tf2tflite-value-pb-test/README.md b/compiler/tf2tflite-value-pb-test/README.md
new file mode 100644 (file)
index 0000000..6c4a68c
--- /dev/null
+++ b/compiler/tf2tflite-value-pb-test/README.md
@@ -0,0 +1 @@
+# tf2tflite-value-pb-test

diff --git a/compiler/tf2tflite-value-pb-test/contrib/.gitignore b/compiler/tf2tflite-value-pb-test/contrib/.gitignore
new file mode 100644 (file)
index 0000000..968c345
--- /dev/null
+++ b/compiler/tf2tflite-value-pb-test/contrib/.gitignore
@@ -0,0 +1,3 @@
+/*
+# Exclude all except below
+!.gitignore

diff --git a/compiler/tf2tflite-value-pb-test/requires.cmake b/compiler/tf2tflite-value-pb-test/requires.cmake
new file mode 100644 (file)
index 0000000..471025e
--- /dev/null
+++ b/compiler/tf2tflite-value-pb-test/requires.cmake
@@ -0,0 +1,6 @@
+require("i5diff")
+require("nnkit-tf")
+require("nnkit-tflite")
+require("nnkit")
+require("tf2tflite")
+require("tfkit")

diff --git a/compiler/tf2tflite-value-pb-test/runner.sh b/compiler/tf2tflite-value-pb-test/runner.sh
new file mode 100755 (executable)
index 0000000..faec125
--- /dev/null
+++ b/compiler/tf2tflite-value-pb-test/runner.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+WORKDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+# Need at least toolchain.config
+if [[ $# -lt 1 ]]; then
+  echo "USAGE: $0 ..."
+  echo
+  echo "ARGUMENTS:"
+  echo "  [toolchain.config path]"
+  echo "  [Prefix1]"
+  echo "  [Prefix2]"
+  echo "  ..."
+  exit 255
+fi
+
+CONFIG_PATH="$1"; shift
+
+source "${CONFIG_PATH}"
+
+echo "-- Found nnkit-run: ${NNKIT_RUN_PATH}"
+echo "-- Found TF backend: ${TF_BACKEND_PATH}"
+echo "-- Found TFLITE backend: ${TFLITE_BACKEND_PATH}"
+echo "-- Found TF2TFLITE: ${TF2TFLITE_PATH}"
+echo "-- Found randomize action: ${RANDOMIZE_ACTION_PATH}"
+echo "-- Found HDF5 export action: ${HDF5_EXPORT_ACTION_PATH}"
+echo "-- Found HDF5 import action: ${HDF5_IMPORT_ACTION_PATH}"
+echo "-- Found i5diff: ${I5DIFF_PATH}"
+echo "-- Found workdir: ${WORKDIR}"
+
+TESTED=()
+PASSED=()
+FAILED=()
+
+pushd "${WORKDIR}"
+while [[ $# -ne 0 ]]; do
+  PREFIX="$1"; shift
+
+  echo "[ RUN      ] ${PREFIX}"
+
+  TESTED+=("${PREFIX}")
+
+  PASSED_TAG="${PREFIX}.passed"
+
+  rm -f "${PASSED_TAG}"
+
+  cat > "${PREFIX}.log" <(
+    exec 2>&1
+
+    source "${PREFIX}.test"
+
+    echo "-- Use '${MODEL_PB_PATH}' and '${MODEL_INFO_PATH}'"
+
+    # Exit immediately if any command fails
+    set -e
+    # Show commands
+    set -x
+
+    # Generate tflite
+    "${TF2TFLITE_PATH}" \
+      "${MODEL_INFO_PATH}" \
+      "${MODEL_PB_PATH}" \
+      "${WORKDIR}/${PREFIX}.tflite"
+
+    # Run TensorFlow
+    "${NNKIT_RUN_PATH}" \
+      --backend "${TF_BACKEND_PATH}" \
+      --backend-arg "${MODEL_PB_PATH}" \
+      --backend-arg "${MODEL_INFO_PATH}" \
+      --pre "${RANDOMIZE_ACTION_PATH}" \
+      --pre "${HDF5_EXPORT_ACTION_PATH}" \
+      --pre-arg "${WORKDIR}/${PREFIX}.input.h5" \
+      --post "${HDF5_EXPORT_ACTION_PATH}" \
+      --post-arg "${WORKDIR}/${PREFIX}.expected.h5"
+
+    # Run TensorFlow Lite
+    "${NNKIT_RUN_PATH}" \
+      --backend "${TFLITE_BACKEND_PATH}" \
+      --backend-arg "${WORKDIR}/${PREFIX}.tflite" \
+      --pre "${HDF5_IMPORT_ACTION_PATH}" \
+      --pre-arg "${WORKDIR}/${PREFIX}.input.h5" \
+      --post "${HDF5_EXPORT_ACTION_PATH}" \
+      --post-arg "${WORKDIR}/${PREFIX}.obtained.h5"
+
+    "${I5DIFF_PATH}" -d 0.001 "${PREFIX}.expected.h5" "${PREFIX}.obtained.h5"
+
+    if [[ $? -eq 0 ]]; then
+      touch "${PASSED_TAG}"
+    fi
+  )
+
+  if [[ -f "${PASSED_TAG}" ]]; then
+    echo "[       OK ] ${PREFIX}"
+    PASSED+=("$PREFIX")
+  else
+    echo "[      FAIL] ${PREFIX}"
+    FAILED+=("$PREFIX")
+  fi
+done
+popd
+
+if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then
+  echo "FAILED"
+  for TEST in "${FAILED[@]}"
+  do
+    echo "- ${TEST}"
+  done
+  exit 255
+fi
+
+echo "PASSED"
+exit 0