[tf2tflite-dredd-pbtxt-test] Adding CMakeLists.txt and runner.sh (#8744)

* [tf2tflite-dredd-pbtxt-test] Adding CMakeLists.txt and runner.sh

CMakeLists.txt and runner.sh files were added.

Signed-off-by: Hyun Sik Yoon <hyunsik.yoon@samsung.com>
* modify comment

* rules -> rule

* TARGET_FILENAME -> TARGET_TESTNAME

diff --git a/compiler/tf2tflite-dredd-pbtxt-test/CMakeLists.txt b/compiler/tf2tflite-dredd-pbtxt-test/CMakeLists.txt
index b8bd383..e30afb0 100644 (file)
--- a/compiler/tf2tflite-dredd-pbtxt-test/CMakeLists.txt
+++ b/compiler/tf2tflite-dredd-pbtxt-test/CMakeLists.txt
@@ -1 +1,176 @@
-#TODO Write this
+nnas_include(TargetRequire)
+
+unset(REQUIRED_TARGETS)
+list(APPEND REQUIRED_TARGETS tfl-inspect)
+list(APPEND REQUIRED_TARGETS tf2tflite)
+list(APPEND REQUIRED_TARGETS tfkit)
+TargetRequire_Return(${REQUIRED_TARGETS})
+
+nncc_find_resource(TensorFlowTests)
+
+set(MODEL_REPO "${TensorFlowTests_DIR}") # Where to find text models to test
+
+unset(KEYS)
+unset(DEPS)
+
+#
+# processing models in test.lst and test.local.lst
+#
+# Example)
+#
+#  Add(NET_0025 RULE test.rule)
+#    -> Read test.pbtxt file under res/TensorFlowTests/NET_0025 and create "NET_0025.tflite"
+#       Then the tflite is tested against rules in test.rule file.
+#
+macro(Add MODEL_DIR)
+
+  set(ARG_OPTION)
+  set(ARG_ONE_VALUE RULE) # rule file name
+  set(ARG_MULTI_VALUE)
+  cmake_parse_arguments(ARG "${ARG_OPTION}" "${ARG_ONE_VALUE}" "${ARG_MULTI_VALUE}" ${ARGN})
+
+  if(NOT ARG_RULE)
+    message( FATAL_ERROR "RULE is mandadatory arg" )
+  endif()
+
+  set(RULE_FILENAME ${ARG_RULE})
+
+  set(TARGET_TESTNAME "${MODEL_DIR}")
+
+  set(MODEL_SOURCE_DIR "${MODEL_REPO}/${MODEL_DIR}")
+
+  set(TXT_SOURCE_PBTXT_PATH "${MODEL_SOURCE_DIR}/test.pbtxt")
+  set(TXT_SOURCE_INFO_PATH "${MODEL_SOURCE_DIR}/test.info")
+  set(TXT_SOURCE_RULE_PATH "${MODEL_SOURCE_DIR}/${RULE_FILENAME}")
+
+  set(TXT_TARGET_PB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.pb")
+  set(TXT_TARGET_PBTXT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.pbtxt")
+  set(TXT_TARGET_INFO_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.info")
+  set(TXT_TARGET_RULE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.rule")
+
+  if(NOT EXISTS "${TXT_SOURCE_PBTXT_PATH}")
+    message(FATAL_ERROR "${TXT_SOURCE_PBTXT_PATH} - pbtxt file does not exist")
+  endif(NOT EXISTS "${TXT_SOURCE_PBTXT_PATH}")
+
+  if(NOT EXISTS "${TXT_SOURCE_INFO_PATH}")
+    message(FATAL_ERROR "${TXT_SOURCE_INFO_PATH} - info file does not exist")
+  endif(NOT EXISTS "${TXT_SOURCE_INFO_PATH}")
+
+  if(NOT EXISTS "${TXT_SOURCE_RULE_PATH}")
+    message(FATAL_ERROR "${TXT_SOURCE_RULE_PATH} - rule file does not exist")
+  endif(NOT EXISTS "${TXT_SOURCE_RULE_PATH}")
+
+  # Copy .pbtxt
+  add_custom_command(OUTPUT ${TXT_TARGET_PBTXT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_PBTXT_PATH}" "${TXT_TARGET_PBTXT_PATH}"
+    DEPENDS ${TXT_SOURCE_PBTXT_PATH}
+    COMMENT "Generate ${TXT_TARGET_PBTXT_PATH}"
+  )
+
+  # Copy .info
+  add_custom_command(OUTPUT ${TXT_TARGET_INFO_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_INFO_PATH}" "${TXT_TARGET_INFO_PATH}"
+    DEPENDS ${TXT_SOURCE_INFO_PATH}
+    COMMENT "Generate ${TXT_TARGET_INFO_PATH}"
+  )
+
+  # Copy .rule
+  add_custom_command(OUTPUT ${TXT_TARGET_RULE_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_RULE_PATH}" "${TXT_TARGET_RULE_PATH}"
+    DEPENDS ${TXT_SOURCE_RULE_PATH}
+    COMMENT "Generate ${TXT_TARGET_RULE_PATH}"
+  )
+
+  # Generate .pb from .pbtxt
+  add_custom_command(OUTPUT ${TXT_TARGET_PB_PATH}
+    COMMAND $<TARGET_FILE:tfkit> encode ${TXT_TARGET_PBTXT_PATH} ${TXT_TARGET_PB_PATH}
+    DEPENDS ${TXT_TARGET_PBTXT_PATH}
+    COMMENT "Generate ${TXT_TARGET_PB_PATH}"
+  )
+
+  # Generate .test file which declares path of target pb, info, rule files
+  # this file is used inside runner.sh
+  set(TEST_CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.test")
+
+  add_custom_command(
+    OUTPUT ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_PB_PATH="${TXT_TARGET_PB_PATH}"' >> ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_INFO_PATH="${TXT_TARGET_INFO_PATH}"' >> ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_RULE_PATH="${TXT_TARGET_RULE_PATH}"' >> ${TEST_CONFIG_FILE}
+    DEPENDS
+      ${TXT_TARGET_PB_PATH}
+      ${TXT_TARGET_INFO_PATH}
+      ${TXT_TARGET_RULE_PATH}
+    COMMENT "Generate ${TARGET_TESTNAME} configuration for TXT"
+  )
+
+  list(APPEND DEPS "${TEST_CONFIG_FILE}")
+  list(APPEND KEYS "${TARGET_TESTNAME}")
+
+endmacro(Add)
+
+# Read "test.lst"
+include("test.lst")
+# Read "test.local.lst" if exists
+include("test.local.lst" OPTIONAL)
+
+#
+# Generate toolchain.config
+#
+set(TOOLCHAIN_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/toolchain.config")
+
+add_custom_command(
+  OUTPUT ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E remove -f ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TFL_INSPECT_PATH=\"$<TARGET_FILE:tfl-inspect>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TF2TFLITE_PATH=\"$<TARGET_FILE:tf2tflite>\"' >> ${TOOLCHAIN_CONFIG}
+  # add more if new excutable file is needed in runner.sh and rule-lib.sh
+  DEPENDS
+    tfl-inspect
+    tf2tflite
+  COMMENT "Generate toolchin configuration"
+)
+
+list(APPEND DEPS "${TOOLCHAIN_CONFIG}")
+
+#
+# copy runner.sh
+#
+set(SOURCE_RUNNER "${CMAKE_CURRENT_SOURCE_DIR}/runner.sh")
+set(TARGET_RUNNER "${CMAKE_CURRENT_BINARY_DIR}/runner.sh")
+
+add_custom_command(
+  OUTPUT ${TARGET_RUNNER}
+  COMMAND ${CMAKE_COMMAND} -E copy "${SOURCE_RUNNER}" "${TARGET_RUNNER}"
+  DEPENDS ${SOURCE_RUNNER}
+  COMMENT "Generate test runner"
+)
+
+list(APPEND DEPS "${TARGET_RUNNER}")
+
+#
+# copy rule-lib.sh (a library of shell script functions)
+#
+set(SOURCE_RULE_LIB "${CMAKE_CURRENT_SOURCE_DIR}/rule-lib.sh")
+set(TARGET_RULE_LIB "${CMAKE_CURRENT_BINARY_DIR}/rule-lib.sh")
+
+add_custom_command(
+  OUTPUT ${TARGET_RULE_LIB}
+  COMMAND ${CMAKE_COMMAND} -E copy "${SOURCE_RULE_LIB}" "${TARGET_RULE_LIB}"
+  DEPENDS ${SOURCE_RULE_LIB}
+  COMMENT "Generate rule lib"
+)
+
+list(APPEND DEPS "${TARGET_RULE_LIB}")
+
+# Generate dependencies
+add_custom_target(tf2tflite_dredd_pbtxt_deps ALL DEPENDS ${DEPS})
+
+add_test(
+  NAME tf2tflite_dredd_pbtxt_test
+  COMMAND
+    "${TARGET_RUNNER}"
+    "${TOOLCHAIN_CONFIG}"
+    ${KEYS}
+)

diff --git a/compiler/tf2tflite-dredd-pbtxt-test/requires.cmake b/compiler/tf2tflite-dredd-pbtxt-test/requires.cmake
index 9fa4bf9..c077b66 100644 (file)
--- a/compiler/tf2tflite-dredd-pbtxt-test/requires.cmake
+++ b/compiler/tf2tflite-dredd-pbtxt-test/requires.cmake
@@ -1 +1,3 @@
-# TODO write this
+require("tfkit")
+require("tf2tflite")
+require("tfl-inspect")

diff --git a/compiler/tf2tflite-dredd-pbtxt-test/rule-lib.sh b/compiler/tf2tflite-dredd-pbtxt-test/rule-lib.sh
new file mode 100755 (executable)
index 0000000..9fa4bf9
--- /dev/null
+++ b/compiler/tf2tflite-dredd-pbtxt-test/rule-lib.sh
@@ -0,0 +1 @@
+# TODO write this

diff --git a/compiler/tf2tflite-dredd-pbtxt-test/runner.sh b/compiler/tf2tflite-dredd-pbtxt-test/runner.sh
new file mode 100755 (executable)
index 0000000..f84137e
--- /dev/null
+++ b/compiler/tf2tflite-dredd-pbtxt-test/runner.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# This script checks tflite file generated by tf2tflite
+
+WORKDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+# Need at least toolchain.config
+if [[ $# -lt 1 ]]; then
+  echo "USAGE: $0 ..."
+  echo
+  echo "ARGUMENTS:"
+  echo "  [toolchain.config path]"
+  echo "  [Prefix1]"
+  echo "  [Prefix2]"
+  echo "  ..."
+  exit 255
+fi
+
+CONFIG_PATH="$1"; shift
+
+source "${CONFIG_PATH}"
+
+echo "-- Found tfl-inspect: ${TFL_INSPECT_PATH}"
+echo "-- Found tf2tflite: ${TF2TFLITE_PATH}"
+echo "-- Found workdir: ${WORKDIR}"
+
+TESTED=()
+PASSED=()
+FAILED=()
+
+pushd "${WORKDIR}"
+
+# loading RULE and functions that checks tflite file
+source rule-lib.sh
+
+while [[ $# -ne 0 ]]; do
+  PREFIX="$1"; shift
+
+  echo "[ RUN      ] ${PREFIX}"
+
+  TESTED+=("${PREFIX}")
+
+  PASSED_TAG="${PREFIX}.passed"
+
+  rm -f "${PASSED_TAG}"
+
+  cat > "${PREFIX}.log" <(
+    exec 2>&1
+
+    source "${PREFIX}.test"
+
+    echo "-- Use '${MODEL_PB_PATH}', '${MODEL_INFO_PATH}', and '${MODEL_RULE_PATH}'"
+
+    # Exit immediately if any command fails
+    set -e
+    # Show commands
+    set -x
+
+    # Generate tflite
+    "${TF2TFLITE_PATH}" \
+      "${MODEL_INFO_PATH}" \
+      "${MODEL_PB_PATH}" \
+      "${WORKDIR}/${PREFIX}.tflite"
+
+    #
+    # Run rule prepared to check tflite file
+    #
+
+    # set vars needed by rule file
+    TFLITE_PATH="${WORKDIR}/${PREFIX}.tflite"
+
+    # Note: turn off 'command printing'. Otherwise printing will be so messy
+    set +x
+    source "${MODEL_RULE_PATH}"
+    set -x
+
+    if [[ $? -eq 0 ]]; then
+      touch "${PASSED_TAG}"
+    fi
+  )
+
+  if [[ -f "${PASSED_TAG}" ]]; then
+    echo "[       OK ] ${PREFIX}"
+    PASSED+=("$PREFIX")
+  else
+    echo "[      FAIL] ${PREFIX}"
+    FAILED+=("$PREFIX")
+  fi
+done
+popd
+
+if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then
+  echo "FAILED"
+  for TEST in "${FAILED[@]}"
+  do
+    echo "- ${TEST}"
+  done
+  exit 255
+fi
+
+echo "PASSED"
+exit 0

diff --git a/compiler/tf2tflite-dredd-pbtxt-test/test.lst b/compiler/tf2tflite-dredd-pbtxt-test/test.lst
new file mode 100644 (file)
index 0000000..5699a2c
--- /dev/null
+++ b/compiler/tf2tflite-dredd-pbtxt-test/test.lst
@@ -0,0 +1,2 @@
+# TODO Add NET_0025 like the following
+# Add(NET_0025 RULE test.rule)  # CBR