[tf2circle-dredd-pbtxt-test] Introducing tf2circle-dredd-pbtxt-test (#8882)
author윤현식/On-Device Lab(SR)/Principal Engineer/삼성전자 <hyunsik.yoon@samsung.com>
Mon, 11 Nov 2019 06:11:00 +0000 (15:11 +0900)
committer박종현/On-Device Lab(SR)/Staff Engineer/삼성전자 <jh1302.park@samsung.com>
Mon, 11 Nov 2019 06:11:00 +0000 (15:11 +0900)
* [tf2circle-dredd-pbtxt-test] Introducing tf2circle-dredd-pbtxt-test

tf2circle-dredd-pbtxt-test is introduced, which is rule base test for circle files generated from pbtxt files under `res/TensorFlowTests/`.

Signed-off-by: Hyun Sik Yoon <hyunsik.yoon@samsung.com>
* Update compiler/tf2circle-dredd-pbtxt-test/runner.sh

thanks, bot.

Co-Authored-By: hyunsik-yoon <hyunsik.yoon@samsung.com>
compiler/tf2circle-dredd-pbtxt-test/.gitignore [new file with mode: 0644]
compiler/tf2circle-dredd-pbtxt-test/CMakeLists.txt [new file with mode: 0644]
compiler/tf2circle-dredd-pbtxt-test/README.md [new file with mode: 0644]
compiler/tf2circle-dredd-pbtxt-test/requires.cmake [new file with mode: 0644]
compiler/tf2circle-dredd-pbtxt-test/runner.sh [new file with mode: 0755]
compiler/tf2circle-dredd-pbtxt-test/test.lst [new file with mode: 0644]

diff --git a/compiler/tf2circle-dredd-pbtxt-test/.gitignore b/compiler/tf2circle-dredd-pbtxt-test/.gitignore
new file mode 100644 (file)
index 0000000..8dbfa90
--- /dev/null
@@ -0,0 +1 @@
+/test.local.lst
diff --git a/compiler/tf2circle-dredd-pbtxt-test/CMakeLists.txt b/compiler/tf2circle-dredd-pbtxt-test/CMakeLists.txt
new file mode 100644 (file)
index 0000000..789e585
--- /dev/null
@@ -0,0 +1,184 @@
+nnas_include(TargetRequire)
+
+unset(REQUIRED_TARGETS)
+list(APPEND REQUIRED_TARGETS circle-inspect)
+list(APPEND REQUIRED_TARGETS circle-verify)
+list(APPEND REQUIRED_TARGETS tf2circle)
+list(APPEND REQUIRED_TARGETS tfkit)
+list(APPEND REQUIRED_TARGETS dredd_rule_lib)
+TargetRequire_Return(${REQUIRED_TARGETS})
+
+nncc_find_resource(TensorFlowTests)
+
+set(MODEL_REPO "${TensorFlowTests_DIR}") # Where to find text models to test
+
+unset(KEYS)
+unset(DEPS)
+
+#
+# processing models in test.lst and test.local.lst
+#
+# Example)
+#
+#  Add(NET_0025 RULE test.rule)
+#    -> Read test.pbtxt file under res/TensorFlowTests/NET_0025 and create "NET_0025.circle"
+#       Then the circle is tested against rules in test.rule file.
+#
+macro(Add MODEL_DIR)
+
+  set(ARG_OPTION)
+  set(ARG_ONE_VALUE RULE) # rule file name
+  set(ARG_MULTI_VALUE)
+  cmake_parse_arguments(ARG "${ARG_OPTION}" "${ARG_ONE_VALUE}" "${ARG_MULTI_VALUE}" ${ARGN})
+
+  if(NOT ARG_RULE)
+    message( FATAL_ERROR "RULE is mandadatory arg" )
+  endif()
+
+  set(RULE_FILENAME ${ARG_RULE})
+
+  set(TARGET_TESTNAME "${MODEL_DIR}")
+
+  set(MODEL_SOURCE_DIR "${MODEL_REPO}/${MODEL_DIR}")
+
+  set(TXT_SOURCE_PBTXT_PATH "${MODEL_SOURCE_DIR}/test.pbtxt")
+  set(TXT_SOURCE_INFO_PATH "${MODEL_SOURCE_DIR}/test.info")
+  set(TXT_SOURCE_RULE_PATH "${MODEL_SOURCE_DIR}/${RULE_FILENAME}")
+
+  set(TXT_TARGET_PB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.pb")
+  set(TXT_TARGET_PBTXT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.pbtxt")
+  set(TXT_TARGET_INFO_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.info")
+  set(TXT_TARGET_RULE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.rule")
+
+  if(NOT EXISTS "${TXT_SOURCE_PBTXT_PATH}")
+    message(FATAL_ERROR "${TXT_SOURCE_PBTXT_PATH} - pbtxt file does not exist")
+  endif(NOT EXISTS "${TXT_SOURCE_PBTXT_PATH}")
+
+  if(NOT EXISTS "${TXT_SOURCE_INFO_PATH}")
+    message(FATAL_ERROR "${TXT_SOURCE_INFO_PATH} - info file does not exist")
+  endif(NOT EXISTS "${TXT_SOURCE_INFO_PATH}")
+
+  if(NOT EXISTS "${TXT_SOURCE_RULE_PATH}")
+    message(FATAL_ERROR "${TXT_SOURCE_RULE_PATH} - rule file does not exist")
+  endif(NOT EXISTS "${TXT_SOURCE_RULE_PATH}")
+
+  # Copy .pbtxt
+  add_custom_command(OUTPUT ${TXT_TARGET_PBTXT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_PBTXT_PATH}" "${TXT_TARGET_PBTXT_PATH}"
+    DEPENDS ${TXT_SOURCE_PBTXT_PATH}
+    COMMENT "Generate ${TXT_TARGET_PBTXT_PATH}"
+  )
+
+  # Copy .info
+  add_custom_command(OUTPUT ${TXT_TARGET_INFO_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_INFO_PATH}" "${TXT_TARGET_INFO_PATH}"
+    DEPENDS ${TXT_SOURCE_INFO_PATH}
+    COMMENT "Generate ${TXT_TARGET_INFO_PATH}"
+  )
+
+  # Copy .rule
+  add_custom_command(OUTPUT ${TXT_TARGET_RULE_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy "${TXT_SOURCE_RULE_PATH}" "${TXT_TARGET_RULE_PATH}"
+    DEPENDS ${TXT_SOURCE_RULE_PATH}
+    COMMENT "Generate ${TXT_TARGET_RULE_PATH}"
+  )
+
+  # Generate .pb from .pbtxt
+  add_custom_command(OUTPUT ${TXT_TARGET_PB_PATH}
+    COMMAND $<TARGET_FILE:tfkit> encode ${TXT_TARGET_PBTXT_PATH} ${TXT_TARGET_PB_PATH}
+    DEPENDS ${TXT_TARGET_PBTXT_PATH}
+    COMMENT "Generate ${TXT_TARGET_PB_PATH}"
+  )
+
+  # Generate .test file which declares path of target pb, info, rule files
+  # this file is used inside runner.sh
+  set(TEST_CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_TESTNAME}.test")
+
+  add_custom_command(
+    OUTPUT ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_PB_PATH="${TXT_TARGET_PB_PATH}"' >> ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_INFO_PATH="${TXT_TARGET_INFO_PATH}"' >> ${TEST_CONFIG_FILE}
+    COMMAND ${CMAKE_COMMAND} -E echo 'MODEL_RULE_PATH="${TXT_TARGET_RULE_PATH}"' >> ${TEST_CONFIG_FILE}
+    DEPENDS
+      ${TXT_TARGET_PB_PATH}
+      ${TXT_TARGET_INFO_PATH}
+      ${TXT_TARGET_RULE_PATH}
+    COMMENT "Generate ${TARGET_TESTNAME} configuration for TXT"
+  )
+
+  list(APPEND DEPS "${TEST_CONFIG_FILE}")
+  list(APPEND KEYS "${TARGET_TESTNAME}")
+
+endmacro(Add)
+
+# Read "test.lst"
+include("test.lst")
+# Read "test.local.lst" if exists
+include("test.local.lst" OPTIONAL)
+
+#
+# Generate toolchain.config
+#
+set(TOOLCHAIN_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/toolchain.config")
+
+add_custom_command(
+  OUTPUT ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E remove -f ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_INSPECT_PATH=\"$<TARGET_FILE:circle-inspect>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_VERIFY_PATH=\"$<TARGET_FILE:circle-verify>\"' >> ${TOOLCHAIN_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TF2CIRCLE_PATH=\"$<TARGET_FILE:tf2circle>\"' >> ${TOOLCHAIN_CONFIG}
+  # add more if new excutable file is needed in runner.sh and rule-lib.sh
+  DEPENDS
+    circle-inspect
+    circle-verify
+    tf2circle
+  COMMENT "Generate toolchin configuration"
+)
+
+list(APPEND DEPS "${TOOLCHAIN_CONFIG}")
+
+#
+# copy runner.sh
+#
+set(SOURCE_RUNNER "${CMAKE_CURRENT_SOURCE_DIR}/runner.sh")
+set(TARGET_RUNNER "${CMAKE_CURRENT_BINARY_DIR}/runner.sh")
+
+add_custom_command(
+  OUTPUT ${TARGET_RUNNER}
+  COMMAND ${CMAKE_COMMAND} -E copy "${SOURCE_RUNNER}" "${TARGET_RUNNER}"
+  DEPENDS ${SOURCE_RUNNER}
+  COMMENT "Generate test runner"
+)
+
+list(APPEND DEPS "${TARGET_RUNNER}")
+
+#
+# copy rule-lib.sh (a library of shell script functions)
+#
+
+# getting path for rule-lib.sh in dredd-rule-lib
+get_target_property(DREDD_RULE_LIB_DIR dredd_rule_lib BINARY_DIR)
+
+set(SOURCE_RULE_LIB "${DREDD_RULE_LIB_DIR}/rule-lib.sh")
+set(TARGET_RULE_LIB "${CMAKE_CURRENT_BINARY_DIR}/rule-lib.sh")
+
+add_custom_command(
+  OUTPUT ${TARGET_RULE_LIB}
+  COMMAND ${CMAKE_COMMAND} -E copy "${SOURCE_RULE_LIB}" "${TARGET_RULE_LIB}"
+  DEPENDS ${SOURCE_RULE_LIB}
+  COMMENT "Generate rule lib"
+)
+
+list(APPEND DEPS "${TARGET_RULE_LIB}")
+
+# Generate dependencies
+add_custom_target(tf2circle_dredd_pbtxt_deps ALL DEPENDS ${DEPS})
+
+add_test(
+  NAME tf2circle_dredd_pbtxt_test
+  COMMAND
+    "${TARGET_RUNNER}"
+    "${TOOLCHAIN_CONFIG}"
+    ${KEYS}
+)
diff --git a/compiler/tf2circle-dredd-pbtxt-test/README.md b/compiler/tf2circle-dredd-pbtxt-test/README.md
new file mode 100644 (file)
index 0000000..8eb9061
--- /dev/null
@@ -0,0 +1,3 @@
+# tf2circle-dredd-pbtxt-test
+
+TODO write content.
diff --git a/compiler/tf2circle-dredd-pbtxt-test/requires.cmake b/compiler/tf2circle-dredd-pbtxt-test/requires.cmake
new file mode 100644 (file)
index 0000000..747d8ab
--- /dev/null
@@ -0,0 +1,5 @@
+require("tfkit")
+require("tf2circle")
+require("circle-inspect")
+require("circle-verify")
+require("dredd-rule-lib")
diff --git a/compiler/tf2circle-dredd-pbtxt-test/runner.sh b/compiler/tf2circle-dredd-pbtxt-test/runner.sh
new file mode 100755 (executable)
index 0000000..5745db1
--- /dev/null
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+# This script checks circle file generated by tf2circle
+
+WORKDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+# Need at least toolchain.config
+if [[ $# -lt 1 ]]; then
+  echo "USAGE: $0 ..."
+  echo
+  echo "ARGUMENTS:"
+  echo "  [toolchain.config path]"
+  echo "  [Prefix1]"
+  echo "  [Prefix2]"
+  echo "  ..."
+  exit 255
+fi
+
+CONFIG_PATH="$1"; shift
+
+source "${CONFIG_PATH}"
+
+echo "-- Found circle-inspect: ${CIRCLE_INSPECT_PATH}"
+echo "-- Found circle-verify: ${CIRCLE_VERIFY_PATH}"
+echo "-- Found tf2circle: ${TF2CIRCLE_PATH}"
+echo "-- Found workdir: ${WORKDIR}"
+
+TESTED=()
+PASSED=()
+FAILED=()
+
+pushd "${WORKDIR}"
+
+# running each rule file
+
+while [[ $# -ne 0 ]]; do
+  PREFIX="$1"; shift
+
+  echo "[ RUN      ] ${PREFIX}"
+
+  TESTED+=("${PREFIX}")
+
+  PASSED_TAG="${PREFIX}.passed"
+
+  rm -f "${PASSED_TAG}"
+
+  cat > "${PREFIX}.log" <(
+    exec 2>&1
+
+    source "${PREFIX}.test"
+
+    echo "-- Use '${MODEL_PB_PATH}', '${MODEL_INFO_PATH}', and '${MODEL_RULE_PATH}'"
+
+    # Exit immediately if any command fails
+    set -e
+    # Show commands
+    set -x
+
+    # Generate circle
+    "${TF2CIRCLE_PATH}" \
+      "${MODEL_INFO_PATH}" \
+      "${MODEL_PB_PATH}" \
+      "${WORKDIR}/${PREFIX}.circle"
+
+    #
+    # Run rule prepared to check circle file
+    #
+
+    # set vars needed by rule file
+    CIRCLE_PATH="${WORKDIR}/${PREFIX}.circle"
+
+    # Note: turn off 'command printing'. Otherwise printing will be so messy
+    set +x
+
+    # set vars required by rule-lib.sh and rule file
+    COMPILED_FILE=${CIRCLE_PATH}
+    INSPECT_PROG_PATH=${CIRCLE_INSPECT_PATH}
+    VERIFY_PROG_PATH=${CIRCLE_VERIFY_PATH}
+
+    source rule-lib.sh
+    source "${MODEL_RULE_PATH}"
+
+    set -x
+
+    if [[ $? -eq 0 ]]; then
+      touch "${PASSED_TAG}"
+    fi
+  )
+
+  if [[ -f "${PASSED_TAG}" ]]; then
+    echo "[       OK ] ${PREFIX}"
+    PASSED+=("$PREFIX")
+  else
+    echo "[      FAIL] ${PREFIX}"
+    FAILED+=("$PREFIX")
+  fi
+done
+popd
+
+if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then
+  echo "FAILED"
+  for TEST in "${FAILED[@]}"
+  do
+    echo "- ${TEST}"
+  done
+  exit 255
+fi
+
+echo "PASSED"
+exit 0
diff --git a/compiler/tf2circle-dredd-pbtxt-test/test.lst b/compiler/tf2circle-dredd-pbtxt-test/test.lst
new file mode 100644 (file)
index 0000000..185b3fc
--- /dev/null
@@ -0,0 +1,2 @@
+# TODO add tests like the following:
+# Add(NET_0030 RULE circle_1.0_rel_requirement.rule)  # Concat