[tflchef] Introduce test framework (#2032)

* [tflchef] Introduce test framework

This commit introduces a simple test framework for tflchef.

Signed-off-by: Jonghyun Park <jh1302.park@samsung.com>
* Update comment

* Rename testcase to make it easy to understand

diff --git a/contrib/tflchef/CMakeLists.txt b/contrib/tflchef/CMakeLists.txt
index e90b026..f5fea28 100644 (file)
--- a/contrib/tflchef/CMakeLists.txt
+++ b/contrib/tflchef/CMakeLists.txt
@@ -20,3 +20,4 @@ add_subdirectory(proto)
 add_subdirectory(core)
 # Tools
 add_subdirectory(tools)
+add_subdirectory(tests)

diff --git a/contrib/tflchef/tests/CMakeLists.txt b/contrib/tflchef/tests/CMakeLists.txt
new file mode 100644 (file)
index 0000000..1d8cb1e
--- /dev/null
+++ b/contrib/tflchef/tests/CMakeLists.txt
@@ -0,0 +1,40 @@
+if(NOT TARGET nni)
+  return()
+endif(NOT TARGET nni)
+
+if(NOT TARGET nnkit_tflite_backend)
+  return()
+endif(NOT TARGET nnkit_tflite_backend)
+
+file(GLOB RECIPES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*/test.recipe")
+
+foreach(RECIPE IN ITEMS ${RECIPES})
+  get_filename_component(RECIPE_PREFIX ${RECIPE} DIRECTORY)
+
+  set(RECIPE_SOURCE_FILE "${RECIPE_PREFIX}.recipe")
+  set(RECIPE_SOURCE_TARGET tflchef_${PREFIX}_recipe)
+
+  set(RECIPE_OUTPUT_FILE "${RECIPE_PREFIX}.tflite")
+  set(RECIPE_OUTPUT_TARGET tflchef_${PREFIX}_tflite)
+
+  # Copy .recipe
+  add_custom_target(${RECIPE_SOURCE_TARGET}
+                    ALL ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/${RECIPE}"
+                                                 "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE_SOURCE_FILE}"
+                    COMMENT "Generate ${RECIPE_PREFIX}.recipe")
+
+  # Generate .tflite
+  add_custom_target(${RECIPE_OUTPUT_TARGET}
+                    ALL $<TARGET_FILE:tflchef-file> ${RECIPE_SOURCE_FILE} ${RECIPE_OUTPUT_FILE}
+                    DEPENDS ${RECIPE_SOURCE_TARGET}
+                    COMMENT "Generate ${RECIPE_PREFIX}.tflite")
+
+  list(APPEND TESTS ${RECIPE_PREFIX})
+endforeach(RECIPE)
+
+add_test(NAME tflchef_test
+         COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/runall"
+                 $<TARGET_FILE:nni>
+                 $<TARGET_FILE:nnkit_tflite_backend>
+                 "${CMAKE_CURRENT_BINARY_DIR}"
+                 ${TESTS})

diff --git a/contrib/tflchef/tests/readme/test.recipe b/contrib/tflchef/tests/readme/test.recipe
new file mode 100644 (file)
index 0000000..36e13d3
--- /dev/null
+++ b/contrib/tflchef/tests/readme/test.recipe
@@ -0,0 +1,39 @@
+operand {
+  name: "ifm"
+  type: FLOAT32
+  shape { dim: 1 dim: 3 dim: 3 dim: 2 }
+}
+operand {
+  name: "ker"
+  type: FLOAT32
+  shape { dim: 1 dim: 1 dim: 1 dim: 2 }
+}
+operand {
+  name: "bias"
+  type: FLOAT32
+  shape { dim: 1 }
+  filler {
+    tag: "constant"
+    arg: "1.1"
+  }
+}
+operand {
+  name: "ofm"
+  type: FLOAT32
+  shape { dim: 1 dim: 3 dim: 3 dim: 1 }
+}
+operation {
+  type: "Conv2D"
+  conv2d_options {
+    padding: VALID
+    stride_w: 1
+    stride_h: 1
+  }
+  input: "ifm"
+  input: "ker"
+  input: "bias"
+  output: "ofm"
+}
+input: "ifm"
+input: "ker"
+output: "ofm"

diff --git a/contrib/tflchef/tests/runall b/contrib/tflchef/tests/runall
new file mode 100755 (executable)
index 0000000..a1f5e0a
--- /dev/null
+++ b/contrib/tflchef/tests/runall
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if [[ $# -le 3 ]]; then
+  echo "USAGE: $0 [nni path] [tflite backend path] [working directory] [prefix 0] [prefix 1] ..."
+  exit 255
+fi
+
+NNI_PATH="$1"; shift
+TFLITE_BACKEND_PATH="$1"; shift
+WORKDIR="$1"; shift
+
+echo "-- Found nni: ${NNI_PATH}"
+echo "-- Found tflite backend: ${TFLITE_BACKEND_PATH}"
+echo "-- Found workdir: ${WORKDIR}"
+
+TESTED=()
+PASSED=()
+FAILED=()
+
+pushd "${WORKDIR}"
+while [[ $# -ne 0 ]]; do
+  PREFIX="$1"; shift
+
+  TESTED+=("${PREFIX}")
+
+  "${NNI_PATH}" --backend "${TFLITE_BACKEND_PATH}" --backend-arg "${PREFIX}.tflite"
+
+  if [[ $? -eq 0 ]]; then
+    PASSED+=("$PREFIX")
+  else
+    FAILED+=("$PREFIX")
+  fi
+done
+popd
+
+echo "SUMMARY: ${#PASSED[@]} PASS AND ${#FAILED[@]} FAIL AMONG ${#TESTED[@]} TESTS"
+
+if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then
+  exit 255
+fi
+
+exit 0