From 8da8d9199dec92589eb07689a42099bd7a591121 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EB=B0=95=EC=B2=9C=EA=B5=90/On-Device=20Lab=28SR=29/Enginee?= =?utf8?q?r/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Tue, 5 Nov 2019 09:20:25 +0900 Subject: [PATCH] Introduce moco-value-pbtxt-test (#8738) This commit introduces moco value test for pbtxt input model. It copies existing moco-integration-test only with renaming. Signed-off-by: Cheongyo Bahk --- compiler/moco-value-pbtxt-test/.gitignore | 1 + compiler/moco-value-pbtxt-test/CMakeLists.txt | 136 ++++++++++++++++++++++++++ compiler/moco-value-pbtxt-test/README.md | 1 + compiler/moco-value-pbtxt-test/requires.cmake | 2 + compiler/moco-value-pbtxt-test/runall.sh | 96 ++++++++++++++++++ compiler/moco-value-pbtxt-test/test.lst | 100 +++++++++++++++++++ 6 files changed, 336 insertions(+) create mode 100644 compiler/moco-value-pbtxt-test/.gitignore create mode 100644 compiler/moco-value-pbtxt-test/CMakeLists.txt create mode 100644 compiler/moco-value-pbtxt-test/README.md create mode 100644 compiler/moco-value-pbtxt-test/requires.cmake create mode 100755 compiler/moco-value-pbtxt-test/runall.sh create mode 100644 compiler/moco-value-pbtxt-test/test.lst diff --git a/compiler/moco-value-pbtxt-test/.gitignore b/compiler/moco-value-pbtxt-test/.gitignore new file mode 100644 index 0000000..8dbfa90 --- /dev/null +++ b/compiler/moco-value-pbtxt-test/.gitignore @@ -0,0 +1 @@ +/test.local.lst diff --git a/compiler/moco-value-pbtxt-test/CMakeLists.txt b/compiler/moco-value-pbtxt-test/CMakeLists.txt new file mode 100644 index 0000000..a469c20 --- /dev/null +++ b/compiler/moco-value-pbtxt-test/CMakeLists.txt @@ -0,0 +1,136 @@ +option(MOCO_VALUE_PBTXT_TEST "Enable moco value test for pbtxt input model" ON) + +if(NOT MOCO_VALUE_PBTXT_TEST) + return() +endif(NOT MOCO_VALUE_PBTXT_TEST) + +if(NOT TARGET tfkit) + message(STATUS "moco: Skip test material preparation as tfkit is not defined") + return() +endif(NOT TARGET tfkit) + +nncc_find_resource(TensorFlowTests) + +# +# Copy [Testcase]/test.pbtxt to Testcase.pbtxt in binary folder +# Copy [Testcase]/test.info to Testcase.info in binary folder +# Encode Testcase.pbtxt to Testcase.pb +# +set(TEST_PBTXT_FILE "test.pbtxt") +set(TEST_INFO_FILE "test.info") +set(TEST_REPO "${TensorFlowTests_DIR}") # Where to find tests +set(TEST_SPACE "${CMAKE_CURRENT_BINARY_DIR}") # Where to run tests + +unset(TESTCASES) + +macro(add NAME) + list(APPEND TESTCASES ${NAME}) +endmacro(add) + +# Read "test.lst" +include("test.lst") +# Read "test.local.lst" if exists +include("test.local.lst" OPTIONAL) + +unset(MOCO_VALUE_PBTXT_DEPS) + +foreach(PREFIX IN ITEMS ${TESTCASES}) + set(PBTXTFILE "${PREFIX}/${TEST_PBTXT_FILE}") + get_filename_component(DIR_NAME ${PBTXTFILE} DIRECTORY) + + set(PBTXT_SOURCE_FILE "${DIR_NAME}.pbtxt") + set(PBTXT_SOURCE_PATH "${TEST_SPACE}/${DIR_NAME}.pbtxt") + + set(PBTXT_INFO_FILE "${DIR_NAME}.info") + set(PBTXT_INFO_PATH "${TEST_SPACE}/${PBTXT_INFO_FILE}") + + set(PB_OUTPUT_FILE "${DIR_NAME}.pb") + set(PB_PATH "${TEST_SPACE}/${PB_OUTPUT_FILE}") + + # Copy files + add_custom_command( + OUTPUT ${PBTXT_SOURCE_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TEST_REPO}/${DIR_NAME}/${TEST_PBTXT_FILE}" + "${PBTXT_SOURCE_PATH}" + DEPENDS "${TEST_REPO}/${DIR_NAME}/${TEST_PBTXT_FILE}" + COMMENT "Copy ${PBTXT_SOURCE_FILE}" + ) + + add_custom_command( + OUTPUT ${PBTXT_INFO_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TEST_REPO}/${DIR_NAME}/${TEST_INFO_FILE}" + "${PBTXT_INFO_PATH}" + DEPENDS "${TEST_REPO}/${DIR_NAME}/${TEST_INFO_FILE}" + COMMENT "Copy ${PBTXT_INFO_FILE}" + ) + + # Use tfkit to encode + add_custom_command( + OUTPUT ${PB_OUTPUT_FILE} + COMMAND $ encode ${PBTXT_SOURCE_PATH} ${PB_OUTPUT_FILE} + DEPENDS tfkit ${PBTXT_SOURCE_PATH} + COMMENT "Generate ${PB_OUTPUT_FILE}" + ) + + list(APPEND MOCO_VALUE_PBTXT_TESTS ${DIR_NAME}) + list(APPEND MOCO_VALUE_PBTXT_DEPS ${PBTXT_INFO_FILE} ${PB_OUTPUT_FILE}) + +endforeach(PREFIX) + +nnas_find_package(TensorFlow QUIET) +if(NOT TensorFlow_FOUND) + message(STATUS "moco: Skip adding test as TensorFlow is not found") + return() +endif(NOT TensorFlow_FOUND) + +## +## Copy runall.sh +## +set(TEST_RUNNER_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/runall.sh") +set(TEST_RUNNER "${CMAKE_CURRENT_BINARY_DIR}/run-tests") + +add_custom_command( + OUTPUT ${TEST_RUNNER} + COMMAND ${CMAKE_COMMAND} -E copy "${TEST_RUNNER_SOURCE}" "${TEST_RUNNER}" + DEPENDS ${TEST_RUNNER_SOURCE} + COMMENT "Generate test runner" +) + +list(APPEND MOCO_VALUE_PBTXT_DEPS "${TEST_RUNNER}") + +### +### Generate test.config +### +set(TOOLCHIAN_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/toolchain.config") + +add_custom_command( + OUTPUT ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E remove -f ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'NNKIT_RUN_PATH=\"$\"' >> ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'TF_BACKEND_PATH=\"$\"' >> ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'MOCO_TF_BACKEND_PATH=\"$\"' >> ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'TF2TFLITE_PATH=\"$\"' >> ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'RANDOMIZE_ACTION_PATH=\"$\"' >> ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'HDF5_EXPORT_ACTION_PATH=\"$\"' >> ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'HDF5_IMPORT_ACTION_PATH=\"$\"' >> ${TOOLCHIAN_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'I5DIFF_PATH=\"$\"' >> ${TOOLCHIAN_CONFIG} + DEPENDS + nnkit-run + nnkit_tf_backend + nnkit_moco_tf_backend + tf2tflite + nnkit_randomize_action + nnkit_HDF5_export_action + nnkit_HDF5_import_action + i5diff + COMMENT "Generate test configuration" +) + +list(APPEND MOCO_VALUE_PBTXT_DEPS "${TOOLCHIAN_CONFIG}") + +# This target enforces CMake to generate all the dependencies during "build" phase +add_custom_target(moco_value_pbtxt_test_deps ALL DEPENDS ${MOCO_VALUE_PBTXT_DEPS}) + +# Run tests +add_test(NAME moco_value_pbtxt_test + COMMAND "${TEST_RUNNER}" "${TOOLCHIAN_CONFIG}" "${TEST_SPACE}" ${MOCO_VALUE_PBTXT_TESTS}) diff --git a/compiler/moco-value-pbtxt-test/README.md b/compiler/moco-value-pbtxt-test/README.md new file mode 100644 index 0000000..f5d1ac2 --- /dev/null +++ b/compiler/moco-value-pbtxt-test/README.md @@ -0,0 +1 @@ +# moco-value-pbtxt-test diff --git a/compiler/moco-value-pbtxt-test/requires.cmake b/compiler/moco-value-pbtxt-test/requires.cmake new file mode 100644 index 0000000..771418f --- /dev/null +++ b/compiler/moco-value-pbtxt-test/requires.cmake @@ -0,0 +1,2 @@ +require("tfkit") +require("nnkit") diff --git a/compiler/moco-value-pbtxt-test/runall.sh b/compiler/moco-value-pbtxt-test/runall.sh new file mode 100755 index 0000000..ee43f1a --- /dev/null +++ b/compiler/moco-value-pbtxt-test/runall.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Need at least 2 arguments +if [[ $# -lt 2 ]]; then + echo "USAGE: $0 ..." + echo + echo "ARGUMENTS:" + echo " [test.config path]" + echo " [WORKDIR]" + echo " [Prefix1]" + echo " [Prefix2]" + echo " ..." + exit 255 +fi + +CONFIG_PATH="$1"; shift +WORKDIR="$1"; shift + +source "${CONFIG_PATH}" + +echo "-- starting moco test tf" +echo "-- Found nnkit-run: ${NNKIT_RUN_PATH}" +echo "-- Found TF backend: ${TF_BACKEND_PATH}" +echo "-- Found moco TF backend: ${MOCO_TF_BACKEND_PATH}" +echo "-- Found randomize action: ${RANDOMIZE_ACTION_PATH}" +echo "-- Found HDF5 export action: ${HDF5_EXPORT_ACTION_PATH}" +echo "-- Found HDF5 import action: ${HDF5_IMPORT_ACTION_PATH}" +echo "-- Found i5diff: ${I5DIFF_PATH}" +echo "-- Found workdir: ${WORKDIR}" + +TESTED=() +PASSED=() +FAILED=() + +pushd "${WORKDIR}" +while [[ $# -ne 0 ]]; do + PREFIX="$1"; shift + + TESTED+=("${PREFIX}") + + PASSED_TAG="${PREFIX}.passed" + + rm -f "${PASSED_TAG}" + + cat > "${PREFIX}.log" <( + exec 2>&1 + + echo "-- Found pb: ${PREFIX}.pb" + + # Show commands + set -x + "${NNKIT_RUN_PATH}" \ + --backend "${TF_BACKEND_PATH}" \ + --backend-arg "${WORKDIR}/${PREFIX}.pb" \ + --backend-arg "${WORKDIR}/${PREFIX}.info" \ + --pre "${RANDOMIZE_ACTION_PATH}" \ + --pre "${HDF5_EXPORT_ACTION_PATH}" \ + --pre-arg "${WORKDIR}/${PREFIX}.input.h5" \ + --post "${HDF5_EXPORT_ACTION_PATH}" \ + --post-arg "${WORKDIR}/${PREFIX}.expected.h5" + + "${NNKIT_RUN_PATH}" \ + --backend "${MOCO_TF_BACKEND_PATH}" \ + --backend-arg "${WORKDIR}/${PREFIX}.pb" \ + --backend-arg "${WORKDIR}/${PREFIX}.info" \ + --pre "${HDF5_IMPORT_ACTION_PATH}" \ + --pre-arg "${WORKDIR}/${PREFIX}.input.h5" \ + --post "${HDF5_EXPORT_ACTION_PATH}" \ + --post-arg "${WORKDIR}/${PREFIX}.obtained.h5" + + "${I5DIFF_PATH}" -d 0.001 "${PREFIX}.expected.h5" "${PREFIX}.obtained.h5" + + if [[ $? -eq 0 ]]; then + touch "${PASSED_TAG}" + fi + ) + + if [[ -f "${PASSED_TAG}" ]]; then + PASSED+=("$PREFIX") + else + FAILED+=("$PREFIX") + fi +done +popd + +if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then + echo "FAILED" + for TEST in "${FAILED[@]}" + do + echo "- ${TEST}" + done + exit 255 +fi + +echo "PASSED" +exit 0 diff --git a/compiler/moco-value-pbtxt-test/test.lst b/compiler/moco-value-pbtxt-test/test.lst new file mode 100644 index 0000000..e352966 --- /dev/null +++ b/compiler/moco-value-pbtxt-test/test.lst @@ -0,0 +1,100 @@ +add(NET_0000) +add(NET_0001) +add(NET_0002) +add(NET_0003) +add(NET_0004) +add(NET_0005) +add(NET_0006) +add(NET_0007) +add(NET_0008) +add(NET_0009) +add(NET_0010) +add(NET_0011) +add(NET_0012) +add(NET_0013) +add(NET_0014) +add(NET_0015) +add(NET_0016) +add(NET_0017) +add(NET_0018) +add(NET_0019) +add(NET_0020) +add(NET_0021) +add(NET_0022) +add(NET_0023) +add(NET_0024) +add(NET_0025) +add(NET_0026) +add(NET_0028) +add(NET_0029) +add(NET_0030) +add(NET_0031) +add(NET_0032) +add(NET_0033) +add(NET_0034) +add(NET_0035) +# add(NET_0036) +add(NET_0037) +add(NET_0038) +add(NET_0039) +add(NET_0040) +add(NET_0041) +add(REGRESSION_0000) +add(REGRESSION_0001) +add(REGRESSION_0002) +add(UNIT_Add_000) +add(UNIT_Add_001) +add(UNIT_Add_002) +add(UNIT_Add_004) +add(UNIT_Add_005) +add(UNIT_AvgPool_000) +add(UNIT_AvgPool_001) +add(UNIT_BiasAdd_000) +add(UNIT_BiasAdd_001) +add(UNIT_BiasAdd_002) +add(UNIT_ConcatV2_000) +add(UNIT_ConcatV2_001) +add(UNIT_ConcatV2_002) +add(UNIT_Const_000) +add(UNIT_Conv2D_000) +add(UNIT_Conv2DBackpropInput_000) +add(UNIT_Conv2DBackpropInput_001) +add(UNIT_DepthwiseConv2dNative_000) +add(UNIT_DepthwiseConv2dNative_001) +add(UNIT_FusedBatchNorm_000) +add(UNIT_FusedBatchNorm_001) +add(UNIT_MaxPool_000) +add(UNIT_MaxPool_001) +add(UNIT_Mean_000) +add(UNIT_Mean_001) +add(UNIT_Mean_002) +add(UNIT_Mean_003) +add(UNIT_Mul_000) +add(UNIT_Mul_001) +add(UNIT_Mul_002) +add(UNIT_Pad_000) +add(UNIT_Placeholder_000) +add(UNIT_Placeholder_001) +add(UNIT_Placeholder_002) +add(UNIT_Placeholder_003) +add(UNIT_RealDiv_000) +add(UNIT_Relu_000) +add(UNIT_Relu6_000) +add(UNIT_Reshape_000) +add(UNIT_Rsqrt_000) +add(UNIT_Softmax_000) +add(UNIT_Softmax_001) +add(UNIT_Softmax_002) +add(UNIT_Softmax_003) +add(UNIT_Sqrt_000) +add(UNIT_SquaredDifference_000) +add(UNIT_SquaredDifference_001) +add(UNIT_Squeeze_000) +add(UNIT_Squeeze_001) +add(UNIT_Squeeze_002) +add(UNIT_Squeeze_003) +add(UNIT_StopGradient_000) +add(UNIT_StopGradient_001) +add(UNIT_Sub_000) +add(UNIT_Sub_001) +add(UNIT_Tanh_000) -- 2.7.4