From 899cc09e5591a0b46603fb75fa032f32a887f631 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EB=B0=95=EC=A2=85=ED=98=84/=EB=8F=99=EC=9E=91=EC=A0=9C?= =?utf8?q?=EC=96=B4Lab=28SR=29/Staff=20Engineer/=EC=82=BC=EC=84=B1?= =?utf8?q?=EC=A0=84=EC=9E=90?= Date: Tue, 28 Aug 2018 09:50:17 +0900 Subject: [PATCH] [enco] Introduce caffe-based test framework (#1189) This commit implementes caffe-based random test framework for enco. Signed-off-by: Jonghyun Park --- contrib/CMakeLists.txt | 6 +- contrib/enco/test/caffe/000/test.prototxt | 8 ++ contrib/enco/test/caffe/CMakeLists.txt | 77 +++++++++++++ contrib/enco/test/caffe/binder.cpp | 172 ++++++++++++++++++++++++++++++ contrib/enco/test/caffe/runall | 66 ++++++++++++ 5 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 contrib/enco/test/caffe/000/test.prototxt create mode 100644 contrib/enco/test/caffe/CMakeLists.txt create mode 100644 contrib/enco/test/caffe/binder.cpp create mode 100755 contrib/enco/test/caffe/runall diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 5ea6cda..b034c86 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1 +1,5 @@ -add_subdirectories() +# NOTE Add caffegen, nnkit, and ann before enco as enco's test framework depends on these projects +add_subdirectory(caffegen) +add_subdirectory(nnkit) +add_subdirectory(ann) +add_subdirectories(EXCLUDES caffegen nnkit ann) diff --git a/contrib/enco/test/caffe/000/test.prototxt b/contrib/enco/test/caffe/000/test.prototxt new file mode 100644 index 0000000..2d05713 --- /dev/null +++ b/contrib/enco/test/caffe/000/test.prototxt @@ -0,0 +1,8 @@ +layer { + name: "data" + type: "Input" + top: "data" + input_param { + shape: { dim: 1 dim: 3 dim: 244 dim: 244 } + } +} diff --git a/contrib/enco/test/caffe/CMakeLists.txt b/contrib/enco/test/caffe/CMakeLists.txt new file mode 100644 index 0000000..dcb53d0 --- /dev/null +++ b/contrib/enco/test/caffe/CMakeLists.txt @@ -0,0 +1,77 @@ +if(NOT TARGET caffegen) + return() +endif(NOT TARGET caffegen) + +if(NOT TARGET enco_caffe_frontend) + return() +endif(NOT TARGET enco_caffe_frontend) + +if(NOT TARGET ann_ref_static) + return() +endif(NOT TARGET ann_ref_static) + +find_program(H5DIFF h5diff) + +if (NOT H5DIFF) + return() +endif(NOT H5DIFF) + +message(STATUS "Enable enco(caffe) test") + +file(GLOB MODELS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "*/test.prototxt") + +foreach(MODEL IN ITEMS ${MODELS}) + get_filename_component(PREFIX ${MODEL} DIRECTORY) + + set(MODEL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${MODEL}) + + set(PROTOTXT_FILE ${PREFIX}.prototxt) + set(CAFFEMODEL_FILE ${PREFIX}.caffemodel) + set(CAFFEMODEL_TARGET enco_caffe_test_${PREFIX}_caffemodel) + set(SOURCE_FILE ${PREFIX}.cpp) + set(SOURCE_TARGET enco_caffe_test_${PREFIX}_generated) + set(BINDER_TARGET enco_caffe_test_${PREFIX}_binder) + + # Copy prototxt + file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${PROTOTXT_FILE} INPUT ${MODEL_FILE}) + + # Generate caffemodel + add_custom_target(${CAFFEMODEL_TARGET} + ALL cat ${PROTOTXT_FILE} + | $ init + | $ encode + > ${CAFFEMODEL_FILE} + DEPENDS caffegen) + + # Generate C++ code + add_custom_target(${SOURCE_TARGET} + ALL $ + --frontend $ + --frontend-arg ${PROTOTXT_FILE} + --frontend-arg ${CAFFEMODEL_FILE} + > ${SOURCE_FILE} + DEPENDS enco-cli enco_caffe_frontend) + set_source_files_properties(${SOURCE_FILE} PROPERTIES GENERATED TRUE) + + # Compile nnkit binder (from generated C++ code) + add_library(${BINDER_TARGET} SHARED binder.cpp ${SOURCE_FILE}) + target_link_libraries(${BINDER_TARGET} nnkit_intf_backend) + target_link_libraries(${BINDER_TARGET} ann_ref_static) + set_target_properties(${BINDER_TARGET} PROPERTIES OUTPUT_NAME ${PREFIX}) + add_dependencies(${BINDER_TARGET} ${SOURCE_TARGET}) + + list(APPEND TESTS ${PREFIX}) +endforeach(MODEL) + +# Run tests +add_test(NAME enco_test_caffe + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/runall" + $ + $ + $ + $ + $ + "${CMAKE_CURRENT_BINARY_DIR}" + ${TESTS}) diff --git a/contrib/enco/test/caffe/binder.cpp b/contrib/enco/test/caffe/binder.cpp new file mode 100644 index 0000000..37a848e --- /dev/null +++ b/contrib/enco/test/caffe/binder.cpp @@ -0,0 +1,172 @@ +// +// Generated API +// +struct Network; + +Network *Network_construct(); +void Network_destruct(Network *net); + +unsigned Network_input_count(const Network *); +const char *Network_input_name(const Network *, unsigned n); +unsigned Network_input_rank(const Network *, unsigned n); +unsigned Network_input_dim(const Network *, unsigned n, unsigned axis); +void Network_input_bind(Network *net, unsigned n, const void *ptr, unsigned len); + +unsigned Network_output_count(const Network *net); +const char *Network_output_name(const Network *, unsigned n); +unsigned Network_output_rank(const Network *, unsigned n); +unsigned Network_output_dim(const Network *, unsigned n, unsigned axis); +void Network_output_bind(Network *net, unsigned n, void *ptr, unsigned len); + +void Network_invoke(Network *net); + +// +// nnkit backend +// +#include +#include +#include + +#include +#include + +#include + +using nncc::foundation::make_unique; +using namespace nncc::core::ADT; + +namespace +{ + +class TensorContext final : public nnkit::TensorContext +{ +public: + TensorContext() = default; + +public: + void allocate(const std::string &name, const tensor::Shape &shape) + { + using nncc::core::ADT::tensor::num_elements; + + auto blob = make_unique>(); + blob->resize(num_elements(shape) * sizeof(float)); + + _names.emplace_back(name); + _shapes.emplace_back(shape); + _blobs.emplace_back(std::move(blob)); + } + +public: + uint8_t *base(uint32_t n) const { return _blobs.at(n)->data(); } + +public: + uint32_t size(void) const override { return _blobs.size(); } + +public: + std::string name(uint32_t n) const override { return _names.at(n); } + +public: + tensor::Shape shape(uint32_t n) const override { return _shapes.at(n); } + +public: + uint32_t size(uint32_t n) const { return _blobs.at(n)->size(); } + +public: + // Float (fp32) tensor support + bool isFloatTensor(uint32_t n) const override { return true; } + void getMutableFloatTensor(uint32_t n, const TensorContext::TypedAccessor &f) override + { + using nncc::core::ADT::tensor::LexicalLayout; + using nncc::core::ADT::tensor::make_overlay; + + auto base = reinterpret_cast(this->base(n)); + auto view = make_overlay(shape(n), base); + + f(*this, n, view); + } + + void getConstFloatTensor(uint32_t n, const TensorContext::TypedReader &f) const override + { + using nncc::core::ADT::tensor::LexicalLayout; + using nncc::core::ADT::tensor::make_overlay; + + auto base = reinterpret_cast(this->base(n)); + auto view = make_overlay(shape(n), base); + + f(*this, n, view); + } + +private: + std::vector _names; + std::vector _shapes; + std::vector>> _blobs; +}; + +class Backend final : public nnkit::Backend +{ +public: + Backend() + { + _net = Network_construct(); + + // Allocate and bind inputs + for (uint32_t n = 0; n < Network_input_count(_net); ++n) + { + const uint32_t rank = Network_input_rank(_net, n); + const std::string name = Network_input_name(_net, n); + + tensor::Shape shape; + + shape.resize(rank); + for (uint32_t axis = 0; axis < rank; ++axis) + { + shape.dim(axis) = Network_input_dim(_net, n, axis); + } + + _inputs.allocate(name, shape); + + Network_input_bind(_net, n, reinterpret_cast(_inputs.base(n)), _inputs.size(n)); + } + + // Allocate and bind outputs + for (uint32_t n = 0; n < Network_output_count(_net); ++n) + { + const uint32_t rank = Network_output_rank(_net, n); + const std::string name = Network_output_name(_net, n); + + tensor::Shape shape; + + shape.resize(rank); + for (uint32_t axis = 0; axis < rank; ++axis) + { + shape.dim(axis) = Network_output_dim(_net, n, axis); + } + + _outputs.allocate(name, shape); + + Network_output_bind(_net, n, reinterpret_cast(_outputs.base(n)), _outputs.size(n)); + } + } + +public: + ~Backend() { Network_destruct(_net); } + +public: + void prepare(const std::function &f) override { f(_inputs); } + void run(void) override { Network_invoke(_net); } + void teardown(const std::function &f) override { f(_outputs); } + +private: + Network *_net; + +private: + TensorContext _inputs; + TensorContext _outputs; +}; + +} // namespace + +extern "C" std::unique_ptr make_backend(const nnkit::CmdlineArguments &args) +{ + return nncc::foundation::make_unique<::Backend>(); +} diff --git a/contrib/enco/test/caffe/runall b/contrib/enco/test/caffe/runall new file mode 100755 index 0000000..592697c --- /dev/null +++ b/contrib/enco/test/caffe/runall @@ -0,0 +1,66 @@ +#!/bin/bash + +if [[ $# -le 6 ]]; then + echo "USAGE: $0 [nni path] [reference backend path] [randomize action path] [HDF5 export action path] [HDF5 import action path] [WORKDIR] ..." + exit 255 +fi + +NNI_PATH="$1"; shift +REFERENCE_BACKEND_PATH="$1"; shift +RANDOMIZE_ACTION_PATH="$1"; shift +HDF5_EXPORT_ACTION_PATH="$1"; shift +HDF5_IMPORT_ACTION_PATH="$1"; shift +WORKDIR="$1"; shift + +echo "-- Found nni: ${NNI_PATH}" +echo "-- Found reference backend: ${REFERENCE_BACKEND_PATH}" +echo "-- Found randomize action: ${RANDOMIZE_ACTION_PATH}" +echo "-- Found HDF5 export action: ${HDF5_EXPORT_ACTION_PATH}" +echo "-- Found HDF5 import action: ${HDF5_IMPORT_ACTION_PATH}" +echo "-- Found workdir: ${WORKDIR}" + +TESTED=() +PASSED=() + +pushd "${WORKDIR}" +while [[ $# -ne 0 ]]; do + PREFIX="$1"; shift + + TESTED+=("${PREFIX}") + + echo "-- Found prototxt: ${PREFIX}.prototxt" + echo "-- Found caffemodel: ${PREFIX}.caffemodel" + echo "-- Found backend: lib${PREFIX}.so" + + "${NNI_PATH}" \ + --backend "${REFERENCE_BACKEND_PATH}" \ + --backend-arg "${WORKDIR}/${PREFIX}.prototxt" \ + --backend-arg "${WORKDIR}/${PREFIX}.caffemodel" \ + --pre "${RANDOMIZE_ACTION_PATH}" \ + --pre "${HDF5_EXPORT_ACTION_PATH}" \ + --pre-arg "${PREFIX}.input.h5" \ + --post "${HDF5_EXPORT_ACTION_PATH}" \ + --post-arg "${PREFIX}.expected.h5" + + "${NNI_PATH}" \ + --backend "./lib${PREFIX}.so" \ + --pre "${HDF5_IMPORT_ACTION_PATH}" \ + --pre-arg "${PREFIX}.input.h5" \ + --post "${HDF5_EXPORT_ACTION_PATH}" \ + --post-arg "${PREFIX}.obtained.h5" + + h5diff -d 0.001 "${PREFIX}.expected.h5" "${PREFIX}.obtained.h5" + + if [[ $? -eq 0 ]]; then + PASSED+=("$PREFIX") + fi +done +popd + +if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then + echo "FAILED" + exit 255 +fi + +echo "PASSED" +exit 0 -- 2.7.4