From 729ddf3466dfb1e9dd06880390ae6e89874243dd Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EA=B9=80=EC=88=98=EC=A7=84/=EB=8F=99=EC=9E=91=EC=A0=9C?= =?utf8?q?=EC=96=B4Lab=28SR=29/Engineer/=EC=82=BC=EC=84=B1=EC=A0=84?= =?utf8?q?=EC=9E=90?= Date: Thu, 2 Aug 2018 16:29:37 +0900 Subject: [PATCH] [neurun] Insert TensorConvert nodes (#2127) * [neurun] Insert TensorConvert nodes This commit inserts TensorConvert nodes. Those will be inserted to between all of nodes. Signed-off-by: sjsujinkim sjsujin.kim@samsung.com * Change to plugin type * Fix build failure --- .../neurun/src/backend/acl_cl/TensorBuilder.cc | 33 +++++++++++++ runtimes/neurun/src/backend/acl_cl/TensorBuilder.h | 3 ++ runtimes/neurun/src/backend/cpu/TensorBuilder.cc | 33 +++++++++++++ runtimes/neurun/src/backend/cpu/TensorBuilder.h | 3 ++ runtimes/neurun/src/compilation.cc | 55 ++++++++++++++++++---- runtimes/neurun/src/compilation.h | 2 +- runtimes/neurun/src/frontend/compilation.cc | 2 +- runtimes/neurun/src/internal/ITensorBuilder.h | 18 +++++++ runtimes/neurun/src/internal/Model.h | 23 +++++++++ runtimes/neurun/src/internal/arm_compute.h | 5 +- .../neurun/src/internal/common/TensorBuilder.cc | 18 +++++++ .../neurun/src/internal/common/TensorBuilder.h | 3 ++ runtimes/neurun/src/model.h | 2 +- 13 files changed, 185 insertions(+), 15 deletions(-) diff --git a/runtimes/neurun/src/backend/acl_cl/TensorBuilder.cc b/runtimes/neurun/src/backend/acl_cl/TensorBuilder.cc index 3161503..5354139 100644 --- a/runtimes/neurun/src/backend/acl_cl/TensorBuilder.cc +++ b/runtimes/neurun/src/backend/acl_cl/TensorBuilder.cc @@ -23,6 +23,39 @@ void TensorBuilder::mark(const ::internal::tflite::operand::Index &ind) _inds.insert(ind.asInt()); } +void TensorBuilder::markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) +{ + _from_common_candidates.emplace_back(op, ind); +} + +void TensorBuilder::markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) +{ + _to_common_candidates.emplace_back(op, ind); +} + +void TensorBuilder::insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) +{ + for (auto param : _from_common_candidates) + { + auto index = operations.find(param.op); + if (index != -1) + { + operations.insert<::internal::tflite::op::TensorConvert::AclFromCommon::Node>( + index, ::internal::tflite::op::TensorConvert::AclFromCommon::Param(param.tensor_index)); + } + } + + for (auto param : _to_common_candidates) + { + auto index = operations.find(param.op); + if (index != -1) + { + operations.insert<::internal::tflite::op::TensorConvert::AclToCommon::Node>( + index + 1, ::internal::tflite::op::TensorConvert::AclToCommon::Param(param.tensor_index)); + } + } +} + void TensorBuilder::prepare(const std::map &tensor_info_ctx) { assert(_tensors.size() == 0); diff --git a/runtimes/neurun/src/backend/acl_cl/TensorBuilder.h b/runtimes/neurun/src/backend/acl_cl/TensorBuilder.h index 63b13d9..e0fce48 100644 --- a/runtimes/neurun/src/backend/acl_cl/TensorBuilder.h +++ b/runtimes/neurun/src/backend/acl_cl/TensorBuilder.h @@ -24,6 +24,9 @@ public: TensorBuilder(::internal::arm_compute::Plan &plan); virtual void mark(const ::internal::tflite::operand::Index &ind) override; + virtual void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) override; + virtual void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) override; + virtual void insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) override; virtual void prepare(const std::map &tensor_info_ctx) override; virtual void allocate(void) override; diff --git a/runtimes/neurun/src/backend/cpu/TensorBuilder.cc b/runtimes/neurun/src/backend/cpu/TensorBuilder.cc index 7b31bec..b907586 100644 --- a/runtimes/neurun/src/backend/cpu/TensorBuilder.cc +++ b/runtimes/neurun/src/backend/cpu/TensorBuilder.cc @@ -23,6 +23,39 @@ void TensorBuilder::mark(const ::internal::tflite::operand::Index &ind) _inds.insert(ind.asInt()); } +void TensorBuilder::markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) +{ + _from_common_candidates.emplace_back(op, ind); +} + +void TensorBuilder::markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) +{ + _to_common_candidates.emplace_back(op, ind); +} + +void TensorBuilder::insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) +{ + for (auto param : _from_common_candidates) + { + auto index = operations.find(param.op); + if (index != -1) + { + operations.insert<::internal::tflite::op::TensorConvert::CpuFromCommon::Node>( + index, ::internal::tflite::op::TensorConvert::CpuFromCommon::Param(param.tensor_index)); + } + } + + for (auto param : _to_common_candidates) + { + auto index = operations.find(param.op); + if (index != -1) + { + operations.insert<::internal::tflite::op::TensorConvert::CpuToCommon::Node>( + index + 1, ::internal::tflite::op::TensorConvert::CpuToCommon::Param(param.tensor_index)); + } + } +} + void TensorBuilder::prepare(const std::map &tensor_info_ctx) { assert(_tensors.size() == 0); diff --git a/runtimes/neurun/src/backend/cpu/TensorBuilder.h b/runtimes/neurun/src/backend/cpu/TensorBuilder.h index 5395161..de252e1 100644 --- a/runtimes/neurun/src/backend/cpu/TensorBuilder.h +++ b/runtimes/neurun/src/backend/cpu/TensorBuilder.h @@ -23,6 +23,9 @@ public: TensorBuilder(::internal::arm_compute::Plan &plan); virtual void mark(const ::internal::tflite::operand::Index &ind) override; + virtual void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) override; + virtual void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) override; + virtual void insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) override; virtual void prepare(const std::map &tensor_info_ctx) override; virtual void allocate(void) override; diff --git a/runtimes/neurun/src/compilation.cc b/runtimes/neurun/src/compilation.cc index f4d0603..009337d 100644 --- a/runtimes/neurun/src/compilation.cc +++ b/runtimes/neurun/src/compilation.cc @@ -389,22 +389,22 @@ void Planner::visit(const ::internal::tflite::op::Softmax::Node &node) void Planner::visit(const ::internal::tflite::op::TensorConvert::CpuFromCommon::Node &node) { - throw std::runtime_error("NYI - Planner::visit(TensorConvert::CpuFromCommon"); + VERBOSE(CpuFromCommon) << "Configure CpuFromCommon operation" << std::endl; } void Planner::visit(const ::internal::tflite::op::TensorConvert::CpuToCommon::Node &node) { - throw std::runtime_error("NYI - Planner::visit(TensorConvert::CpuToCommon"); + VERBOSE(CpuToCommon) << "Configure CpuToCommon operation" << std::endl; } void Planner::visit(const ::internal::tflite::op::TensorConvert::AclFromCommon::Node &node) { - throw std::runtime_error("NYI - Planner::visit(TensorConvert::FromCommon"); + VERBOSE(AclFromCommon) << "Configure AclFromCommon operation" << std::endl; } void Planner::visit(const ::internal::tflite::op::TensorConvert::AclToCommon::Node &node) { - throw std::runtime_error("NYI - Planner::visit(TensorConvert::AclToCommon"); + VERBOSE(AclToCommon) << "Configure AclToCommon operation" << std::endl; } class TensorMarker : public ::internal::tflite::op::NodeVisitor @@ -430,6 +430,14 @@ public: private: void mark(int32_t ind) { _tensor_builder.mark(::internal::tflite::operand::Index{ind}); } + void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) + { + _tensor_builder.markFromCommon(op, ind); + } + void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) + { + _tensor_builder.markToCommon(op, ind); + } private: ::internal::ITensorBuilder &_tensor_builder; @@ -442,6 +450,9 @@ void TensorMarker::visit(const ::internal::tflite::op::Conv2D::implicit::Node &n mark(param.ifm_index); mark(param.ker_index); mark(param.bias_index); + + markToCommon(node, param.ofm_index); + markFromCommon(node, param.ifm_index); } void TensorMarker::visit(const ::internal::tflite::op::MaxPool2D::implicit::Node &node) @@ -449,6 +460,9 @@ void TensorMarker::visit(const ::internal::tflite::op::MaxPool2D::implicit::Node const auto ¶m = node.param(); mark(param.ofm_index); mark(param.ifm_index); + + markToCommon(node, param.ofm_index); + markFromCommon(node, param.ifm_index); } void TensorMarker::visit(const ::internal::tflite::op::AvgPool2D::implicit::Node &node) @@ -456,16 +470,22 @@ void TensorMarker::visit(const ::internal::tflite::op::AvgPool2D::implicit::Node const auto ¶m = node.param(); mark(param.ofm_index); mark(param.ifm_index); + + markToCommon(node, param.ofm_index); + markFromCommon(node, param.ifm_index); } void TensorMarker::visit(const ::internal::tflite::op::Concat::Node &node) { const auto ¶m = node.param(); mark(param.ofm_index); + markToCommon(node, param.ofm_index); for (auto ind : param.ifm_indexes) { mark(ind); } + + markFromCommon(node, param.ifm_indexes[0]); } void TensorMarker::visit(const ::internal::tflite::op::FullyConnected::Node &node) @@ -475,6 +495,9 @@ void TensorMarker::visit(const ::internal::tflite::op::FullyConnected::Node &nod mark(param.input_index); mark(param.weight_index); mark(param.bias_index); + + markToCommon(node, param.output_index); + markFromCommon(node, param.input_index); } void TensorMarker::visit(const ::internal::tflite::op::Reshape::Node &node) @@ -482,6 +505,9 @@ void TensorMarker::visit(const ::internal::tflite::op::Reshape::Node &node) const auto ¶m = node.param(); mark(param.output_index); mark(param.input_index); + + markToCommon(node, param.output_index); + markFromCommon(node, param.input_index); } void TensorMarker::visit(const ::internal::tflite::op::Softmax::Node &node) @@ -489,6 +515,9 @@ void TensorMarker::visit(const ::internal::tflite::op::Softmax::Node &node) const auto ¶m = node.param(); mark(param.output_index); mark(param.input_index); + + markToCommon(node, param.output_index); + markFromCommon(node, param.input_index); } void TensorMarker::visit(const ::internal::tflite::op::TensorConvert::CpuFromCommon::Node &node) @@ -624,7 +653,7 @@ int ANeuralNetworksCompilation::finish() auto &plan = this->plan(); const auto &operands = plan.model().operands(); - const auto &operations = plan.model().operations(); + auto &operations = plan.model().operations(); ::internal::BackendManager backend_manager{plan}; BackendResolver backend_resolver{backend_manager}; @@ -632,18 +661,24 @@ int ANeuralNetworksCompilation::finish() for (uint32_t n = 0; n < operations.size(); ++n) { - operations.at(n).accept(Planner{operands, plan_builder, backend_resolver}); + const auto &op = operations.at(n); + auto tensor_builder = backend_resolver.getTensorBuilder(typeid(op)); + op.accept(TensorMarker{*tensor_builder}); } - // TODO Add optimization passes + auto tensor_builders = backend_resolver.getAllTensorBuilders(); + + for (auto tensor_builder : tensor_builders) + { + tensor_builder->insertTensorConvertNodes(operations); + } for (uint32_t n = 0; n < operations.size(); ++n) { - const auto &op = operations.at(n); - auto tensor_builder = backend_resolver.getTensorBuilder(typeid(op)); - op.accept(TensorMarker{*tensor_builder}); + operations.at(n).accept(Planner{operands, plan_builder, backend_resolver}); } + // TODO Add optimization passes plan_builder.finalize(backend_resolver); return ANEURALNETWORKS_NO_ERROR; diff --git a/runtimes/neurun/src/compilation.h b/runtimes/neurun/src/compilation.h index db6a5a1..0209798 100644 --- a/runtimes/neurun/src/compilation.h +++ b/runtimes/neurun/src/compilation.h @@ -7,7 +7,7 @@ struct ANeuralNetworksCompilation { public: - ANeuralNetworksCompilation(const std::shared_ptr &model) + ANeuralNetworksCompilation(const std::shared_ptr &model) : _plan{new internal::arm_compute::Plan{model}} { // DO NOTHING diff --git a/runtimes/neurun/src/frontend/compilation.cc b/runtimes/neurun/src/frontend/compilation.cc index fd871dc..e056e21 100644 --- a/runtimes/neurun/src/frontend/compilation.cc +++ b/runtimes/neurun/src/frontend/compilation.cc @@ -9,7 +9,7 @@ int ANeuralNetworksCompilation_create(ANeuralNetworksModel *model, ANeuralNetworksCompilation **compilation) { - std::shared_ptr internal; + std::shared_ptr internal; model->release(internal); diff --git a/runtimes/neurun/src/internal/ITensorBuilder.h b/runtimes/neurun/src/internal/ITensorBuilder.h index b3938f4..90e3f49 100644 --- a/runtimes/neurun/src/internal/ITensorBuilder.h +++ b/runtimes/neurun/src/internal/ITensorBuilder.h @@ -9,13 +9,31 @@ namespace internal { +struct TensorConversionParam +{ + TensorConversionParam(const ::internal::tflite::op::Node &op, int tensor_index) + : op(op), tensor_index(tensor_index) + { + } + + const ::internal::tflite::op::Node &op; + int tensor_index; +}; + struct ITensorBuilder { virtual ~ITensorBuilder(void) = default; virtual void mark(const ::internal::tflite::operand::Index &ind) = 0; + virtual void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) = 0; + virtual void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) = 0; + virtual void insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) = 0; // TODO Add an interface for adding subsumption info virtual void prepare(const std::map &tensor_info_ctx) = 0; virtual void allocate(void) = 0; + +protected: + std::vector _from_common_candidates; + std::vector _to_common_candidates; }; } // namespace internal diff --git a/runtimes/neurun/src/internal/Model.h b/runtimes/neurun/src/internal/Model.h index 0e27d83..b8fbf0b 100644 --- a/runtimes/neurun/src/internal/Model.h +++ b/runtimes/neurun/src/internal/Model.h @@ -243,12 +243,35 @@ private: return (*this); } + Sequence &insert(uint32_t nth, std::unique_ptr &&node) + { + _ops.insert(_ops.begin() + nth, std::move(node)); + return (*this); + } + public: template Sequence &emplace_back(Args &&... args) { return emplace_back(std::unique_ptr(new T{std::forward(args)...})); } + template Sequence &insert(uint32_t nth, Args &&... args) + { + return insert(nth, std::unique_ptr(new T{std::forward(args)...})); + } + + int find(const op::Node &node) + { + for (int i = 0; i < _ops.size(); ++i) + { + if (&at(i) == &node) + { + return i; + } + } + return -1; + } + private: std::vector> _ops; }; diff --git a/runtimes/neurun/src/internal/arm_compute.h b/runtimes/neurun/src/internal/arm_compute.h index 4945823..9e2706b 100644 --- a/runtimes/neurun/src/internal/arm_compute.h +++ b/runtimes/neurun/src/internal/arm_compute.h @@ -120,12 +120,13 @@ namespace arm_compute class Plan { public: - Plan(const std::shared_ptr &model) : _model(model) + Plan(const std::shared_ptr<::internal::tflite::Model> &model) : _model(model) { // DO NOTHING } public: + ::internal::tflite::Model &model(void) { return *_model; } const ::internal::tflite::Model &model(void) const { return *_model; } public: @@ -137,7 +138,7 @@ public: const op::Sequence &operations(void) const { return _ops; } private: - std::shared_ptr _model; + std::shared_ptr<::internal::tflite::Model> _model; operand::Context _operands; op::Sequence _ops; }; diff --git a/runtimes/neurun/src/internal/common/TensorBuilder.cc b/runtimes/neurun/src/internal/common/TensorBuilder.cc index b316219..2be0b09 100644 --- a/runtimes/neurun/src/internal/common/TensorBuilder.cc +++ b/runtimes/neurun/src/internal/common/TensorBuilder.cc @@ -19,6 +19,24 @@ void TensorBuilder::mark(const ::internal::tflite::operand::Index &ind) _inds.insert(ind.asInt()); } +void TensorBuilder::markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) +{ + // DO NOTHING + throw std::runtime_error("Wrong Approach"); +} + +void TensorBuilder::markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) +{ + // DO NOTHING + throw std::runtime_error("Wrong Approach"); +} + +void TensorBuilder::insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) +{ + // DO NOTHING + throw std::runtime_error("Wrong Approach"); +} + void TensorBuilder::prepare(const std::map &tensor_info_ctx) { assert(_tensors.size() == 0); diff --git a/runtimes/neurun/src/internal/common/TensorBuilder.h b/runtimes/neurun/src/internal/common/TensorBuilder.h index 61e3dbe..dd184b1 100644 --- a/runtimes/neurun/src/internal/common/TensorBuilder.h +++ b/runtimes/neurun/src/internal/common/TensorBuilder.h @@ -20,6 +20,9 @@ public: TensorBuilder(); virtual void mark(const ::internal::tflite::operand::Index &ind) override; + virtual void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) override; + virtual void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) override; + virtual void insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) override; virtual void prepare(const std::map &tensor_info_ctx) override; virtual void allocate(void) override; diff --git a/runtimes/neurun/src/model.h b/runtimes/neurun/src/model.h index cc71887..8e27199 100644 --- a/runtimes/neurun/src/model.h +++ b/runtimes/neurun/src/model.h @@ -12,7 +12,7 @@ public: internal::tflite::Model &deref(void) { return *_model; } public: - void release(std::shared_ptr &model) { model = _model; } + void release(std::shared_ptr &model) { model = _model; } private: std::shared_ptr _model; -- 2.7.4