[neurun] Insert TensorConvert nodes (#2127)

* [neurun] Insert TensorConvert nodes

This commit inserts TensorConvert nodes. Those will be inserted to between all of nodes.

Signed-off-by: sjsujinkim sjsujin.kim@samsung.com
* Change to plugin type

* Fix build failure

diff --git a/runtimes/neurun/src/backend/acl_cl/TensorBuilder.cc b/runtimes/neurun/src/backend/acl_cl/TensorBuilder.cc
index 3161503..5354139 100644 (file)
--- a/runtimes/neurun/src/backend/acl_cl/TensorBuilder.cc
+++ b/runtimes/neurun/src/backend/acl_cl/TensorBuilder.cc
@@ -23,6 +23,39 @@ void TensorBuilder::mark(const ::internal::tflite::operand::Index &ind)
   _inds.insert(ind.asInt());
 }
 
+void TensorBuilder::markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind)
+{
+  _from_common_candidates.emplace_back(op, ind);
+}
+
+void TensorBuilder::markToCommon(const ::internal::tflite::op::Node &op, int32_t ind)
+{
+  _to_common_candidates.emplace_back(op, ind);
+}
+
+void TensorBuilder::insertTensorConvertNodes(::internal::tflite::op::Sequence &operations)
+{
+  for (auto param : _from_common_candidates)
+  {
+    auto index = operations.find(param.op);
+    if (index != -1)
+    {
+      operations.insert<::internal::tflite::op::TensorConvert::AclFromCommon::Node>(
+          index, ::internal::tflite::op::TensorConvert::AclFromCommon::Param(param.tensor_index));
+    }
+  }
+
+  for (auto param : _to_common_candidates)
+  {
+    auto index = operations.find(param.op);
+    if (index != -1)
+    {
+      operations.insert<::internal::tflite::op::TensorConvert::AclToCommon::Node>(
+          index + 1, ::internal::tflite::op::TensorConvert::AclToCommon::Param(param.tensor_index));
+    }
+  }
+}
+
 void TensorBuilder::prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx)
 {
   assert(_tensors.size() == 0);

diff --git a/runtimes/neurun/src/backend/acl_cl/TensorBuilder.h b/runtimes/neurun/src/backend/acl_cl/TensorBuilder.h
index 63b13d9..e0fce48 100644 (file)
--- a/runtimes/neurun/src/backend/acl_cl/TensorBuilder.h
+++ b/runtimes/neurun/src/backend/acl_cl/TensorBuilder.h
@@ -24,6 +24,9 @@ public:
   TensorBuilder(::internal::arm_compute::Plan &plan);
 
   virtual void mark(const ::internal::tflite::operand::Index &ind) override;
+  virtual void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) override;
+  virtual void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) override;
+  virtual void insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) override;
   virtual void prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx) override;
   virtual void allocate(void) override;
 

diff --git a/runtimes/neurun/src/backend/cpu/TensorBuilder.cc b/runtimes/neurun/src/backend/cpu/TensorBuilder.cc
index 7b31bec..b907586 100644 (file)
--- a/runtimes/neurun/src/backend/cpu/TensorBuilder.cc
+++ b/runtimes/neurun/src/backend/cpu/TensorBuilder.cc
@@ -23,6 +23,39 @@ void TensorBuilder::mark(const ::internal::tflite::operand::Index &ind)
   _inds.insert(ind.asInt());
 }
 
+void TensorBuilder::markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind)
+{
+  _from_common_candidates.emplace_back(op, ind);
+}
+
+void TensorBuilder::markToCommon(const ::internal::tflite::op::Node &op, int32_t ind)
+{
+  _to_common_candidates.emplace_back(op, ind);
+}
+
+void TensorBuilder::insertTensorConvertNodes(::internal::tflite::op::Sequence &operations)
+{
+  for (auto param : _from_common_candidates)
+  {
+    auto index = operations.find(param.op);
+    if (index != -1)
+    {
+      operations.insert<::internal::tflite::op::TensorConvert::CpuFromCommon::Node>(
+          index, ::internal::tflite::op::TensorConvert::CpuFromCommon::Param(param.tensor_index));
+    }
+  }
+
+  for (auto param : _to_common_candidates)
+  {
+    auto index = operations.find(param.op);
+    if (index != -1)
+    {
+      operations.insert<::internal::tflite::op::TensorConvert::CpuToCommon::Node>(
+          index + 1, ::internal::tflite::op::TensorConvert::CpuToCommon::Param(param.tensor_index));
+    }
+  }
+}
+
 void TensorBuilder::prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx)
 {
   assert(_tensors.size() == 0);

diff --git a/runtimes/neurun/src/backend/cpu/TensorBuilder.h b/runtimes/neurun/src/backend/cpu/TensorBuilder.h
index 5395161..de252e1 100644 (file)
--- a/runtimes/neurun/src/backend/cpu/TensorBuilder.h
+++ b/runtimes/neurun/src/backend/cpu/TensorBuilder.h
@@ -23,6 +23,9 @@ public:
   TensorBuilder(::internal::arm_compute::Plan &plan);
 
   virtual void mark(const ::internal::tflite::operand::Index &ind) override;
+  virtual void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) override;
+  virtual void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) override;
+  virtual void insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) override;
   virtual void prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx) override;
   virtual void allocate(void) override;
 

diff --git a/runtimes/neurun/src/compilation.cc b/runtimes/neurun/src/compilation.cc
index f4d0603..009337d 100644 (file)
--- a/runtimes/neurun/src/compilation.cc
+++ b/runtimes/neurun/src/compilation.cc
@@ -389,22 +389,22 @@ void Planner::visit(const ::internal::tflite::op::Softmax::Node &node)
 
 void Planner::visit(const ::internal::tflite::op::TensorConvert::CpuFromCommon::Node &node)
 {
-  throw std::runtime_error("NYI - Planner::visit(TensorConvert::CpuFromCommon");
+  VERBOSE(CpuFromCommon) << "Configure CpuFromCommon operation" << std::endl;
 }
 
 void Planner::visit(const ::internal::tflite::op::TensorConvert::CpuToCommon::Node &node)
 {
-  throw std::runtime_error("NYI - Planner::visit(TensorConvert::CpuToCommon");
+  VERBOSE(CpuToCommon) << "Configure CpuToCommon operation" << std::endl;
 }
 
 void Planner::visit(const ::internal::tflite::op::TensorConvert::AclFromCommon::Node &node)
 {
-  throw std::runtime_error("NYI - Planner::visit(TensorConvert::FromCommon");
+  VERBOSE(AclFromCommon) << "Configure AclFromCommon operation" << std::endl;
 }
 
 void Planner::visit(const ::internal::tflite::op::TensorConvert::AclToCommon::Node &node)
 {
-  throw std::runtime_error("NYI - Planner::visit(TensorConvert::AclToCommon");
+  VERBOSE(AclToCommon) << "Configure AclToCommon operation" << std::endl;
 }
 
 class TensorMarker : public ::internal::tflite::op::NodeVisitor
@@ -430,6 +430,14 @@ public:
 
 private:
   void mark(int32_t ind) { _tensor_builder.mark(::internal::tflite::operand::Index{ind}); }
+  void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind)
+  {
+    _tensor_builder.markFromCommon(op, ind);
+  }
+  void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind)
+  {
+    _tensor_builder.markToCommon(op, ind);
+  }
 
 private:
   ::internal::ITensorBuilder &_tensor_builder;
@@ -442,6 +450,9 @@ void TensorMarker::visit(const ::internal::tflite::op::Conv2D::implicit::Node &n
   mark(param.ifm_index);
   mark(param.ker_index);
   mark(param.bias_index);
+
+  markToCommon(node, param.ofm_index);
+  markFromCommon(node, param.ifm_index);
 }
 
 void TensorMarker::visit(const ::internal::tflite::op::MaxPool2D::implicit::Node &node)
@@ -449,6 +460,9 @@ void TensorMarker::visit(const ::internal::tflite::op::MaxPool2D::implicit::Node
   const auto &param = node.param();
   mark(param.ofm_index);
   mark(param.ifm_index);
+
+  markToCommon(node, param.ofm_index);
+  markFromCommon(node, param.ifm_index);
 }
 
 void TensorMarker::visit(const ::internal::tflite::op::AvgPool2D::implicit::Node &node)
@@ -456,16 +470,22 @@ void TensorMarker::visit(const ::internal::tflite::op::AvgPool2D::implicit::Node
   const auto &param = node.param();
   mark(param.ofm_index);
   mark(param.ifm_index);
+
+  markToCommon(node, param.ofm_index);
+  markFromCommon(node, param.ifm_index);
 }
 
 void TensorMarker::visit(const ::internal::tflite::op::Concat::Node &node)
 {
   const auto &param = node.param();
   mark(param.ofm_index);
+  markToCommon(node, param.ofm_index);
   for (auto ind : param.ifm_indexes)
   {
     mark(ind);
   }
+
+  markFromCommon(node, param.ifm_indexes[0]);
 }
 
 void TensorMarker::visit(const ::internal::tflite::op::FullyConnected::Node &node)
@@ -475,6 +495,9 @@ void TensorMarker::visit(const ::internal::tflite::op::FullyConnected::Node &nod
   mark(param.input_index);
   mark(param.weight_index);
   mark(param.bias_index);
+
+  markToCommon(node, param.output_index);
+  markFromCommon(node, param.input_index);
 }
 
 void TensorMarker::visit(const ::internal::tflite::op::Reshape::Node &node)
@@ -482,6 +505,9 @@ void TensorMarker::visit(const ::internal::tflite::op::Reshape::Node &node)
   const auto &param = node.param();
   mark(param.output_index);
   mark(param.input_index);
+
+  markToCommon(node, param.output_index);
+  markFromCommon(node, param.input_index);
 }
 
 void TensorMarker::visit(const ::internal::tflite::op::Softmax::Node &node)
@@ -489,6 +515,9 @@ void TensorMarker::visit(const ::internal::tflite::op::Softmax::Node &node)
   const auto &param = node.param();
   mark(param.output_index);
   mark(param.input_index);
+
+  markToCommon(node, param.output_index);
+  markFromCommon(node, param.input_index);
 }
 
 void TensorMarker::visit(const ::internal::tflite::op::TensorConvert::CpuFromCommon::Node &node)
@@ -624,7 +653,7 @@ int ANeuralNetworksCompilation::finish()
 
   auto &plan = this->plan();
   const auto &operands = plan.model().operands();
-  const auto &operations = plan.model().operations();
+  auto &operations = plan.model().operations();
 
   ::internal::BackendManager backend_manager{plan};
   BackendResolver backend_resolver{backend_manager};
@@ -632,18 +661,24 @@ int ANeuralNetworksCompilation::finish()
 
   for (uint32_t n = 0; n < operations.size(); ++n)
   {
-    operations.at(n).accept(Planner{operands, plan_builder, backend_resolver});
+    const auto &op = operations.at(n);
+    auto tensor_builder = backend_resolver.getTensorBuilder(typeid(op));
+    op.accept(TensorMarker{*tensor_builder});
   }
 
-  // TODO Add optimization passes
+  auto tensor_builders = backend_resolver.getAllTensorBuilders();
+
+  for (auto tensor_builder : tensor_builders)
+  {
+    tensor_builder->insertTensorConvertNodes(operations);
+  }
 
   for (uint32_t n = 0; n < operations.size(); ++n)
   {
-    const auto &op = operations.at(n);
-    auto tensor_builder = backend_resolver.getTensorBuilder(typeid(op));
-    op.accept(TensorMarker{*tensor_builder});
+    operations.at(n).accept(Planner{operands, plan_builder, backend_resolver});
   }
 
+  // TODO Add optimization passes
   plan_builder.finalize(backend_resolver);
 
   return ANEURALNETWORKS_NO_ERROR;

diff --git a/runtimes/neurun/src/compilation.h b/runtimes/neurun/src/compilation.h
index db6a5a1..0209798 100644 (file)
--- a/runtimes/neurun/src/compilation.h
+++ b/runtimes/neurun/src/compilation.h
@@ -7,7 +7,7 @@
 struct ANeuralNetworksCompilation
 {
 public:
-  ANeuralNetworksCompilation(const std::shared_ptr<const internal::tflite::Model> &model)
+  ANeuralNetworksCompilation(const std::shared_ptr<internal::tflite::Model> &model)
       : _plan{new internal::arm_compute::Plan{model}}
   {
     // DO NOTHING

diff --git a/runtimes/neurun/src/frontend/compilation.cc b/runtimes/neurun/src/frontend/compilation.cc
index fd871dc..e056e21 100644 (file)
--- a/runtimes/neurun/src/frontend/compilation.cc
+++ b/runtimes/neurun/src/frontend/compilation.cc
@@ -9,7 +9,7 @@
 int ANeuralNetworksCompilation_create(ANeuralNetworksModel *model,
                                       ANeuralNetworksCompilation **compilation)
 {
-  std::shared_ptr<const internal::tflite::Model> internal;
+  std::shared_ptr<internal::tflite::Model> internal;
 
   model->release(internal);
 

diff --git a/runtimes/neurun/src/internal/ITensorBuilder.h b/runtimes/neurun/src/internal/ITensorBuilder.h
index b3938f4..90e3f49 100644 (file)
--- a/runtimes/neurun/src/internal/ITensorBuilder.h
+++ b/runtimes/neurun/src/internal/ITensorBuilder.h
@@ -9,13 +9,31 @@
 namespace internal
 {
 
+struct TensorConversionParam
+{
+  TensorConversionParam(const ::internal::tflite::op::Node &op, int tensor_index)
+      : op(op), tensor_index(tensor_index)
+  {
+  }
+
+  const ::internal::tflite::op::Node &op;
+  int tensor_index;
+};
+
 struct ITensorBuilder
 {
   virtual ~ITensorBuilder(void) = default;
   virtual void mark(const ::internal::tflite::operand::Index &ind) = 0;
+  virtual void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) = 0;
+  virtual void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) = 0;
+  virtual void insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) = 0;
   // TODO Add an interface for adding subsumption info
   virtual void prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx) = 0;
   virtual void allocate(void) = 0;
+
+protected:
+  std::vector<TensorConversionParam> _from_common_candidates;
+  std::vector<TensorConversionParam> _to_common_candidates;
 };
 
 } // namespace internal

diff --git a/runtimes/neurun/src/internal/Model.h b/runtimes/neurun/src/internal/Model.h
index 0e27d83..b8fbf0b 100644 (file)
--- a/runtimes/neurun/src/internal/Model.h
+++ b/runtimes/neurun/src/internal/Model.h
@@ -243,12 +243,35 @@ private:
     return (*this);
   }
 
+  Sequence &insert(uint32_t nth, std::unique_ptr<op::Node> &&node)
+  {
+    _ops.insert(_ops.begin() + nth, std::move(node));
+    return (*this);
+  }
+
 public:
   template <typename T, typename... Args> Sequence &emplace_back(Args &&... args)
   {
     return emplace_back(std::unique_ptr<T>(new T{std::forward<Args>(args)...}));
   }
 
+  template <typename T, typename... Args> Sequence &insert(uint32_t nth, Args &&... args)
+  {
+    return insert(nth, std::unique_ptr<T>(new T{std::forward<Args>(args)...}));
+  }
+
+  int find(const op::Node &node)
+  {
+    for (int i = 0; i < _ops.size(); ++i)
+    {
+      if (&at(i) == &node)
+      {
+        return i;
+      }
+    }
+    return -1;
+  }
+
 private:
   std::vector<std::unique_ptr<op::Node>> _ops;
 };

diff --git a/runtimes/neurun/src/internal/arm_compute.h b/runtimes/neurun/src/internal/arm_compute.h
index 4945823..9e2706b 100644 (file)
--- a/runtimes/neurun/src/internal/arm_compute.h
+++ b/runtimes/neurun/src/internal/arm_compute.h
@@ -120,12 +120,13 @@ namespace arm_compute
 class Plan
 {
 public:
-  Plan(const std::shared_ptr<const ::internal::tflite::Model> &model) : _model(model)
+  Plan(const std::shared_ptr<::internal::tflite::Model> &model) : _model(model)
   {
     // DO NOTHING
   }
 
 public:
+  ::internal::tflite::Model &model(void) { return *_model; }
   const ::internal::tflite::Model &model(void) const { return *_model; }
 
 public:
@@ -137,7 +138,7 @@ public:
   const op::Sequence &operations(void) const { return _ops; }
 
 private:
-  std::shared_ptr<const ::internal::tflite::Model> _model;
+  std::shared_ptr<::internal::tflite::Model> _model;
   operand::Context _operands;
   op::Sequence _ops;
 };

diff --git a/runtimes/neurun/src/internal/common/TensorBuilder.cc b/runtimes/neurun/src/internal/common/TensorBuilder.cc
index b316219..2be0b09 100644 (file)
--- a/runtimes/neurun/src/internal/common/TensorBuilder.cc
+++ b/runtimes/neurun/src/internal/common/TensorBuilder.cc
@@ -19,6 +19,24 @@ void TensorBuilder::mark(const ::internal::tflite::operand::Index &ind)
   _inds.insert(ind.asInt());
 }
 
+void TensorBuilder::markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind)
+{
+  // DO NOTHING
+  throw std::runtime_error("Wrong Approach");
+}
+
+void TensorBuilder::markToCommon(const ::internal::tflite::op::Node &op, int32_t ind)
+{
+  // DO NOTHING
+  throw std::runtime_error("Wrong Approach");
+}
+
+void TensorBuilder::insertTensorConvertNodes(::internal::tflite::op::Sequence &operations)
+{
+  // DO NOTHING
+  throw std::runtime_error("Wrong Approach");
+}
+
 void TensorBuilder::prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx)
 {
   assert(_tensors.size() == 0);

diff --git a/runtimes/neurun/src/internal/common/TensorBuilder.h b/runtimes/neurun/src/internal/common/TensorBuilder.h
index 61e3dbe..dd184b1 100644 (file)
--- a/runtimes/neurun/src/internal/common/TensorBuilder.h
+++ b/runtimes/neurun/src/internal/common/TensorBuilder.h
@@ -20,6 +20,9 @@ public:
   TensorBuilder();
 
   virtual void mark(const ::internal::tflite::operand::Index &ind) override;
+  virtual void markFromCommon(const ::internal::tflite::op::Node &op, int32_t ind) override;
+  virtual void markToCommon(const ::internal::tflite::op::Node &op, int32_t ind) override;
+  virtual void insertTensorConvertNodes(::internal::tflite::op::Sequence &operations) override;
   virtual void prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx) override;
   virtual void allocate(void) override;
 

diff --git a/runtimes/neurun/src/model.h b/runtimes/neurun/src/model.h
index cc71887..8e27199 100644 (file)
--- a/runtimes/neurun/src/model.h
+++ b/runtimes/neurun/src/model.h
@@ -12,7 +12,7 @@ public:
   internal::tflite::Model &deref(void) { return *_model; }
 
 public:
-  void release(std::shared_ptr<const internal::tflite::Model> &model) { model = _model; }
+  void release(std::shared_ptr<internal::tflite::Model> &model) { model = _model; }
 
 private:
   std::shared_ptr<internal::tflite::Model> _model;