[neurun] Add Common Tensor/TensorBuilder (#2082)

* [neurun] Add Common Tensor/TensorBuilder

This commit adds Common Tensor/TensorBuilder for converting between backends.

The name of 'Common' would be changed or kept.

* Remove common tensor info context

* Replace common_tensor_builder to BackendManager

diff --git a/runtimes/neurun/src/backend/acl_cl/StageGenerator.cc b/runtimes/neurun/src/backend/acl_cl/StageGenerator.cc
index a484c4f..0c52fb6 100644 (file)
--- a/runtimes/neurun/src/backend/acl_cl/StageGenerator.cc
+++ b/runtimes/neurun/src/backend/acl_cl/StageGenerator.cc
@@ -96,8 +96,9 @@ void ActivationBuilder::append(FuseCode code, ::arm_compute::ICLTensor *ifm_allo
 //
 StageGenerator::StageGenerator(
     const ::internal::tflite::operand::Set &ctx,
-    const std::shared_ptr<::internal::arm_compute::TensorBuilder> &tensor_builder)
-    : _ctx(ctx), _tensor_builder(tensor_builder)
+    const std::shared_ptr<::internal::arm_compute::TensorBuilder> &tensor_builder,
+    const std::shared_ptr<::internal::common::TensorBuilder> &common_tensor_builder)
+    : _ctx(ctx), _tensor_builder(tensor_builder), _common_tensor_builder(common_tensor_builder)
 {
   // DO NOTHING
 }

diff --git a/runtimes/neurun/src/backend/acl_cl/StageGenerator.h b/runtimes/neurun/src/backend/acl_cl/StageGenerator.h
index 5480012..cbdf1e7 100644 (file)
--- a/runtimes/neurun/src/backend/acl_cl/StageGenerator.h
+++ b/runtimes/neurun/src/backend/acl_cl/StageGenerator.h
@@ -5,6 +5,7 @@
 
 #include "internal/Model.h"
 #include "backend/acl_cl/TensorBuilder.h"
+#include "internal/common/TensorBuilder.h"
 
 namespace internal
 {
@@ -15,7 +16,8 @@ class StageGenerator : public ::internal::IStageGenerator
 {
 public:
   StageGenerator(const ::internal::tflite::operand::Set &ctx,
-                 const std::shared_ptr<::internal::arm_compute::TensorBuilder> &tensor_builder);
+                 const std::shared_ptr<::internal::arm_compute::TensorBuilder> &tensor_builder,
+                 const std::shared_ptr<::internal::common::TensorBuilder> &common_tensor_builder);
 
   virtual std::shared_ptr<ITensorBuilder> tensor_builder() override { return _tensor_builder; }
 
@@ -38,6 +40,7 @@ public:
 private:
   const ::internal::tflite::operand::Set &_ctx;
   std::shared_ptr<::internal::arm_compute::TensorBuilder> _tensor_builder;
+  std::shared_ptr<::internal::common::TensorBuilder> _common_tensor_builder;
 };
 
 } // namespace arm_compute

diff --git a/runtimes/neurun/src/backend/cpu/StageGenerator.cc b/runtimes/neurun/src/backend/cpu/StageGenerator.cc
index 1b0c270..f759f45 100644 (file)
--- a/runtimes/neurun/src/backend/cpu/StageGenerator.cc
+++ b/runtimes/neurun/src/backend/cpu/StageGenerator.cc
@@ -24,8 +24,10 @@ namespace cpu
 
 StageGenerator::StageGenerator(
     const ::internal::tflite::operand::Set &operand_ctx,
-    const std::shared_ptr<::internal::cpu::TensorBuilder> &tensor_builder)
-    : _ctx(operand_ctx), _tensor_builder(tensor_builder)
+    const std::shared_ptr<::internal::cpu::TensorBuilder> &tensor_builder,
+    const std::shared_ptr<::internal::common::TensorBuilder> &common_tensor_builder)
+    : _ctx(operand_ctx), _tensor_builder(tensor_builder),
+      _common_tensor_builder(common_tensor_builder)
 {
   // DO NOTHING
 }

diff --git a/runtimes/neurun/src/backend/cpu/StageGenerator.h b/runtimes/neurun/src/backend/cpu/StageGenerator.h
index 9c41759..eafee8a 100644 (file)
--- a/runtimes/neurun/src/backend/cpu/StageGenerator.h
+++ b/runtimes/neurun/src/backend/cpu/StageGenerator.h
@@ -7,6 +7,8 @@
 #include "internal/cpu.h"
 #include "TensorBuilder.h"
 
+#include "internal/common/TensorBuilder.h"
+
 namespace internal
 {
 namespace cpu
@@ -16,7 +18,8 @@ class StageGenerator : public ::internal::IStageGenerator
 {
 public:
   StageGenerator(const ::internal::tflite::operand::Set &ctx,
-                 const std::shared_ptr<::internal::cpu::TensorBuilder> &tensor_builder);
+                 const std::shared_ptr<::internal::cpu::TensorBuilder> &tensor_builder,
+                 const std::shared_ptr<::internal::common::TensorBuilder> &common_tensor_builder);
 
   virtual std::shared_ptr<ITensorBuilder> tensor_builder() override { return _tensor_builder; }
 
@@ -39,6 +42,7 @@ public:
 private:
   const ::internal::tflite::operand::Set &_ctx;
   std::shared_ptr<::internal::cpu::TensorBuilder> _tensor_builder;
+  std::shared_ptr<::internal::common::TensorBuilder> _common_tensor_builder;
 };
 
 } // namespace cpu

diff --git a/runtimes/neurun/src/compilation.cc b/runtimes/neurun/src/compilation.cc
index 7f652a6..a584af5 100644 (file)
--- a/runtimes/neurun/src/compilation.cc
+++ b/runtimes/neurun/src/compilation.cc
@@ -14,6 +14,8 @@
 #include "internal/Padding.h"
 #include "internal/IInitializerGenerator.h"
 #include "internal/IStageGenerator.h"
+#include "internal/common/Tensor.h"
+#include "internal/common/TensorBuilder.h"
 #include "util/EnvVar.h"
 
 #include "compilation.h"
@@ -75,13 +77,13 @@ struct IPlanBuilder
 class BackendResolver
 {
 public:
-  BackendResolver(::internal::BackendManager &backend_manager)
+  BackendResolver(::internal::BackendManager &backend_manager) : _backend_manager(backend_manager)
   {
 #define OP(InternalName, NnApiName)                                                       \
   {                                                                                       \
     const auto &backend_str =                                                             \
         ::nnfw::util::EnvVar{std::string("OP_BACKEND_") + #NnApiName}.asString("acl_cl"); \
-    auto backend = backend_manager.get(backend_str);                                      \
+    auto backend = _backend_manager.get(backend_str);                                     \
     _gen_map[typeid(::internal::tflite::op::InternalName::Node)] = backend;               \
   }
 
@@ -104,9 +106,11 @@ public:
   std::shared_ptr<::internal::IStageGenerator> getStageGenerator(const std::type_index &type);
   std::shared_ptr<::internal::ITensorBuilder> getTensorBuilder(const std::type_index &type);
   std::set<std::shared_ptr<::internal::ITensorBuilder>> getAllTensorBuilders();
+  std::shared_ptr<::internal::common::TensorBuilder> getCommonTensorBuilder();
 
 private:
   std::unordered_map<std::type_index, ::internal::Backend> _gen_map;
+  ::internal::BackendManager &_backend_manager;
 };
 
 std::shared_ptr<::internal::IInitializerGenerator>
@@ -137,6 +141,11 @@ std::set<std::shared_ptr<::internal::ITensorBuilder>> BackendResolver::getAllTen
   return ret;
 }
 
+std::shared_ptr<::internal::common::TensorBuilder> BackendResolver::getCommonTensorBuilder()
+{
+  return _backend_manager.getCommonTensorBuilder();
+}
+
 #include "backend/acl_cl/InitializerGenerator.h"
 #include "backend/acl_cl/StageGenerator.h"
 //#include "internal/cpu/InitializerGenerator.h"
@@ -523,7 +532,7 @@ public:
   void addStage(const Stage &stage) override;
 
 public:
-  void finalize(const std::set<std::shared_ptr<::internal::ITensorBuilder>> &tensor_builders);
+  void finalize(BackendResolver &backend_resolver);
 
 public:
   const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx() { return _tensor_info_ctx; }
@@ -551,9 +560,11 @@ void PlanBuilder::addInitializer(const ::internal::tflite::operand::Index &ind,
 
 void PlanBuilder::addStage(const Stage &stage) { _stages.emplace_back(stage); }
 
-void PlanBuilder::finalize(
-    const std::set<std::shared_ptr<::internal::ITensorBuilder>> &tensor_builders)
+void PlanBuilder::finalize(BackendResolver &backend_resolver)
 {
+  auto tensor_builders = backend_resolver.getAllTensorBuilders();
+  auto common_tensor_builder = backend_resolver.getCommonTensorBuilder();
+
   // Mark tensors
   const auto &operations = _plan.model().operations();
 
@@ -563,6 +574,8 @@ void PlanBuilder::finalize(
     tensor_builder->prepare(_tensor_info_ctx);
   }
 
+  common_tensor_builder->prepare(_tensor_info_ctx);
+
   // Process Stage
   ExecutionBuilder execution_builder{_plan};
 
@@ -578,6 +591,8 @@ void PlanBuilder::finalize(
     tensor_builder->allocate();
   }
 
+  common_tensor_builder->allocate();
+
   // Fill weight/bias
   for (auto it = _initializer_ctx.begin(); it != _initializer_ctx.end(); ++it)
   {
@@ -627,7 +642,7 @@ int ANeuralNetworksCompilation_finish(ANeuralNetworksCompilation *compilation)
     op.accept(TensorMarker{*tensor_builder});
   }
 
-  plan_builder.finalize(backend_resolver.getAllTensorBuilders());
+  plan_builder.finalize(backend_resolver);
 
   return ANEURALNETWORKS_NO_ERROR;
 }

diff --git a/runtimes/neurun/src/internal/BackendManager.cc b/runtimes/neurun/src/internal/BackendManager.cc
index a6f8d86..52b2874 100644 (file)
--- a/runtimes/neurun/src/internal/BackendManager.cc
+++ b/runtimes/neurun/src/internal/BackendManager.cc
@@ -14,13 +14,15 @@ BackendManager::BackendManager(::internal::arm_compute::Plan &plan) : _plan(plan
 {
   const auto &operands = _plan.model().operands();
 
+  _common_tensor_builder = std::make_shared<::internal::common::TensorBuilder>();
+
   // Add arm_compute backend
   {
     auto acl_tensor_builder = std::make_shared<::internal::arm_compute::TensorBuilder>(_plan);
     auto acl_initializer_gen =
         std::make_shared<::internal::arm_compute::InitializerGenerator>(operands);
-    auto acl_stage_gen =
-        std::make_shared<::internal::arm_compute::StageGenerator>(operands, acl_tensor_builder);
+    auto acl_stage_gen = std::make_shared<::internal::arm_compute::StageGenerator>(
+        operands, acl_tensor_builder, _common_tensor_builder);
 
     // TODO Do not use magic string for backend id
     _gen_map["acl_cl"] = {acl_initializer_gen, acl_stage_gen};
@@ -30,8 +32,8 @@ BackendManager::BackendManager(::internal::arm_compute::Plan &plan) : _plan(plan
   {
     auto cpu_tensor_builder = std::make_shared<::internal::cpu::TensorBuilder>(_plan);
     auto cpu_initializer_gen = std::make_shared<::internal::cpu::InitializerGenerator>(operands);
-    auto cpu_stage_gen =
-        std::make_shared<::internal::cpu::StageGenerator>(operands, cpu_tensor_builder);
+    auto cpu_stage_gen = std::make_shared<::internal::cpu::StageGenerator>(
+        operands, cpu_tensor_builder, _common_tensor_builder);
 
     // TODO Do not use magic string for backend id
     _gen_map["cpu"] = {cpu_initializer_gen, cpu_stage_gen};
@@ -40,4 +42,9 @@ BackendManager::BackendManager(::internal::arm_compute::Plan &plan) : _plan(plan
 
 Backend BackendManager::get(const std::string &key) { return _gen_map.at(key); }
 
+std::shared_ptr<::internal::common::TensorBuilder> BackendManager::getCommonTensorBuilder()
+{
+  return _common_tensor_builder;
+}
+
 } // namespace internal

diff --git a/runtimes/neurun/src/internal/BackendManager.h b/runtimes/neurun/src/internal/BackendManager.h
index 6d866c9..1946a04 100644 (file)
--- a/runtimes/neurun/src/internal/BackendManager.h
+++ b/runtimes/neurun/src/internal/BackendManager.h
@@ -7,6 +7,7 @@
 #include "internal/IInitializerGenerator.h"
 #include "internal/IStageGenerator.h"
 #include "internal/ITensorBuilder.h"
+#include "internal/common/TensorBuilder.h"
 
 namespace internal
 {
@@ -36,9 +37,12 @@ public:
 
   Backend get(const std::string &key);
 
+  std::shared_ptr<::internal::common::TensorBuilder> getCommonTensorBuilder();
+
 private:
   ::internal::arm_compute::Plan &_plan;
   std::map<std::string, Backend> _gen_map;
+  std::shared_ptr<::internal::common::TensorBuilder> _common_tensor_builder;
 };
 
 } // namespace internal

diff --git a/runtimes/neurun/src/internal/common/Tensor.h b/runtimes/neurun/src/internal/common/Tensor.h
new file mode 100644 (file)
index 0000000..a5a1b96
--- /dev/null
+++ b/runtimes/neurun/src/internal/common/Tensor.h
@@ -0,0 +1,54 @@
+#ifndef __INTERNAL_COMMON_TENSOR_H__
+#define __INTERNAL_COMMON_TENSOR_H__
+
+#include <arm_compute/core/ITensor.h>
+#include <arm_compute/core/TensorInfo.h>
+
+namespace internal
+{
+namespace common
+{
+
+class Tensor : public ::arm_compute::ITensor
+{
+public:
+  Tensor() = default;
+
+  Tensor(::arm_compute::TensorInfo info) : _info(info)
+  {
+    // DO_NOTING
+  }
+
+  Tensor(uint8_t *buffer) : _buffer(buffer)
+  {
+    // DO NOTHING
+  }
+
+public:
+  void setBuffer(uint8_t *buffer) { _buffer = buffer; }
+
+public:
+  ::arm_compute::TensorInfo *info() const override
+  {
+    return const_cast<::arm_compute::TensorInfo *>(&_info);
+  }
+
+  ::arm_compute::TensorInfo *info() override { return &_info; }
+
+  uint8_t *buffer() const override { return _buffer; }
+
+  void allocate()
+  {
+    uint32_t size = _info.total_size(); // NOTE This size may not be accurate
+    _buffer = new uint8_t[size];        // NOTE The allocated buffer is never deallocated.
+  }
+
+private:
+  ::arm_compute::TensorInfo _info;
+  uint8_t *_buffer = nullptr;
+};
+
+} // common
+} // internal
+
+#endif // __INTERNAL_COMMON_TENSOR_H__

diff --git a/runtimes/neurun/src/internal/common/TensorBuilder.cc b/runtimes/neurun/src/internal/common/TensorBuilder.cc
new file mode 100644 (file)
index 0000000..b316219
--- /dev/null
+++ b/runtimes/neurun/src/internal/common/TensorBuilder.cc
@@ -0,0 +1,51 @@
+#include "TensorBuilder.h"
+
+#include <algorithm>
+
+namespace internal
+{
+namespace common
+{
+
+TensorBuilder::TensorBuilder()
+{
+  // DO NOTHING
+}
+
+void TensorBuilder::mark(const ::internal::tflite::operand::Index &ind)
+{
+  assert(_tensors.size() == 0);
+
+  _inds.insert(ind.asInt());
+}
+
+void TensorBuilder::prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx)
+{
+  assert(_tensors.size() == 0);
+
+  for (auto ind_int : _inds)
+  {
+    ::internal::tflite::operand::Index ind{ind_int};
+    auto tensor = std::make_shared<::internal::common::Tensor>(tensor_info_ctx.at(ind.asInt()));
+    _tensors[ind.asInt()] = tensor;
+  }
+}
+
+void TensorBuilder::allocate(void)
+{
+  assert(_inds.size() == _tensors.size());
+
+  for (auto it : _tensors)
+  {
+    it.second->allocate();
+  }
+}
+
+std::shared_ptr<::internal::common::Tensor>
+TensorBuilder::at(const ::internal::tflite::operand::Index &ind)
+{
+  return _tensors.at(ind.asInt());
+}
+
+} // namespace common
+} // namespace internal

diff --git a/runtimes/neurun/src/internal/common/TensorBuilder.h b/runtimes/neurun/src/internal/common/TensorBuilder.h
new file mode 100644 (file)
index 0000000..61e3dbe
--- /dev/null
+++ b/runtimes/neurun/src/internal/common/TensorBuilder.h
@@ -0,0 +1,36 @@
+#ifndef __INTERNAL_COMMON_TENSOR_BUILDER_H__
+#define __INTERNAL_COMMON_TENSOR_BUILDER_H__
+
+#include <unordered_set>
+#include <unordered_map>
+
+#include "internal/ITensorBuilder.h"
+#include "internal/common/Tensor.h"
+
+namespace internal
+{
+namespace common
+{
+
+class Plan;
+
+class TensorBuilder : public ::internal::ITensorBuilder
+{
+public:
+  TensorBuilder();
+
+  virtual void mark(const ::internal::tflite::operand::Index &ind) override;
+  virtual void prepare(const std::map<int, ::arm_compute::TensorInfo> &tensor_info_ctx) override;
+  virtual void allocate(void) override;
+
+  std::shared_ptr<::internal::common::Tensor> at(const ::internal::tflite::operand::Index &ind);
+
+private:
+  std::unordered_set<int> _inds;
+  std::unordered_map<int, std::shared_ptr<::internal::common::Tensor>> _tensors;
+};
+
+} // namespace common
+} // namespace internal
+
+#endif // __INTERNAL_COMMON_TENSOR_BUILDER_H__