[MemoryManager] Apply MemoryManager to Executors (#5447)

Apply MemoryManager to { Linear|Dataflow|Parallel } Executor

Signed-off-by: Yongseop Kim <yons.kim@samsung.com>

diff --git a/runtimes/neurun/backend/acl_common/TemplTensorBuilder.h b/runtimes/neurun/backend/acl_common/TemplTensorBuilder.h
index f928137..3cf450d 100644 (file)
--- a/runtimes/neurun/backend/acl_common/TemplTensorBuilder.h
+++ b/runtimes/neurun/backend/acl_common/TemplTensorBuilder.h
@@ -66,8 +66,7 @@ public:
   std::shared_ptr<backend::operand::IObject> wrapTensor(const model::OperandIndex &ind) override;
   void iterate(const IterateFunction &fn) override;
 
-  // TODO Pass MemoryManager to Executor instead of TensorBuilder
-  // std::unique_ptr<IMemoryManager> releaseMemoryManager(void) override;
+  std::unique_ptr<IMemoryManager> releaseMemoryManager(void) override;
 
   std::shared_ptr<T_ITensor> at(const ::neurun::model::OperandIndex &ind);
   /**
@@ -336,6 +335,13 @@ void TemplTensorBuilder<T_ITensor, T_Tensor, T_SubTensor, T_Object>::dimCorrecti
   _apply_dim_correction_map[index] = apply_dim_correction;
 }
 
+template <typename T_ITensor, typename T_Tensor, typename T_SubTensor, typename T_Object>
+std::unique_ptr<IMemoryManager>
+TemplTensorBuilder<T_ITensor, T_Tensor, T_SubTensor, T_Object>::releaseMemoryManager(void)
+{
+  return std::move(_mem_mgr);
+}
+
 } // namespace acl_common
 } // namespace backend
 } // namespace neurun

diff --git a/runtimes/neurun/backend/cpu/TensorBuilder.cc b/runtimes/neurun/backend/cpu/TensorBuilder.cc
index a6f52b5..4a6ebea 100644 (file)
--- a/runtimes/neurun/backend/cpu/TensorBuilder.cc
+++ b/runtimes/neurun/backend/cpu/TensorBuilder.cc
@@ -90,6 +90,11 @@ std::shared_ptr<operand::Tensor> TensorBuilder::at(const ::neurun::model::Operan
   return _mem_mgr->tensors().at(ind);
 }
 
+std::unique_ptr<IMemoryManager> TensorBuilder::releaseMemoryManager(void)
+{
+  return std::move(_mem_mgr);
+}
+
 } // namespace cpu
 } // namespace backend
 } // namespace neurun

diff --git a/runtimes/neurun/backend/cpu/TensorBuilder.h b/runtimes/neurun/backend/cpu/TensorBuilder.h
index fa89479..72711c8 100644 (file)
--- a/runtimes/neurun/backend/cpu/TensorBuilder.h
+++ b/runtimes/neurun/backend/cpu/TensorBuilder.h
@@ -68,8 +68,7 @@ public:
 
   void iterate(const IterateFunction &fn) override;
 
-  // TODO Pass MemoryManager to Executor instead of TensorBuilder
-  // std::unique_ptr<IMemoryManager> releaseMemoryManager(void) override;
+  std::unique_ptr<IMemoryManager> releaseMemoryManager(void) override;
 
   std::shared_ptr<operand::Tensor> at(const ::neurun::model::OperandIndex &ind);
 

diff --git a/runtimes/neurun/core/include/backend/ITensorBuilder.h b/runtimes/neurun/core/include/backend/ITensorBuilder.h
index 5d29ee3..683e1cc 100644 (file)
--- a/runtimes/neurun/core/include/backend/ITensorBuilder.h
+++ b/runtimes/neurun/core/include/backend/ITensorBuilder.h
@@ -25,6 +25,7 @@
 #include "compiler/SubTensorInfo.h"
 #include "operand/ITensor.h"
 #include "model/Layout.h"
+#include "IMemoryManager.h"
 
 namespace neurun
 {
@@ -59,6 +60,8 @@ struct ITensorBuilder
   tensorAt(const model::OperandIndex &ind) = 0;
   virtual std::shared_ptr<backend::operand::IObject> wrapTensor(const model::OperandIndex &ind) = 0;
   virtual void iterate(const IterateFunction &fn) = 0;
+
+  virtual std::unique_ptr<IMemoryManager> releaseMemoryManager(void) = 0;
 };
 
 } // namespace backend

diff --git a/runtimes/neurun/core/src/compiler/ExecutorFactory.cc b/runtimes/neurun/core/src/compiler/ExecutorFactory.cc
index 0953ae5..b3b32ba 100644 (file)
--- a/runtimes/neurun/core/src/compiler/ExecutorFactory.cc
+++ b/runtimes/neurun/core/src/compiler/ExecutorFactory.cc
@@ -29,6 +29,7 @@
 #include "SubTensorAnalyzer.h"
 #include "PlanBuilder.h"
 #include "ConstantInitializer.h"
+#include "cpp14/memory.h"
 
 namespace neurun
 {
@@ -108,10 +109,21 @@ exec::IExecutor *ExecutorFactory::createLinearExecutor(graph::Graph &graph)
 
   ConstantInitializer{graph, *operand_context, *linear->getLowerInfo()}();
 
+  // Prepare each MemoryManager on each backend
+  auto mem_mgrs = nnfw::cpp14::make_unique<backend::MemoryManagerSet>();
+  for (auto &tensor_builder : tensor_builders)
+  {
+    mem_mgrs->insert(std::move(tensor_builder->releaseMemoryManager()));
+  }
+
   auto plan = std::make_shared<Plan>(operation_sequence);
-  return new exec::LinearExecutor{graph.shareModel(),        linear->releaseSubgraphContext(),
-                                  operand_context,           linear->releaseLowerInfo(),
-                                  linear->releaseElements(), plan};
+  return new exec::LinearExecutor{graph.shareModel(),
+                                  linear->releaseSubgraphContext(),
+                                  operand_context,
+                                  linear->releaseLowerInfo(),
+                                  std::move(mem_mgrs),
+                                  linear->releaseElements(),
+                                  plan};
 }
 
 exec::IExecutor *ExecutorFactory::createDataflowExecutor(graph::Graph &graph, bool parallel)
@@ -225,17 +237,26 @@ exec::IExecutor *ExecutorFactory::createDataflowExecutor(graph::Graph &graph, bo
 
   ConstantInitializer{graph, *operand_context, *lower_info}();
 
+  // Prepare each MemoryManager on each backend
+  auto mem_mgrs = nnfw::cpp14::make_unique<backend::MemoryManagerSet>();
+  for (auto &tensor_builder : tensor_builders)
+  {
+    mem_mgrs->insert(std::move(tensor_builder->releaseMemoryManager()));
+  }
+
   if (parallel)
   {
-    return new exec::ParallelExecutor{graph.shareModel(), std::move(graph.releaseSubgraphContext()),
-                                      operand_context, std::move(lower_info),
-                                      std::move(execution_builder.releaseCodeMap())};
+    return new exec::ParallelExecutor{
+        graph.shareModel(),  std::move(graph.releaseSubgraphContext()),
+        operand_context,     std::move(lower_info),
+        std::move(mem_mgrs), std::move(execution_builder.releaseCodeMap())};
   }
   else
   {
     auto exec = new exec::DataflowExecutor{
-        graph.shareModel(), std::move(graph.releaseSubgraphContext()), operand_context,
-        std::move(lower_info), std::move(execution_builder.releaseCodeMap())};
+        graph.shareModel(),  std::move(graph.releaseSubgraphContext()),
+        operand_context,     std::move(lower_info),
+        std::move(mem_mgrs), std::move(execution_builder.releaseCodeMap())};
     if (config::ConfigManager::instance().get<bool>(config::PROFILING_MODE))
     {
       auto et = std::make_shared<backend::ExecTime>(graph.backend_resolver()->getAllBackends());

diff --git a/runtimes/neurun/core/src/exec/DataflowExecutor.cc b/runtimes/neurun/core/src/exec/DataflowExecutor.cc
index 06ec002..4bbef71 100644 (file)
--- a/runtimes/neurun/core/src/exec/DataflowExecutor.cc
+++ b/runtimes/neurun/core/src/exec/DataflowExecutor.cc
@@ -73,8 +73,10 @@ DataflowExecutor::DataflowExecutor(const std::shared_ptr<const model::Model> &mo
                                    std::unique_ptr<model::SubgraphContext> subg_ctx,
                                    const std::shared_ptr<compiler::OperandContext> &operand_context,
                                    std::unique_ptr<graph::LowerInfoMap> lower_info,
+                                   std::unique_ptr<backend::MemoryManagerSet> mem_mgrs,
                                    CodeMap &&code_map)
-    : ExecutorBase{model, std::move(subg_ctx), operand_context, std::move(lower_info)},
+    : ExecutorBase{model, std::move(subg_ctx), operand_context, std::move(lower_info),
+                   std::move(mem_mgrs)},
       _code_map{std::move(code_map)}
 {
   VERBOSE(DataflowExecutor) << "Constructing Dataflow Executor" << std::endl;

diff --git a/runtimes/neurun/core/src/exec/DataflowExecutor.h b/runtimes/neurun/core/src/exec/DataflowExecutor.h
index 6103e38..20f80e6 100644 (file)
--- a/runtimes/neurun/core/src/exec/DataflowExecutor.h
+++ b/runtimes/neurun/core/src/exec/DataflowExecutor.h
@@ -56,7 +56,8 @@ public:
   DataflowExecutor(const std::shared_ptr<const model::Model> &model,
                    std::unique_ptr<model::SubgraphContext> subg_ctx,
                    const std::shared_ptr<compiler::OperandContext> &operand_context,
-                   std::unique_ptr<graph::LowerInfoMap> lower_info, CodeMap &&code_map);
+                   std::unique_ptr<graph::LowerInfoMap> lower_info,
+                   std::unique_ptr<backend::MemoryManagerSet> mem_mgrs, CodeMap &&code_map);
 
   void executeImpl() override;
 

diff --git a/runtimes/neurun/core/src/exec/ExecutorBase.cc b/runtimes/neurun/core/src/exec/ExecutorBase.cc
index 0a7593b..f48bfb5 100644 (file)
--- a/runtimes/neurun/core/src/exec/ExecutorBase.cc
+++ b/runtimes/neurun/core/src/exec/ExecutorBase.cc
@@ -24,9 +24,11 @@ namespace exec
 ExecutorBase::ExecutorBase(const std::shared_ptr<const model::Model> &model,
                            std::unique_ptr<model::SubgraphContext> subg_ctx,
                            const std::shared_ptr<compiler::OperandContext> &operand_context,
-                           std::unique_ptr<graph::LowerInfoMap> lower_info)
+                           std::unique_ptr<graph::LowerInfoMap> lower_info,
+                           std::unique_ptr<backend::MemoryManagerSet> mem_mgrs)
     : _observers(), _model{model}, _subg_ctx{std::move(subg_ctx)},
-      _operand_context{operand_context}, _lower_info{std::move(lower_info)}
+      _operand_context{operand_context}, _lower_info{std::move(lower_info)},
+      _mem_mgrs{std::move(mem_mgrs)}
 {
   _sources.resize(_model->inputs.size());
   _sinks.resize(_model->outputs.size());

diff --git a/runtimes/neurun/core/src/exec/ExecutorBase.h b/runtimes/neurun/core/src/exec/ExecutorBase.h
index 821bc2b..cf6d7df 100644 (file)
--- a/runtimes/neurun/core/src/exec/ExecutorBase.h
+++ b/runtimes/neurun/core/src/exec/ExecutorBase.h
@@ -31,6 +31,7 @@
 #include "model/Subgraph.h"
 #include "backend/ExecTime.h"
 #include "exec/IFunction.h"
+#include "backend/IMemoryManager.h"
 #include <list>
 
 namespace neurun
@@ -44,7 +45,8 @@ public:
   ExecutorBase(const std::shared_ptr<const model::Model> &model,
                std::unique_ptr<model::SubgraphContext> subg_ctx,
                const std::shared_ptr<compiler::OperandContext> &operand_context,
-               std::unique_ptr<graph::LowerInfoMap> lower_info);
+               std::unique_ptr<graph::LowerInfoMap> lower_info,
+               std::unique_ptr<backend::MemoryManagerSet> mem_mgrs);
 
   virtual ~ExecutorBase() = default;
 
@@ -136,6 +138,7 @@ protected:
   std::unique_ptr<model::SubgraphContext> _subg_ctx;
   std::shared_ptr<compiler::OperandContext> _operand_context;
   std::unique_ptr<graph::LowerInfoMap> _lower_info;
+  std::unique_ptr<backend::MemoryManagerSet> _mem_mgrs;
   std::vector<std::unique_ptr<ISource>> _sources;
   std::vector<std::unique_ptr<ISink>> _sinks;
 };

diff --git a/runtimes/neurun/core/src/exec/LinearExecutor.h b/runtimes/neurun/core/src/exec/LinearExecutor.h
index bc60424..36e429d 100644 (file)
--- a/runtimes/neurun/core/src/exec/LinearExecutor.h
+++ b/runtimes/neurun/core/src/exec/LinearExecutor.h
@@ -46,9 +46,11 @@ public:
                  std::unique_ptr<model::SubgraphContext> subg_ctx,
                  const std::shared_ptr<compiler::OperandContext> &operand_context,
                  std::unique_ptr<graph::LowerInfoMap> lower_info,
+                 std::unique_ptr<backend::MemoryManagerSet> mem_mgrs,
                  std::unique_ptr<std::vector<linear::Element>> elements,
                  const std::shared_ptr<const neurun::compiler::Plan> &plan)
-      : ExecutorBase{model, std::move(subg_ctx), operand_context, std::move(lower_info)},
+      : ExecutorBase{model, std::move(subg_ctx), operand_context, std::move(lower_info),
+                     std::move(mem_mgrs)},
         _plan{plan}, _elements{std::move(elements)}
   {
   }

diff --git a/runtimes/neurun/core/src/exec/ParallelExecutor.cc b/runtimes/neurun/core/src/exec/ParallelExecutor.cc
index 9a06086..3a1d3ad 100644 (file)
--- a/runtimes/neurun/core/src/exec/ParallelExecutor.cc
+++ b/runtimes/neurun/core/src/exec/ParallelExecutor.cc
@@ -58,8 +58,13 @@ ParallelExecutor::ParallelExecutor(const std::shared_ptr<const model::Model> &mo
                                    std::unique_ptr<model::SubgraphContext> subg_ctx,
                                    const std::shared_ptr<compiler::OperandContext> &operand_context,
                                    std::unique_ptr<graph::LowerInfoMap> lower_info,
+                                   std::unique_ptr<backend::MemoryManagerSet> mem_mgrs,
                                    CodeMap &&code_map)
-    : DataflowExecutor{model, std::move(subg_ctx), operand_context, std::move(lower_info),
+    : DataflowExecutor{model,
+                       std::move(subg_ctx),
+                       operand_context,
+                       std::move(lower_info),
+                       std::move(mem_mgrs),
                        std::move(code_map)}
 {
   VERBOSE(ParallelExecutor) << "Constructing Parallel Executor" << std::endl;

diff --git a/runtimes/neurun/core/src/exec/ParallelExecutor.h b/runtimes/neurun/core/src/exec/ParallelExecutor.h
index 12215e8..61793d8 100644 (file)
--- a/runtimes/neurun/core/src/exec/ParallelExecutor.h
+++ b/runtimes/neurun/core/src/exec/ParallelExecutor.h
@@ -56,7 +56,8 @@ public:
   ParallelExecutor(const std::shared_ptr<const model::Model> &model,
                    std::unique_ptr<model::SubgraphContext> subg_ctx,
                    const std::shared_ptr<compiler::OperandContext> &operand_context,
-                   std::unique_ptr<graph::LowerInfoMap> lower_info, CodeMap &&code_map);
+                   std::unique_ptr<graph::LowerInfoMap> lower_info,
+                   std::unique_ptr<backend::MemoryManagerSet> mem_mgrs, CodeMap &&code_map);
 
   void executeImpl() override;