[Interpreter] Introduce buffer allocator and deallocator (#5260)

* Introduce introduce buffer allocator and deallocator

Allocate buffer if need for operation output
Deallocate buffer if all uses are finished

Signed-off-by: Hyeongseok Oh <hseok82.oh@samsung.com>
* Rename allocatedIfNeed to allocatedIfNeeded

diff --git a/runtimes/neurun/core/src/exec/interp/ExecEnv.h b/runtimes/neurun/core/src/exec/interp/ExecEnv.h
index c2dba93..f24bca5 100644 (file)
--- a/runtimes/neurun/core/src/exec/interp/ExecEnv.h
+++ b/runtimes/neurun/core/src/exec/interp/ExecEnv.h
@@ -88,11 +88,47 @@ public:
     return (_tensors.find(index) != _tensors.end());
   }
 
+  /**
+   * @brief     Allocate tensor using operand info
+   * @param[in] index     Tensor index
+   * @param[in] info      Operand info
+   * @note      If already allocated, just return
+   * @TODO      More smart allocation policy
+   */
+  void allocateIfNeeded(const model::OperandIndex index, const model::OperandInfo &info)
+  {
+    // already allocated, or constant
+    if (contains(index))
+    {
+      return;
+    }
+
+    auto tensor = std::make_shared<Tensor>(info);
+    tensor->setBuffer(new uint8_t[tensor->total_size()]);
+    assignTensor(index, tensor);
+    _buffers.insert(index);
+  }
+
+  /**
+   * @brief     Free buffer if allocated by allocateIfNeed
+   * @param[in] index Tensor index
+   * @note      If allocated by outside, just return
+   */
+  void freeIfAllocated(const model::OperandIndex index)
+  {
+    if (_buffers.find(index) != _buffers.end())
+    {
+      delete[] _tensors.at(index)->buffer();
+    }
+  }
+
 private:
   std::shared_ptr<const model::Model> _model;
   // Tensor map to use in interpreter
   // It should map tensors that have allocated or assigned buffer pointer
   std::unordered_map<model::OperandIndex, std::shared_ptr<ITensor>> _tensors;
+  // Tensors allocated by allocateIfNeed (buffer)
+  std::unordered_set<model::OperandIndex> _buffers;
 };
 
 } // namespace interp

diff --git a/runtimes/neurun/core/src/exec/interp/Interpreter.cc b/runtimes/neurun/core/src/exec/interp/Interpreter.cc
index d85372f..4ee8983 100644 (file)
--- a/runtimes/neurun/core/src/exec/interp/Interpreter.cc
+++ b/runtimes/neurun/core/src/exec/interp/Interpreter.cc
@@ -89,6 +89,7 @@ void Interpreter::run()
 
   // Execution
   std::unordered_set<model::OperandIndex> ready_check;
+  std::unordered_set<model::OperationIndex> executed;
   OperationExecutor executor{_env.get()};
   while (!operand_stack.empty())
   {
@@ -135,6 +136,7 @@ void Interpreter::run()
       // 1. Prepare output tensor
       // 2. Call operation kernel
       executor.execute(current_operation_index);
+      executed.insert(current_operation_index);
 
       // 3. Push each output into operand stack
       const auto def_operands = _env->model().operations.at(current_operation_index).getOutputs();
@@ -144,6 +146,26 @@ void Interpreter::run()
                              << std::endl;
         operand_stack.push(def_operand);
       }
+
+      // 4. Free if lifetime of buffer operands used by input is finished
+      for (auto input_index : _env->model().operations.at(current_operation_index).getInputs())
+      {
+        const auto use_operators = _env->model().operands.at(input_index).getUses();
+        bool dead_buffer = true;
+        for (auto use_operator : use_operators.list())
+        {
+          if (executed.find(use_operator) == executed.end())
+          {
+            dead_buffer = false;
+            break;
+          }
+        }
+
+        if (dead_buffer)
+        {
+          _env->freeIfAllocated(input_index);
+        }
+      }
     }
   }
 }

diff --git a/runtimes/neurun/core/src/exec/interp/operations/Add.cc b/runtimes/neurun/core/src/exec/interp/operations/Add.cc
index 82674d0..bbdca36 100644 (file)
--- a/runtimes/neurun/core/src/exec/interp/operations/Add.cc
+++ b/runtimes/neurun/core/src/exec/interp/operations/Add.cc
@@ -59,10 +59,7 @@ void invokeAdd(ExecEnv *env, const model::Operation &node)
   // Output's shape and type should be same with input (don't consider broadcast)
   auto output_info = lhs_tensor->tensorInfo();
   // We can handle already allocated (ex. model output)
-  if (!env->contains(out_index))
-  {
-    throw std::runtime_error("NYI: buffer allocator");
-  }
+  env->allocateIfNeeded(out_index, output_info);
 
   auto out_tensor = env->tensorAt(out_index);
 

diff --git a/runtimes/neurun/test/interp/ExecManager.cc b/runtimes/neurun/test/interp/ExecManager.cc
index 6aabe7f..1a07325 100644 (file)
--- a/runtimes/neurun/test/interp/ExecManager.cc
+++ b/runtimes/neurun/test/interp/ExecManager.cc
@@ -317,11 +317,11 @@ TEST_F(InterpExecManagerTest, executeTwoStep)
   EXPECT_NO_THROW(_executor->setInput(input1, reinterpret_cast<const void *>(input1_buffer), 16));
   EXPECT_NO_THROW(_executor->setInput(input2, reinterpret_cast<const void *>(input2_buffer), 16));
   EXPECT_NO_THROW(_executor->setOutput(output, reinterpret_cast<void *>(output_buffer), 16));
-  EXPECT_THROW(_executor->execute(), std::runtime_error);
-  // EXPECT_EQ(output_buffer[0], 5);
-  // EXPECT_EQ(output_buffer[1], -2);
-  // EXPECT_EQ(output_buffer[2], 0);
-  // EXPECT_EQ(output_buffer[3], -1);
+  EXPECT_NO_THROW(_executor->execute());
+  EXPECT_EQ(output_buffer[0], 5);
+  EXPECT_EQ(output_buffer[1], -2);
+  EXPECT_EQ(output_buffer[2], 0);
+  EXPECT_EQ(output_buffer[3], -1);
 }
 
 } // namespace