Fix deadlock when access two acl_cl at once (#6500)

To avoid deadlock when operation need to access two acl_cl at once (source, target), introduce access function to access two tensor

Signed-off-by: Hyeongseok Oh <hseok82.oh@samsung.com>

diff --git a/runtimes/neurun/backend/acl_cl/kernel/ConcatLayer.cc b/runtimes/neurun/backend/acl_cl/kernel/ConcatLayer.cc
index bbe97c5..de6b3c0 100644 (file)
--- a/runtimes/neurun/backend/acl_cl/kernel/ConcatLayer.cc
+++ b/runtimes/neurun/backend/acl_cl/kernel/ConcatLayer.cc
@@ -84,12 +84,12 @@ template <typename T> bool ConcatLayer::concatenate()
   {
     uint32_t axis_offset = 0;
 
-    auto outout_fn = [&](::neurun::backend::operand::ITensor &out_tensor) {
-      for (auto input : _input_allocs)
+    auto fn = [&](::neurun::backend::operand::ITensor &out_tensor,
+                  ::neurun::backend::operand::ITensor &in_tensor) {
       {
         auto &out_cl_tensor =
             static_cast<::neurun::backend::acl_cl::operand::ICLTensor &>(out_tensor);
-        auto input_fn = [&](::neurun::backend::operand::ITensor &in_tensor) {
+        {
           auto &in_cl_tensor =
               static_cast<::neurun::backend::acl_cl::operand::ICLTensor &>(in_tensor);
           for (uint32_t i = 0; i < in_cl_tensor.info()->dimension(0); i++)
@@ -120,11 +120,14 @@ template <typename T> bool ConcatLayer::concatenate()
             axis_offset += in_cl_tensor.info()->dimension(2);
           if (_axis == 3)
             axis_offset += in_cl_tensor.info()->dimension(3);
-        };
-        input->access(input_fn);
+        }
       }
     };
-    _output_alloc->access(outout_fn);
+
+    for (auto input : _input_allocs)
+    {
+      _output_alloc->access(fn, *input);
+    }
   }
 
   VERBOSE(Concat_RUN) << "End   Concat" << std::endl;

diff --git a/runtimes/neurun/backend/acl_cl/operand/Object.cc b/runtimes/neurun/backend/acl_cl/operand/Object.cc
index 104ce9c..d49cb26 100644 (file)
--- a/runtimes/neurun/backend/acl_cl/operand/Object.cc
+++ b/runtimes/neurun/backend/acl_cl/operand/Object.cc
@@ -47,6 +47,30 @@ void Object::access(const std::function<void(backend::operand::ITensor &tensor)>
   _tensor->unmap(queue);
 }
 
+void Object::access(const std::function<void(backend::operand::ITensor &tensor,
+                                             backend::operand::ITensor &tensor_other)> &fn,
+                    backend::acl_cl::operand::Object &tensor_other) const
+{
+  // This is an optional input
+  if (_tensor->total_size() == 0 || tensor_other.ptr()->total_size() == 0)
+    return;
+
+  auto &queue = ::arm_compute::CLScheduler::get().queue();
+  // Calling access for more than one tensor causing an error in map(). May be sum of memory
+  // requests are too large.
+  // TODO: Need to investigate. Ideally it must have a non-static mutex and
+  //       counter and in case of parallel call of THE SAME OBJECT,
+  //       call the map() for the first one and unmap() for the last one
+  std::lock_guard<std::mutex> lock{_mu};
+  _tensor->map(queue);
+  tensor_other.ptr()->map(queue);
+
+  fn(*_tensor, *tensor_other.ptr());
+
+  tensor_other.ptr()->unmap(queue);
+  _tensor->unmap(queue);
+}
+
 } // namespace operand
 } // namespace acl_cl
 } // namespace backend

diff --git a/runtimes/neurun/backend/acl_cl/operand/Object.h b/runtimes/neurun/backend/acl_cl/operand/Object.h
index 4499e45..c0789ee 100644 (file)
--- a/runtimes/neurun/backend/acl_cl/operand/Object.h
+++ b/runtimes/neurun/backend/acl_cl/operand/Object.h
@@ -52,6 +52,9 @@ private:
 
 public:
   void access(const std::function<void(backend::operand::ITensor &tensor)> &fn) const override;
+  void access(const std::function<void(backend::operand::ITensor &tensor,
+                                       backend::operand::ITensor &tensor_other)> &fn,
+              backend::acl_cl::operand::Object &tensor_other) const;
 };
 
 } // namespace operand