Change ConcatLayer to support NHWC (#5165)

This commit ConcatLayer to support NHWC.

Signed-off-by: jiseob.jang <jiseob.jang@samsung.com>

diff --git a/runtimes/neurun/backend/acl_cl/kernel/ConcatLayer.cc b/runtimes/neurun/backend/acl_cl/kernel/ConcatLayer.cc
index a2a7136..dd23260 100644 (file)
--- a/runtimes/neurun/backend/acl_cl/kernel/ConcatLayer.cc
+++ b/runtimes/neurun/backend/acl_cl/kernel/ConcatLayer.cc
@@ -68,6 +68,7 @@ template <typename T> bool ConcatLayer::concatenate()
 
     for (auto input : _input_allocs)
     {
+      assert(_output_alloc->layout() == input->layout());
       assert(matchSizeExceptAxis(_output_alloc, input, _axis));
       axis_sum += input->dimension(_axis);
     }
@@ -84,38 +85,38 @@ template <typename T> bool ConcatLayer::concatenate()
     auto &queue = ::arm_compute::CLScheduler::get().queue();
 
     _output_alloc->map(queue);
-    util::feature::nchw::View<T> output_view{_output_alloc};
 
     for (auto input : _input_allocs)
     {
       input->map(queue);
-      const util::feature::nchw::View<T> input_reader{input};
 
-      for (uint32_t n = 0; n < input_reader.shape().N; n++)
+      for (int32_t i = 0; i < input->info()->dimension(0); i++)
       {
-        for (uint32_t c = 0; c < input_reader.shape().C; c++)
+        for (int32_t j = 0; j < input->info()->dimension(1); j++)
         {
-          for (uint32_t h = 0; h < input_reader.shape().H; h++)
+          for (int32_t k = 0; k < input->info()->dimension(2); k++)
           {
-            for (uint32_t w = 0; w < input_reader.shape().W; w++)
+            for (int32_t l = 0; l < input->info()->dimension(3); l++)
             {
-              uint32_t no = (_axis == 3) ? axis_offset : 0;
-              uint32_t co = (_axis == 2) ? axis_offset : 0;
-              uint32_t ho = (_axis == 1) ? axis_offset : 0;
-              uint32_t wo = (_axis == 0) ? axis_offset : 0;
-              output_view.at(n + no, c + co, h + ho, w + wo) = input_reader.at(n, c, h, w);
+              int32_t io = (_axis == 0) ? axis_offset : 0;
+              int32_t jo = (_axis == 1) ? axis_offset : 0;
+              int32_t ko = (_axis == 2) ? axis_offset : 0;
+              int32_t lo = (_axis == 3) ? axis_offset : 0;
+              T value = *reinterpret_cast<T *>(input->handle()->ptr_to_element({i, j, k, l}));
+              *reinterpret_cast<T *>(_output_alloc->handle()->ptr_to_element(
+                  {i + io, j + jo, k + ko, l + lo})) = value;
             }
           }
         }
       }
-      if (_axis == 3)
-        axis_offset += input_reader.shape().N;
-      if (_axis == 2)
-        axis_offset += input_reader.shape().C;
-      if (_axis == 1)
-        axis_offset += input_reader.shape().H;
       if (_axis == 0)
-        axis_offset += input_reader.shape().W;
+        axis_offset += input->info()->dimension(0);
+      if (_axis == 1)
+        axis_offset += input->info()->dimension(1);
+      if (_axis == 2)
+        axis_offset += input->info()->dimension(2);
+      if (_axis == 3)
+        axis_offset += input->info()->dimension(3);
 
       input->unmap(queue);
     }
@@ -139,8 +140,9 @@ void ConcatLayer::configure(
   // TODO Handle when axis is negative
   assert(axis >= 0);
 
-  // This map converts NHWC to NCHW(reversed)
-  // NHWC -> WHCN
+  // This map converts to be reversed
+  // NHWC -> CWHN
+  // NCHW -> WHCN
   _axis =
       ::neurun::backend::acl_common::ToARMComputeAxis(output_alloc->num_dimensions(), axis).value();
 

diff --git a/runtimes/neurun/backend/acl_cl/operand/CLSubTensor.h b/runtimes/neurun/backend/acl_cl/operand/CLSubTensor.h
index 04f2ba1..e2817af 100644 (file)
--- a/runtimes/neurun/backend/acl_cl/operand/CLSubTensor.h
+++ b/runtimes/neurun/backend/acl_cl/operand/CLSubTensor.h
@@ -51,6 +51,10 @@ protected:
   uint8_t *doMap(cl::CommandQueue &q, bool blocking) override;
   void doUnmap(cl::CommandQueue &q) override;
 
+public:
+  // This method is used to prevent the use of memcpy for SubTensor
+  bool has_padding() const override { return true; }
+
 private:
   std::shared_ptr<arm_compute::CLSubTensor> _cl_sub_tensor;
 };

diff --git a/runtimes/neurun/backend/acl_neon/kernel/ConcatLayer.cc b/runtimes/neurun/backend/acl_neon/kernel/ConcatLayer.cc
index 5581d16..8d73980 100644 (file)
--- a/runtimes/neurun/backend/acl_neon/kernel/ConcatLayer.cc
+++ b/runtimes/neurun/backend/acl_neon/kernel/ConcatLayer.cc
@@ -66,6 +66,7 @@ template <typename T> bool ConcatLayer::concatenate()
 
     for (auto input : _input_allocs)
     {
+      assert(_output_alloc->layout() == input->layout());
       assert(matchSizeExceptAxis(_output_alloc, input, _axis));
       axis_sum += input->dimension(_axis);
     }
@@ -83,33 +84,33 @@ template <typename T> bool ConcatLayer::concatenate()
 
     for (auto input : _input_allocs)
     {
-      const util::feature::nchw::View<T> input_reader{input};
-
-      for (uint32_t n = 0; n < input_reader.shape().N; n++)
+      for (int32_t i = 0; i < input->info()->dimension(0); i++)
       {
-        for (uint32_t c = 0; c < input_reader.shape().C; c++)
+        for (int32_t j = 0; j < input->info()->dimension(1); j++)
         {
-          for (uint32_t h = 0; h < input_reader.shape().H; h++)
+          for (int32_t k = 0; k < input->info()->dimension(2); k++)
           {
-            for (uint32_t w = 0; w < input_reader.shape().W; w++)
+            for (int32_t l = 0; l < input->info()->dimension(3); l++)
             {
-              uint32_t no = (_axis == 3) ? axis_offset : 0;
-              uint32_t co = (_axis == 2) ? axis_offset : 0;
-              uint32_t ho = (_axis == 1) ? axis_offset : 0;
-              uint32_t wo = (_axis == 0) ? axis_offset : 0;
-              output_view.at(n + no, c + co, h + ho, w + wo) = input_reader.at(n, c, h, w);
+              uint32_t io = (_axis == 0) ? axis_offset : 0;
+              uint32_t jo = (_axis == 1) ? axis_offset : 0;
+              uint32_t ko = (_axis == 2) ? axis_offset : 0;
+              uint32_t lo = (_axis == 3) ? axis_offset : 0;
+              T value = *reinterpret_cast<T *>(input->handle()->ptr_to_element({i, j, k, l}));
+              *reinterpret_cast<T *>(_output_alloc->handle()->ptr_to_element(
+                  {i + io, j + jo, k + ko, l + lo})) = value;
             }
           }
         }
       }
-      if (_axis == 3)
-        axis_offset += input_reader.shape().N;
-      if (_axis == 2)
-        axis_offset += input_reader.shape().C;
-      if (_axis == 1)
-        axis_offset += input_reader.shape().H;
       if (_axis == 0)
-        axis_offset += input_reader.shape().W;
+        axis_offset += input->info()->dimension(0);
+      if (_axis == 1)
+        axis_offset += input->info()->dimension(1);
+      if (_axis == 2)
+        axis_offset += input->info()->dimension(2);
+      if (_axis == 3)
+        axis_offset += input->info()->dimension(3);
     }
   }
 
@@ -130,8 +131,9 @@ void ConcatLayer::configure(
   // TODO Handle when axis is negative
   assert(axis >= 0);
 
-  // This map converts NHWC to NCHW(reversed)
-  // NHWC -> WHCN
+  // This map converts to be reversed
+  // NHWC -> CWHN
+  // NCHW -> WHCN
   _axis =
       ::neurun::backend::acl_common::ToARMComputeAxis(output_alloc->num_dimensions(), axis).value();
 

diff --git a/runtimes/neurun/backend/acl_neon/operand/NESubTensor.h b/runtimes/neurun/backend/acl_neon/operand/NESubTensor.h
index 04e6505..67388db 100644 (file)
--- a/runtimes/neurun/backend/acl_neon/operand/NESubTensor.h
+++ b/runtimes/neurun/backend/acl_neon/operand/NESubTensor.h
@@ -43,6 +43,10 @@ public:
   arm_compute::SubTensor *handle() const override;
   arm_compute::SubTensor *handle() override;
 
+public:
+  // This method is used to prevent the use of memcpy for SubTensor
+  bool has_padding() const override { return true; }
+
 private:
   std::shared_ptr<arm_compute::SubTensor> _ne_sub_tensor;
 };