Make cpu KernelGenerator supports NCHW frontend layout (#6605)

This commit makes cpu KernelGenerator supports NCHW frontend layout.

Signed-off-by: jiseob.jang <jiseob.jang@samsung.com>

diff --git a/runtimes/neurun/backend/cpu/KernelGenerator.cc b/runtimes/neurun/backend/cpu/KernelGenerator.cc
index 2708783..3b6a25b 100644 (file)
--- a/runtimes/neurun/backend/cpu/KernelGenerator.cc
+++ b/runtimes/neurun/backend/cpu/KernelGenerator.cc
@@ -75,8 +75,8 @@ void KernelGenerator::visit(const model::operation::Conv2DNode &node)
   const auto bias_index{node.getInputs().at(Conv2DNode::Input::BIAS)};
 
   const auto stride = node.param().stride;
-  const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature();
-  const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature();
+  const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature(_current_subg_layout);
+  const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature(_current_subg_layout);
   // Kernel format is [depth_out, kernel_height, kernel_width, depth_in].
   const auto &ker_shape = _ctx.at(ker_index).shape();
   const auto ker_height = ker_shape.dim(1);
@@ -85,10 +85,14 @@ void KernelGenerator::visit(const model::operation::Conv2DNode &node)
                                                       stride, ker_width, ker_height);
   const auto activation = node.param().activation;
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index));
-  const auto ifm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ifm_index));
-  const auto ker_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ker_index));
-  const auto bias_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(bias_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index), _current_subg_layout);
+  const auto ifm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ifm_index), _current_subg_layout);
+  const auto ker_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ker_index), model::Layout::UNKNOWN);
+  const auto bias_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(bias_index), model::Layout::UNKNOWN);
 
   auto ofm_alloc = _tensor_builder->at(ofm_index);
   auto ifm_alloc = _tensor_builder->at(ifm_index);
@@ -115,8 +119,8 @@ void KernelGenerator::visit(const model::operation::DepthwiseConv2DNode &node)
   const auto bias_index{node.getInputs().at(DepthwiseConv2DNode::Input::BIAS)};
 
   const auto stride = node.param().stride;
-  const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature();
-  const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature();
+  const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature(_current_subg_layout);
+  const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature(_current_subg_layout);
   // Kernel format is [1, kernel_height, kernel_width, depth_out].
   const auto &ker_shape = _ctx.at(ker_index).shape();
   const auto ker_height = ker_shape.dim(1);
@@ -124,10 +128,14 @@ void KernelGenerator::visit(const model::operation::DepthwiseConv2DNode &node)
   const auto padding = neurun::util::calculatePadding(node.param().padding, ifm_shape, ofm_shape,
                                                       stride, ker_width, ker_height);
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index));
-  const auto ifm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ifm_index));
-  const auto ker_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ker_index));
-  const auto bias_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(bias_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index), _current_subg_layout);
+  const auto ifm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ifm_index), _current_subg_layout);
+  const auto ker_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ker_index), model::Layout::UNKNOWN);
+  const auto bias_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(bias_index), model::Layout::UNKNOWN);
 
   const auto multiplier = node.param().multiplier;
   const auto activation = node.param().activation;
@@ -156,14 +164,16 @@ void KernelGenerator::visit(const model::operation::MaxPool2DNode &node)
   const auto kw = node.param().kw;
 
   const auto stride = node.param().stride;
-  const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature();
-  const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature();
+  const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature(_current_subg_layout);
+  const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature(_current_subg_layout);
   const auto padding =
       neurun::util::calculatePadding(node.param().padding, ifm_shape, ofm_shape, stride, kw, kh);
   const auto activation = node.param().activation;
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index));
-  const auto ifm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ifm_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index), _current_subg_layout);
+  const auto ifm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ifm_index), _current_subg_layout);
 
   auto ofm_alloc = _tensor_builder->at(ofm_index).get();
   auto ifm_alloc = _tensor_builder->at(ifm_index).get();
@@ -185,14 +195,16 @@ void KernelGenerator::visit(const model::operation::AvgPool2DNode &node)
   const auto kh = node.param().kh;
   const auto kw = node.param().kw;
   const auto stride = node.param().stride;
-  const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature();
-  const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature();
+  const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature(_current_subg_layout);
+  const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature(_current_subg_layout);
   const auto padding =
       neurun::util::calculatePadding(node.param().padding, ifm_shape, ofm_shape, stride, kw, kh);
   const auto activation = node.param().activation;
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index));
-  const auto ifm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ifm_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index), _current_subg_layout);
+  const auto ifm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ifm_index), _current_subg_layout);
 
   auto ofm_alloc = _tensor_builder->at(ofm_index).get();
   auto ifm_alloc = _tensor_builder->at(ifm_index).get();
@@ -212,10 +224,12 @@ void KernelGenerator::visit(const model::operation::ConcatNode &node)
 
   const auto axis = node.param().axis;
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index), _current_subg_layout);
   std::vector<::neurun::backend::cpu::kernel::Shape> ifm_backend_shapes;
   for (auto &in_idx : node.getInputs())
-    ifm_backend_shapes.emplace_back(::neurun::backend::cpu::kernel::getShape(_ctx.at(in_idx)));
+    ifm_backend_shapes.emplace_back(
+        ::neurun::backend::cpu::kernel::getShape(_ctx.at(in_idx), _current_subg_layout));
 
   auto output_alloc = _tensor_builder->at(ofm_index).get();
 
@@ -239,10 +253,14 @@ void KernelGenerator::visit(const model::operation::FullyConnectedNode &node)
   const auto weight_index{node.getInputs().at(FullyConnectedNode::Input::WEIGHT)};
   const auto bias_index{node.getInputs().at(FullyConnectedNode::Input::BIAS)};
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(output_index));
-  const auto ifm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(input_index));
-  const auto weight_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(weight_index));
-  const auto bias_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(bias_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(output_index), _current_subg_layout);
+  const auto ifm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(input_index), _current_subg_layout);
+  const auto weight_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(weight_index), model::Layout::UNKNOWN);
+  const auto bias_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(bias_index), model::Layout::UNKNOWN);
 
   const auto activation = node.param().activation;
 
@@ -267,8 +285,10 @@ void KernelGenerator::visit(const model::operation::ReshapeNode &node)
   const auto output_index{node.getOutputs().at(0)};
   const auto input_index{node.getInputs().at(model::operation::ReshapeNode::Input::INPUT)};
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(output_index));
-  const auto ifm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(input_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(output_index), _current_subg_layout);
+  const auto ifm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(input_index), _current_subg_layout);
 
   auto output_alloc = _tensor_builder->at(output_index).get();
   auto input_alloc = _tensor_builder->at(input_index).get();
@@ -286,8 +306,10 @@ void KernelGenerator::visit(const model::operation::SoftmaxNode &node)
   const auto output_index{node.getOutputs().at(0)};
   const auto input_index{node.getInputs().at(model::operation::SoftmaxNode::Input::INPUT)};
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(output_index));
-  const auto ifm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(input_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(output_index), _current_subg_layout);
+  const auto ifm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(input_index), _current_subg_layout);
 
   const auto beta = node.param().beta;
 
@@ -308,9 +330,12 @@ void KernelGenerator::visit(const model::operation::AddNode &node)
   const auto lhs_index{node.getInputs().at(model::operation::AddNode::Input::LHS)};
   const auto rhs_index{node.getInputs().at(model::operation::AddNode::Input::RHS)};
 
-  const auto ofm_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index));
-  const auto lhs_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(lhs_index));
-  const auto rhs_backend_shape = ::neurun::backend::cpu::kernel::getShape(_ctx.at(rhs_index));
+  const auto ofm_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(ofm_index), _current_subg_layout);
+  const auto lhs_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(lhs_index), _current_subg_layout);
+  const auto rhs_backend_shape =
+      ::neurun::backend::cpu::kernel::getShape(_ctx.at(rhs_index), _current_subg_layout);
 
   const auto activation = node.param().activation;
 

diff --git a/runtimes/neurun/backend/cpu/kernel/OperationUtils.cc b/runtimes/neurun/backend/cpu/kernel/OperationUtils.cc
index d30eb00..40b7ef3 100644 (file)
--- a/runtimes/neurun/backend/cpu/kernel/OperationUtils.cc
+++ b/runtimes/neurun/backend/cpu/kernel/OperationUtils.cc
@@ -180,12 +180,22 @@ int32_t CalculateInputRadius(int input_integer_bits, int input_left_shift)
   return static_cast<int32_t>(std::floor(max_input_rescaled));
 }
 
-Shape getShape(const ::neurun::model::Operand &o)
+Shape getShape(const ::neurun::model::Operand &o, ::neurun::model::Layout frontend_layout)
 {
   Shape shape;
 
+  auto dims = o.shape().dims();
+  if (frontend_layout == ::neurun::model::Layout::NCHW && o.shape().rank() == 4)
+  {
+    // NCHW -> NHWC
+    uint32_t permutation[4] = {0, 2, 3, 1};
+    for (int i = 0; i < o.shape().rank(); ++i)
+    {
+      dims.at(i) = o.shape().dim(permutation[i]);
+    }
+  }
+  shape.dimensions = std::vector<uint32_t>(dims.begin(), dims.end());
   shape.type = static_cast<OperandType>(static_cast<int32_t>(o.typeInfo().type()));
-  shape.dimensions = std::vector<uint32_t>(o.shape().dims().begin(), o.shape().dims().end());
   shape.scale = o.typeInfo().scale();
   shape.offset = o.typeInfo().offset();
 

diff --git a/runtimes/neurun/backend/cpu/kernel/OperationUtils.h b/runtimes/neurun/backend/cpu/kernel/OperationUtils.h
index bab0c5b..93a8f41 100644 (file)
--- a/runtimes/neurun/backend/cpu/kernel/OperationUtils.h
+++ b/runtimes/neurun/backend/cpu/kernel/OperationUtils.h
@@ -118,7 +118,7 @@ void CalculateActivationRangeUint8(model::Activation activation, const Shape &ou
 
 int32_t CalculateInputRadius(int input_integer_bits, int input_left_shift);
 
-Shape getShape(const ::neurun::model::Operand &o);
+Shape getShape(const ::neurun::model::Operand &o, ::neurun::model::Layout frontend_layout);
 
 uint32_t sizeOfData(OperandType type, const std::vector<uint32_t> &dimensions);