[nnc] Support for Caffe Embed layer (#2646)

* Add support for Embed layer.
* Refactor creating of BiasAdd op in Caffe importer.

Signed-off-by: Sergei Barannikov <s.barannikov@samsung.com>

diff --git a/contrib/nnc/passes/caffe_frontend/caffe_importer.cpp b/contrib/nnc/passes/caffe_frontend/caffe_importer.cpp
index bfb9b30..646486e 100644 (file)
--- a/contrib/nnc/passes/caffe_frontend/caffe_importer.cpp
+++ b/contrib/nnc/passes/caffe_frontend/caffe_importer.cpp
@@ -128,6 +128,9 @@ void CaffeImporter::createMIRNodesFromLayer(const LayerParameter& layer) {
     case CaffeOpType ::eltwise:
       outputs = _opCreator->convertEltwise(layer, inputs);
       break;
+    case CaffeOpType::embed:
+      outputs = _opCreator->convertEmbed(layer, inputs);
+      break;
     case CaffeOpType ::deconvolution:
       outputs = _opCreator->convertDeconvolution(layer, inputs);
       break;
@@ -162,6 +165,7 @@ void CaffeImporter::collectUnsupportedOp(const LayerParameter& lp) {
     case CaffeOpType::split:
     case CaffeOpType::eltwise:
     case CaffeOpType::ELU:
+    case CaffeOpType::embed:
     case CaffeOpType::tanh:
       // No checks
       break;

diff --git a/contrib/nnc/passes/caffe_frontend/caffe_op_creator.cpp b/contrib/nnc/passes/caffe_frontend/caffe_op_creator.cpp
index 0360361..6657ec9 100644 (file)
--- a/contrib/nnc/passes/caffe_frontend/caffe_op_creator.cpp
+++ b/contrib/nnc/passes/caffe_frontend/caffe_op_creator.cpp
@@ -14,24 +14,25 @@
  * limitations under the License.
  */
 
+#include "core/modelIR/operations/BatchNormOp.h"
+#include "core/modelIR/operations/BiasAddOp.h"
+#include "core/modelIR/operations/CappedReluOp.h"
 #include "core/modelIR/operations/ConcatOp.h"
 #include "core/modelIR/operations/Conv2DOp.h"
+#include "core/modelIR/operations/Deconv2DOp.h"
 #include "core/modelIR/operations/DepthwiseConv2DOp.h"
-#include "core/modelIR/operations/ReluOp.h"
-#include "core/modelIR/operations/CappedReluOp.h"
-#include "core/modelIR/operations/SoftmaxOp.h"
+#include "core/modelIR/operations/DropoutOp.h"
+#include "core/modelIR/operations/ElementwiseOp.h"
+#include "core/modelIR/operations/EluOp.h"
+#include "core/modelIR/operations/FullyConnectedOp.h"
+#include "core/modelIR/operations/GatherOp.h"
 #include "core/modelIR/operations/PoolOp.h"
-#include "core/modelIR/operations/BiasAddOp.h"
+#include "core/modelIR/operations/ReluOp.h"
 #include "core/modelIR/operations/ReshapeOp.h"
-#include "core/modelIR/operations/FullyConnectedOp.h"
 #include "core/modelIR/operations/ScaleOp.h"
-#include "core/modelIR/operations/BatchNormOp.h"
-#include "core/modelIR/operations/DropoutOp.h"
-#include "core/modelIR/operations/ElementwiseOp.h"
-#include "core/modelIR/operations/Deconv2DOp.h"
+#include "core/modelIR/operations/SoftmaxOp.h"
 #include "core/modelIR/operations/TanhOp.h"
 #include "core/modelIR/operations/TransposeOp.h"
-#include "core/modelIR/operations/EluOp.h"
 
 #include "core/modelIR/Index.h"
 #include "core/modelIR/ShapeRange.h"
@@ -182,45 +183,43 @@ void CaffeOpCreator::checkConvolution(const ConvolutionParameter& opts,
 std::vector<mir::IODescriptor>
 CaffeOpCreator::convertConvolution(const caffe::LayerParameter& layer,
                                    const std::vector<mir::IODescriptor>& inputs) {
-  auto& opts = layer.convolution_param();
+  const auto& params = layer.convolution_param();
   Shape strides;
   std::vector<int32_t> padding;
 
-  convertConvolutionParam(opts, strides, padding);
+  convertConvolutionParam(params, strides, padding);
 
   assert(layer.blobs(0).shape().dim_size() == 4);
   auto kernel_weights = convertBlob(layer.blobs(0));
   kernel_weights = transposeTensor<2, 3, 1, 0>(kernel_weights);
 
-  Operation* conv2d;
+  Operation* result;
   auto in_group_size = kernel_weights->getShape().dim(2);
   auto out_channels = kernel_weights->getShape().dim(3);
-  int32_t num_groups = opts.group();
+  int32_t num_groups = params.group();
   bool is_depthwise = (num_groups != 1) && (in_group_size == 1) && (out_channels == num_groups);
   if (is_depthwise) {
     // This is depthwise convolution
     // TODO handle properly kernel with layer multiplier
     auto transposed_tensor = transposeTensor<0, 1, 3, 2>(kernel_weights);
-    conv2d = createOp<ops::DepthwiseConv2DOp>(layer.name(), convertCaffeToMIR(inputs[0]),
+    result = createOp<ops::DepthwiseConv2DOp>(layer.name(), convertCaffeToMIR(inputs[0]),
                                               *transposed_tensor, strides, padding, padding);
   } else {
     if (num_groups != 1) {
       // first we need to convert kernel of grouped convolution to appropriate ordinary kernel
-      kernel_weights = fixGroupedKernel(opts.group(), kernel_weights);
+      kernel_weights = fixGroupedKernel(params.group(), kernel_weights);
     }
-    conv2d = createOp<ops::Conv2DOp>(layer.name(), convertCaffeToMIR(inputs[0]), *kernel_weights,
+    result = createOp<ops::Conv2DOp>(layer.name(), convertCaffeToMIR(inputs[0]), *kernel_weights,
                                      strides, padding, padding);
   }
 
   // Add the bias, if any.
-  if (opts.bias_term()) {
+  if (params.bias_term()) {
     auto bias_weights = convertBlob(layer.blobs(1));
-    auto bias_add = createOp<ops::BiasAddOp>(layer.name() + ".bias", conv2d->getOutput(0),
-                                             *bias_weights);
-    return {convertMIRToCaffe(bias_add->getOutput(0))};
-  } else {
-    return {convertMIRToCaffe(conv2d->getOutput(0))};
+    result = createOp<ops::BiasAddOp>(layer.name() + ".bias", result->getOutput(0), *bias_weights);
   }
+
+  return {convertMIRToCaffe(result->getOutput(0))};
 }
 
 std::vector<mir::IODescriptor>
@@ -270,29 +269,26 @@ void CaffeOpCreator::checkInnerProduct(const InnerProductParameter& opts,
 std::vector<mir::IODescriptor>
 CaffeOpCreator::convertInnerProduct(const LayerParameter& layer,
                                     const std::vector<mir::IODescriptor>& inputs) {
-  auto& opts = layer.inner_product_param();
+  const auto& params = layer.inner_product_param();
   auto weights = convertBlob(layer.blobs(0));
 
-  if (!opts.transpose())
+  if (!params.transpose())
     weights = transposeTensor<1, 0>(weights);
 
   auto& input_shape = inputs[0].op->getOutputShape(inputs[0].index);
-  // Transform input into 2-D tensor by flattening axes before/after opts.axis().
-  assert(opts.axis() == 1);
+  // Transform input into 2-D tensor by flattening axes before/after params.axis().
+  assert(params.axis() == 1);
   Shape shape{input_shape.dim(0), input_shape.numElements() / input_shape.dim(0)};
   auto reshape = createOp<ops::ReshapeOp>(layer.name() + ".reshape", inputs[0], shape);
-  auto fully_connected = createOp<ops::FullyConnectedOp>(layer.name() + ".fc",
-                                                         reshape->getOutput(0), *weights);
+  auto result = createOp<ops::FullyConnectedOp>(layer.name(), reshape->getOutput(0), *weights);
 
   // Add the bias, if any.
-  if (opts.bias_term()) {
+  if (params.bias_term()) {
     auto bias_weights = convertBlob(layer.blobs(1));
-    auto bias_add = createOp<ops::BiasAddOp>(layer.name() + ".bias", fully_connected->getOutput(0),
-                                             *bias_weights);
-    return {bias_add->getOutput(0)};
-  } else {
-    return {fully_connected->getOutput(0)};
+    result = createOp<ops::BiasAddOp>(layer.name() + ".bias", result->getOutput(0), *bias_weights);
   }
+
+  return {result->getOutput(0)};
 }
 
 std::vector<mir::IODescriptor>
@@ -474,19 +470,17 @@ CaffeOpCreator::convertReLU(const caffe::LayerParameter& layer,
 std::vector<mir::IODescriptor>
 CaffeOpCreator::convertScale(const caffe::LayerParameter& layer,
                              const std::vector<mir::IODescriptor>& inputs) {
-  auto& opts = layer.scale_param();
-  auto scale_weights = convertBlob(layer.blobs(0));
-  auto scale = createOp<ops::ScaleOp>(layer.name(), convertCaffeToMIR(inputs[0]), *scale_weights);
+  const auto& params = layer.scale_param();
+  auto weights = convertBlob(layer.blobs(0));
+  auto result = createOp<ops::ScaleOp>(layer.name(), convertCaffeToMIR(inputs[0]), *weights);
 
   // Add the bias, if any.
-  if (opts.bias_term()) {
+  if (params.bias_term()) {
     auto bias_weights = convertBlob(layer.blobs(1));
-    auto bias_add = createOp<ops::BiasAddOp>(layer.name() + ".bias", scale->getOutput(0),
-                                             *bias_weights);
-    return {convertMIRToCaffe(bias_add->getOutput(0))};
-  } else {
-    return {convertMIRToCaffe(scale->getOutput(0))};
+    result = createOp<ops::BiasAddOp>(layer.name() + ".bias", result->getOutput(0), *bias_weights);
   }
+
+  return {result->getOutput(0)};
 }
 
 void CaffeOpCreator::checkBatchNorm(const caffe::LayerParameter& layer,
@@ -550,6 +544,23 @@ CaffeOpCreator::convertELU(const caffe::LayerParameter& layer,
 }
 
 std::vector<mir::IODescriptor>
+CaffeOpCreator::convertEmbed(const caffe::LayerParameter& layer,
+                             const std::vector<mir::IODescriptor>& inputs) {
+  const auto& params = layer.embed_param();
+  auto data = createOp<ops::ConstantOp>(layer.name() + ".weights", *convertBlob(layer.blobs(0)));
+  // FIXME Indices in Caffe have floating type, while in ModelIR they are integral.
+  auto result = createOp<ops::GatherOp>(layer.name(), data->getOutput(0), inputs[0], 0);
+
+  // Add the bias, if any.
+  if (params.bias_term()) {
+    auto bias_weights = convertBlob(layer.blobs(1));
+    result = createOp<ops::BiasAddOp>(layer.name() + ".bias", result->getOutput(0), *bias_weights);
+  }
+
+  return {result->getOutput(0)};
+}
+
+std::vector<mir::IODescriptor>
 CaffeOpCreator::convertTanH(const caffe::LayerParameter& layer,
                             const std::vector<mir::IODescriptor>& inputs) {
   auto tanh = createOp<ops::TanhOp>(layer.name(), inputs[0]);

diff --git a/contrib/nnc/passes/caffe_frontend/caffe_op_creator.h b/contrib/nnc/passes/caffe_frontend/caffe_op_creator.h
index 728ea76..2c1b78a 100644 (file)
--- a/contrib/nnc/passes/caffe_frontend/caffe_op_creator.h
+++ b/contrib/nnc/passes/caffe_frontend/caffe_op_creator.h
@@ -89,6 +89,10 @@ public:
              const std::vector<mir::IODescriptor>& inputs);
 
   std::vector<mir::IODescriptor>
+  convertEmbed(const caffe::LayerParameter& layer,
+               const std::vector<mir::IODescriptor>& inputs);
+
+  std::vector<mir::IODescriptor>
   convertTanH(const caffe::LayerParameter& layer,
               const std::vector<mir::IODescriptor>& inputs);