modules/dnn/src/onnx/onnx_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2018, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 #include "../precomp.hpp"
   9 #include <opencv2/dnn/shape_utils.hpp>
  10
  11 #ifdef HAVE_PROTOBUF
  12
  13 #include <iostream>
  14 #include <fstream>
  15 #include <string>
  16 #include <limits>
  17 #include <algorithm>
  18
  19
  20 #if defined(__GNUC__) && __GNUC__ >= 5
  21 #pragma GCC diagnostic push
  22 #pragma GCC diagnostic ignored "-Wsuggest-override"
  23 #endif
  24 #include "opencv-onnx.pb.h"
  25 #if defined(__GNUC__) && __GNUC__ >= 5
  26 #pragma GCC diagnostic pop
  27 #endif
  28
  29 #include "onnx_graph_simplifier.hpp"
  30
  31 namespace cv {
  32 namespace dnn {
  33 CV__DNN_EXPERIMENTAL_NS_BEGIN
  34
  35
  36 class ONNXImporter
  37 {
  38     opencv_onnx::ModelProto model_proto;
  39     struct LayerInfo {
  40         int layerId;
  41         int outputId;
  42         LayerInfo(int _layerId, int _outputId) : layerId(_layerId), outputId(_outputId) {}
  43     };
  44
  45     std::map<std::string, Mat> getGraphTensors(
  46                                     const opencv_onnx::GraphProto& graph_proto);
  47     Mat getBlob(const opencv_onnx::NodeProto& node_proto, const std::map<std::string, Mat>& constBlobs, int index);
  48
  49     LayerParams getLayerParams(const opencv_onnx::NodeProto& node_proto);
  50     bool isCeilMode(const LayerParams& layerParams);
  51
  52 public:
  53
  54     ONNXImporter(const char *onnxFile)
  55     {
  56         std::fstream input(onnxFile, std::ios::in | std::ios::binary);
  57
  58         if (!model_proto.ParseFromIstream(&input))
  59             CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model");
  60     }
  61
  62     ONNXImporter(const char* buffer, size_t sizeBuffer)
  63     {
  64         struct _Buf : public std::streambuf
  65         {
  66             _Buf(const char* buffer, size_t sizeBuffer)
  67             {
  68                 char* p = const_cast<char*>(buffer);
  69                 setg(p, p, p + sizeBuffer);
  70             }
  71         };
  72
  73         _Buf buf(buffer, sizeBuffer);
  74         std::istream input(&buf);
  75
  76         if (!model_proto.ParseFromIstream(&input))
  77             CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array.");
  78     }
  79
  80     void populateNet(Net dstNet);
  81 };
  82
  83 inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey)
  84 {
  85     if (layerParams.has(oldKey)) {
  86         layerParams.set(newKey, layerParams.get(oldKey));
  87         layerParams.erase(oldKey);
  88     }
  89 }
  90
  91 void releaseONNXTensor(opencv_onnx::TensorProto& tensor_proto)
  92 {
  93     if (!tensor_proto.raw_data().empty()) {
  94         delete tensor_proto.release_raw_data();
  95     }
  96 }
  97
  98 void runLayer(LayerParams& params, const std::vector<Mat>& inputs,
  99               std::vector<Mat>& outputs)
 100 {
 101     Ptr<Layer> layer = LayerFactory::createLayerInstance(params.type, params);
 102     CV_Assert((bool)layer);
 103
 104     std::vector<MatShape> inpShapes(inputs.size());
 105     int ddepth = CV_32F;
 106     for (size_t i = 0; i < inputs.size(); ++i)
 107     {
 108         inpShapes[i] = shape(inputs[i]);
 109         if (i > 0 && ddepth != inputs[i].depth())
 110             CV_Error(Error::StsNotImplemented, "Mixed input data types.");
 111         ddepth = inputs[i].depth();
 112     }
 113
 114     std::vector<MatShape> outShapes, internalShapes;
 115     layer->getMemoryShapes(inpShapes, 0, outShapes, internalShapes);
 116
 117     std::vector<Mat> internals(internalShapes.size());
 118     outputs.resize(outShapes.size());
 119     for (size_t i = 0; i < outShapes.size(); ++i)
 120         outputs[i].create(outShapes[i], ddepth);
 121     for (size_t i = 0; i < internalShapes.size(); ++i)
 122         internals[i].create(internalShapes[i], ddepth);
 123
 124     layer->finalize(inputs, outputs);
 125     layer->forward(inputs, outputs, internals);
 126 }
 127
 128 std::map<std::string, Mat> ONNXImporter::getGraphTensors(
 129                                         const opencv_onnx::GraphProto& graph_proto)
 130 {
 131   opencv_onnx::TensorProto tensor_proto;
 132   std::map<std::string, Mat> layers_weights;
 133
 134   for (int i = 0; i < graph_proto.initializer_size(); i++)
 135   {
 136     tensor_proto = graph_proto.initializer(i);
 137     Mat mat = getMatFromTensor(tensor_proto);
 138     releaseONNXTensor(tensor_proto);
 139     layers_weights.insert(std::make_pair(tensor_proto.name(), mat));
 140   }
 141   return layers_weights;
 142 }
 143
 144 static DictValue parse(const ::google::protobuf::RepeatedField< ::google::protobuf::int64>& src) {
 145     std::vector<int32_t> dst(src.size());
 146     convertInt64ToInt32(src, dst, src.size());
 147     return DictValue::arrayInt(&dst[0], src.size());
 148 }
 149
 150 LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_proto)
 151 {
 152     LayerParams lp;
 153     for(int i = 0; i < node_proto.attribute_size(); i++)
 154     {
 155         opencv_onnx::AttributeProto attribute_proto = node_proto.attribute(i);
 156         std::string attribute_name = attribute_proto.name();
 157
 158         if(attribute_name == "kernel_shape")
 159         {
 160             CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
 161             lp.set("kernel_size", parse(attribute_proto.ints()));
 162         }
 163         else if(attribute_name == "strides")
 164         {
 165             CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
 166             lp.set("stride", parse(attribute_proto.ints()));
 167         }
 168         else if(attribute_name == "pads")
 169         {
 170             if (node_proto.op_type() == "Pad")
 171             {
 172                 // Padding layer.
 173                 // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN.
 174                 // We need to shuffle it to begin0, end0, begin1, end1, ...
 175                 CV_Assert(attribute_proto.ints_size() % 2 == 0);
 176                 const int dims = attribute_proto.ints_size() / 2;
 177                 std::vector<int32_t> paddings;
 178                 paddings.reserve(attribute_proto.ints_size());
 179                 for (int i = 0; i < dims; ++i)
 180                 {
 181                     paddings.push_back(attribute_proto.ints(i));
 182                     paddings.push_back(attribute_proto.ints(dims + i));
 183                 }
 184                 lp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size()));
 185             }
 186             else
 187             {
 188                 // Convolution or pooling.
 189                 CV_Assert(attribute_proto.ints_size() == 4 || attribute_proto.ints_size() == 6);
 190                 lp.set("pad", parse(attribute_proto.ints()));
 191             }
 192         }
 193         else if(attribute_name == "auto_pad")
 194         {
 195             if (attribute_proto.s() == "SAME_UPPER" || attribute_proto.s() == "SAME_LOWER") {
 196                 lp.set("pad_mode",  "SAME");
 197             }
 198             else if (attribute_proto.s() == "VALID") {
 199                 lp.set("pad_mode", "VALID");
 200             }
 201         }
 202         else if(attribute_name == "dilations")
 203         {
 204             CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
 205             lp.set("dilation", parse(attribute_proto.ints()));
 206         }
 207         else if (attribute_proto.has_i())
 208         {
 209             ::google::protobuf::int64 src = attribute_proto.i();
 210             if (src < std::numeric_limits<int32_t>::min() || src > std::numeric_limits<int32_t>::max())
 211                 CV_Error(Error::StsOutOfRange, "Input is out of OpenCV 32S range");
 212             else
 213                 lp.set(attribute_name, saturate_cast<int32_t>(src));
 214         }
 215         else if (attribute_proto.has_f())
 216         {
 217             lp.set(attribute_name, attribute_proto.f());
 218         }
 219         else if (attribute_proto.has_s())
 220         {
 221             lp.set(attribute_name, attribute_proto.s());
 222         }
 223         else if (attribute_proto.floats_size() > 0)
 224         {
 225             lp.set(attribute_name, DictValue::arrayReal(
 226                 attribute_proto.floats().data(), attribute_proto.floats_size()));
 227         }
 228         else if (attribute_proto.ints_size() > 0)
 229         {
 230             lp.set(attribute_proto.name(), parse(attribute_proto.ints()));
 231         }
 232         else if (attribute_proto.has_t())
 233         {
 234             opencv_onnx::TensorProto tensor = attribute_proto.t();
 235             Mat blob = getMatFromTensor(tensor);
 236             lp.blobs.push_back(blob);
 237         }
 238         else if (attribute_proto.has_g() || attribute_proto.strings_size() > 0 ||
 239                     attribute_proto.tensors_size() > 0 || attribute_proto.graphs_size() > 0)
 240         {
 241                 CV_Error(Error::StsNotImplemented, "Unexpected attribute type");
 242         }
 243         else
 244             CV_Error(Error::StsNotImplemented, "Unsupported attribute type");
 245     }
 246     return lp;
 247 }
 248
 249 Mat ONNXImporter::getBlob(const opencv_onnx::NodeProto& node_proto,
 250                     const std::map<std::string, Mat>& constBlobs, int index)
 251 {
 252     CV_Assert(index < node_proto.input_size());
 253     std::map<std::string, Mat>::const_iterator constBlob;
 254     constBlob = constBlobs.find(node_proto.input(index));
 255     if (constBlob == constBlobs.end()) {
 256         CV_Error(Error::StsObjectNotFound,
 257              "Blob " + node_proto.input(index) + " not found in const blobs");
 258     }
 259     return constBlob->second;
 260 }
 261
 262 void ONNXImporter::populateNet(Net dstNet)
 263 {
 264     CV_Assert(model_proto.has_graph());
 265     opencv_onnx::GraphProto graph_proto = model_proto.graph();
 266
 267     simplifySubgraphs(graph_proto);
 268
 269     std::map<std::string, Mat> constBlobs = getGraphTensors(graph_proto);
 270     // List of internal blobs shapes.
 271     std::map<std::string, MatShape> outShapes;
 272     // Add all the inputs shapes. It includes as constant blobs as network's inputs shapes.
 273     for (int i = 0; i < graph_proto.input_size(); ++i)
 274     {
 275         opencv_onnx::ValueInfoProto valueInfoProto = graph_proto.input(i);
 276         CV_Assert(valueInfoProto.has_type());
 277         opencv_onnx::TypeProto typeProto = valueInfoProto.type();
 278         CV_Assert(typeProto.has_tensor_type());
 279         opencv_onnx::TypeProto::Tensor tensor = typeProto.tensor_type();
 280         CV_Assert(tensor.has_shape());
 281         opencv_onnx::TensorShapeProto tensorShape = tensor.shape();
 282
 283         MatShape inpShape(tensorShape.dim_size());
 284         for (int j = 0; j < inpShape.size(); ++j)
 285         {
 286             inpShape[j] = tensorShape.dim(j).dim_value();
 287         }
 288         outShapes[valueInfoProto.name()] = inpShape;
 289     }
 290
 291     std::string framework_name;
 292     if (model_proto.has_producer_name()) {
 293         framework_name = model_proto.producer_name();
 294     }
 295
 296     // create map with network inputs (without const blobs)
 297     std::map<std::string, LayerInfo> layer_id;
 298     std::map<std::string, LayerInfo>::iterator layerId;
 299     std::map<std::string, MatShape>::iterator shapeIt;
 300     // fill map: push layer name, layer id and output id
 301     std::vector<String> netInputs;
 302     for (int j = 0; j < graph_proto.input_size(); j++)
 303     {
 304         const std::string& name = graph_proto.input(j).name();
 305         if (constBlobs.find(name) == constBlobs.end()) {
 306             netInputs.push_back(name);
 307             layer_id.insert(std::make_pair(name, LayerInfo(0, netInputs.size() - 1)));
 308         }
 309     }
 310     dstNet.setInputsNames(netInputs);
 311
 312     int layersSize = graph_proto.node_size();
 313     LayerParams layerParams;
 314     opencv_onnx::NodeProto node_proto;
 315
 316     for(int li = 0; li < layersSize; li++)
 317     {
 318         node_proto = graph_proto.node(li);
 319         layerParams = getLayerParams(node_proto);
 320         CV_Assert(node_proto.output_size() >= 1);
 321         layerParams.name = node_proto.output(0);
 322
 323         std::string layer_type = node_proto.op_type();
 324         layerParams.type = layer_type;
 325
 326
 327         if (layer_type == "MaxPool")
 328         {
 329             layerParams.type = "Pooling";
 330             layerParams.set("pool", "MAX");
 331             layerParams.set("ceil_mode", layerParams.has("pad_mode"));
 332         }
 333         else if (layer_type == "AveragePool")
 334         {
 335             layerParams.type = "Pooling";
 336             layerParams.set("pool", "AVE");
 337             layerParams.set("ceil_mode", layerParams.has("pad_mode"));
 338             layerParams.set("ave_pool_padded_area", framework_name == "pytorch");
 339         }
 340         else if (layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool" || layer_type == "ReduceMean")
 341         {
 342             CV_Assert(node_proto.input_size() == 1);
 343             layerParams.type = "Pooling";
 344             layerParams.set("pool", layer_type == "GlobalMaxPool"? "MAX" : "AVE");
 345             layerParams.set("global_pooling", layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool");
 346
 347             if (layer_type == "ReduceMean")
 348             {
 349                 if (layerParams.get<int>("keepdims") == 0 || !layerParams.has("axes"))
 350                     CV_Error(Error::StsNotImplemented, "Unsupported mode of ReduceMean operation.");
 351
 352                 MatShape inpShape = outShapes[node_proto.input(0)];
 353                 if (inpShape.size() != 4 && inpShape.size() != 5)
 354                     CV_Error(Error::StsNotImplemented, "Unsupported input shape of reduce_mean operation.");
 355
 356                 DictValue axes = layerParams.get("axes");
 357                 CV_Assert(axes.size() <= inpShape.size() - 2);
 358                 std::vector<int> kernel_size(inpShape.size() - 2, 1);
 359                 for (int i = 0; i < axes.size(); i++) {
 360                     int axis = axes.get<int>(i);
 361                     CV_Assert_N(axis >= 2 + i, axis < inpShape.size());
 362                     kernel_size[axis - 2] = inpShape[axis];
 363                 }
 364
 365                 layerParams.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size()));
 366             }
 367         }
 368         else if (layer_type == "Slice")
 369         {
 370             if (layerParams.has("steps")) {
 371                 DictValue steps = layerParams.get("steps");
 372                 for (int i = 0; i < steps.size(); ++i) {
 373                     if (steps.get<int>(i) != 1)
 374                         CV_Error(Error::StsNotImplemented,
 375                                  "Slice layer only supports steps = 1");
 376                 }
 377             }
 378
 379             int axis = 0;
 380             if (layerParams.has("axes")) {
 381                 DictValue axes = layerParams.get("axes");
 382                 for (int i = 1; i < axes.size(); ++i) {
 383                     CV_Assert(axes.get<int>(i - 1) == axes.get<int>(i) - 1);
 384                 }
 385                 axis = axes.get<int>(0);
 386             }
 387             layerParams.set("axis", axis);
 388
 389             DictValue starts = layerParams.get("starts");
 390             DictValue ends = layerParams.get("ends");
 391             CV_Assert(starts.size() == ends.size());
 392
 393             std::vector<int> begin;
 394             std::vector<int> end;
 395             if (axis > 0) {
 396                 begin.resize(axis, 0);
 397                 end.resize(axis, -1);
 398             }
 399
 400             for (int i = 0; i < starts.size(); ++i)
 401             {
 402                 begin.push_back(starts.get<int>(i));
 403                 int finish = ends.get<int>(i);
 404                 end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim
 405             }
 406             layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size()));
 407             layerParams.set("end", DictValue::arrayInt(&end[0], end.size()));
 408          }
 409         else if (layer_type == "Split")
 410         {
 411             if (layerParams.has("split"))
 412             {
 413                 DictValue splits = layerParams.get("split");
 414                 const int numSplits = splits.size();
 415                 CV_Assert(numSplits > 1);
 416
 417                 std::vector<int> slicePoints(numSplits - 1, splits.get<int>(0));
 418                 for (int i = 1; i < splits.size() - 1; ++i)
 419                 {
 420                     slicePoints[i] = slicePoints[i - 1] + splits.get<int>(i - 1);
 421                 }
 422                 layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size()));
 423             }
 424             else
 425             {
 426                 layerParams.set("num_split", node_proto.output_size());
 427             }
 428             layerParams.type = "Slice";
 429         }
 430         else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub")
 431         {
 432             bool isSub = layer_type == "Sub";
 433             CV_CheckEQ(node_proto.input_size(), 2, "");
 434             if (layer_id.find(node_proto.input(1)) == layer_id.end())
 435             {
 436                 Mat blob = getBlob(node_proto, constBlobs, 1);
 437                 blob = blob.reshape(1, 1);
 438                 if (blob.total() == 1) {
 439                     layerParams.type = "Power";
 440                     layerParams.set("shift", (isSub ? -1 : 1) * blob.at<float>(0));
 441                 }
 442                 else {
 443                     layerParams.type = "Scale";
 444                     layerParams.set("bias_term", true);
 445                     layerParams.blobs.push_back((isSub ? -1 : 1) * blob);
 446                 }
 447             }
 448             else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)])
 449             {
 450                 layerParams.type = "Eltwise";
 451                 if (isSub)
 452                 {
 453                     static float subCoeffs[] = {1.f, -1.f};
 454                     layerParams.set("coeff", DictValue::arrayReal<float*>(subCoeffs, 2));
 455                 }
 456             }
 457             else
 458             {
 459                 if (isSub)
 460                 {
 461                     LayerParams powerParams;
 462                     powerParams.name = layerParams.name + "/neg";
 463                     powerParams.type = "Power";
 464                     powerParams.set("scale", -1);
 465
 466                     //Create Power layer
 467                     int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams);
 468                     //Connect to input
 469                     layerId = layer_id.find(node_proto.input(1));
 470                     CV_Assert(layerId != layer_id.end());
 471                     dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
 472                     //Add shape
 473                     layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0)));
 474                     outShapes[powerParams.name] = outShapes[node_proto.input(1)];
 475
 476                     //Replace input to Power
 477                     node_proto.set_input(1, powerParams.name);
 478                 }
 479                 layerParams.type = "Scale";
 480                 layerParams.set("bias_term", true);
 481             }
 482         }
 483         else if (layer_type == "Max")
 484         {
 485             layerParams.type = "Eltwise";
 486             layerParams.set("operation", "max");
 487         }
 488         else if (layer_type == "Neg")
 489         {
 490             layerParams.type = "Power";
 491             layerParams.set("scale", -1);
 492         }
 493         else if (layer_type == "Constant")
 494         {
 495             CV_Assert(node_proto.input_size() == 0);
 496             CV_Assert(layerParams.blobs.size() == 1);
 497             constBlobs.insert(std::make_pair(layerParams.name, layerParams.blobs[0]));
 498             continue;
 499         }
 500         else if (layer_type == "ImageScaler")
 501         {
 502             const float scale = layerParams.has("scale") ? layerParams.get<float>("scale") : 1.0f;
 503             layerParams.erase("scale");
 504
 505             if (layerParams.has("bias"))
 506             {
 507                 layerParams.type = "Scale";
 508                 layerParams.blobs.push_back(
 509                     Mat(Size(1,  layerParams.get("bias").size()), CV_32FC1, scale));
 510
 511                 layerParams.set("bias_term", true);
 512                 Mat bias(1, layerParams.get("bias").size(), CV_32FC1);
 513                 for (int j = 0; j < bias.total(); j++) {
 514                     bias.at<float>(0, j) = layerParams.get("bias").getRealValue(j);
 515                 }
 516                 layerParams.blobs.push_back(bias);
 517                 layerParams.erase("bias");
 518             }
 519             else {
 520                 layerParams.set("scale", scale);
 521                 layerParams.type = "Power";
 522             }
 523         }
 524         else if (layer_type == "Clip")
 525         {
 526             layerParams.type = "ReLU6";
 527             replaceLayerParam(layerParams, "min", "min_value");
 528             replaceLayerParam(layerParams, "max", "max_value");
 529
 530         }
 531         else if (layer_type == "LeakyRelu")
 532         {
 533             layerParams.type = "ReLU";
 534             replaceLayerParam(layerParams, "alpha", "negative_slope");
 535         }
 536         else if (layer_type == "LRN")
 537         {
 538             replaceLayerParam(layerParams, "size", "local_size");
 539         }
 540         else if (layer_type == "InstanceNormalization")
 541         {
 542             if (node_proto.input_size() != 3)
 543                 CV_Error(Error::StsNotImplemented,
 544                          "Expected input, scale, bias");
 545
 546             layerParams.blobs.resize(4);
 547             layerParams.blobs[2] = getBlob(node_proto, constBlobs, 1);  // weightData
 548             layerParams.blobs[3] = getBlob(node_proto, constBlobs, 2);  // biasData
 549             layerParams.set("has_bias", true);
 550             layerParams.set("has_weight", true);
 551
 552             // Get number of channels in input
 553             int size = layerParams.blobs[2].total();
 554             layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean
 555             layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std
 556
 557             LayerParams mvnParams;
 558             mvnParams.name = layerParams.name + "/MVN";
 559             mvnParams.type = "MVN";
 560             mvnParams.set("eps", layerParams.get<float>("epsilon"));
 561             layerParams.erase("epsilon");
 562
 563             //Create MVN layer
 564             int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams);
 565             //Connect to input
 566             layerId = layer_id.find(node_proto.input(0));
 567             CV_Assert(layerId != layer_id.end());
 568             dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
 569             //Add shape
 570             layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0)));
 571             outShapes[mvnParams.name] = outShapes[node_proto.input(0)];
 572
 573             //Replace Batch Norm's input to MVN
 574             node_proto.set_input(0, mvnParams.name);
 575             layerParams.type = "BatchNorm";
 576         }
 577         else if (layer_type == "BatchNormalization")
 578         {
 579             if (node_proto.input_size() != 5)
 580                 CV_Error(Error::StsNotImplemented,
 581                          "Expected input, scale, bias, mean and var");
 582
 583             layerParams.type = "BatchNorm";
 584             replaceLayerParam(layerParams, "epsilon", "eps");
 585             replaceLayerParam(layerParams, "spatial", "use_global_stats");
 586
 587             Mat meanData = getBlob(node_proto, constBlobs, 3);
 588             Mat stdData =  getBlob(node_proto, constBlobs, 4);
 589
 590             layerParams.blobs.push_back(meanData);
 591             layerParams.blobs.push_back(stdData);
 592
 593             if (!node_proto.input(1).empty()) {
 594                 layerParams.set("has_weight", true);
 595                 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 1));  // weightData
 596             } else {
 597                 layerParams.set("has_weight", false);
 598             }
 599
 600             if (!node_proto.input(2).empty()) {
 601                 layerParams.set("has_bias", true);
 602                 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 2)); // biasData
 603             } else {
 604                 layerParams.set("has_bias", false);
 605             }
 606         }
 607         else if (layer_type == "Gemm")
 608         {
 609             CV_Assert(node_proto.input_size() >= 2);
 610             layerParams.type = "InnerProduct";
 611             Mat weights = getBlob(node_proto, constBlobs, 1);
 612             int ind_num_out = 0;
 613             if (layerParams.has("transB") && !layerParams.get<int>("transB")) {
 614                 transpose(weights, weights);
 615                 ind_num_out = 1;
 616             }
 617             layerParams.blobs.push_back(weights);
 618
 619             if (node_proto.input_size() == 3) {
 620                 Mat bias = getBlob(node_proto, constBlobs, 2);
 621                 layerParams.blobs.push_back(bias);
 622             }
 623
 624             layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]);
 625             layerParams.set("bias_term", node_proto.input_size() == 3);
 626         }
 627         else if (layer_type == "MatMul")
 628         {
 629             CV_Assert(node_proto.input_size() == 2);
 630             layerParams.type = "InnerProduct";
 631             Mat blob = getBlob(node_proto, constBlobs, 1);
 632             layerParams.blobs.push_back(blob.t());
 633             layerParams.set("bias_term", false);
 634             layerParams.set("num_output", layerParams.blobs[0].size[0]);
 635         }
 636         else if (layer_type == "Mul" || layer_type == "Div")
 637         {
 638             CV_Assert(node_proto.input_size() == 2);
 639
 640             bool isDiv = layer_type == "Div";
 641             int constId = -1;
 642             bool haveVariables = false;
 643             for (int i = 0; i < 2; ++i)
 644             {
 645                 if (constBlobs.find(node_proto.input(i)) != constBlobs.end())
 646                     constId = i;
 647                 else
 648                     haveVariables = true;
 649             }
 650             if (constId != -1 && haveVariables)
 651             {
 652                 Mat blob = getBlob(node_proto, constBlobs, constId);
 653                 blob = blob.reshape(1, 1);
 654                 if (blob.total() == 1) {
 655                     float coeff = isDiv ? 1.0 / blob.at<float>(0) : blob.at<float>(0);
 656                     layerParams.set("scale", coeff);
 657                     layerParams.type = "Power";
 658                 }
 659                 else {
 660                     if (isDiv)
 661                         divide(1.0, blob, blob);
 662                     layerParams.blobs.push_back(blob);
 663                     layerParams.type = "Scale";
 664                 }
 665             }
 666             else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)])
 667             {
 668                 layerParams.type = "Eltwise";
 669                 layerParams.set("operation", isDiv ? "div" : "prod");
 670             }
 671             else
 672             {
 673                 if (isDiv)
 674                 {
 675                     LayerParams powerParams;
 676                     powerParams.name = layerParams.name + "/inv";
 677                     powerParams.type = "Power";
 678                     powerParams.set("power", -1);
 679
 680                     //Create Power layer
 681                     int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams);
 682                     //Connect to input
 683                     layerId = layer_id.find(node_proto.input(1));
 684                     CV_Assert(layerId != layer_id.end());
 685                     dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
 686                     //Add shape
 687                     layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0)));
 688                     outShapes[powerParams.name] = outShapes[node_proto.input(1)];
 689
 690                     //Replace input to Power
 691                     node_proto.set_input(1, powerParams.name);
 692                 }
 693                 layerParams.type = "Scale";
 694             }
 695
 696             if (!haveVariables)
 697             {
 698                 Mat inp0 = getBlob(node_proto, constBlobs, 0);
 699                 Mat inp1 = getBlob(node_proto, constBlobs, 1);
 700                 if (inp0.size != inp1.size)
 701                     CV_Error(Error::StsNotImplemented, "Constant multiply with different shapes");
 702
 703                 Mat out;
 704                 if (isDiv)
 705                     divide(inp0, inp1, out);
 706                 else
 707                     multiply(inp0, inp1, out);
 708
 709                 out = out.reshape(1, inp0.dims, inp0.size);
 710                 out.dims = inp0.dims;  // to workaround dims == 1
 711                 constBlobs.insert(std::make_pair(layerParams.name, out));
 712                 continue;
 713             }
 714         }
 715         else if (layer_type == "Conv")
 716         {
 717             CV_Assert(node_proto.input_size() >= 2);
 718             layerParams.type = "Convolution";
 719             for (int j = 1; j < node_proto.input_size(); j++) {
 720                 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
 721             }
 722             layerParams.set("num_output", layerParams.blobs[0].size[0]);
 723             layerParams.set("bias_term", node_proto.input_size() == 3);
 724         }
 725         else if (layer_type == "ConvTranspose")
 726         {
 727             CV_Assert(node_proto.input_size() >= 2);
 728             layerParams.type = "Deconvolution";
 729             for (int j = 1; j < node_proto.input_size(); j++) {
 730                 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
 731             }
 732             layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get<int>("group", 1));
 733             layerParams.set("bias_term", node_proto.input_size() == 3);
 734
 735             if (!layerParams.has("kernel_size"))
 736                 CV_Error(Error::StsNotImplemented,
 737                          "Required attribute 'kernel_size' is not present.");
 738
 739             if (layerParams.has("output_shape"))
 740             {
 741                 const DictValue& outShape = layerParams.get("output_shape");
 742                 DictValue strides = layerParams.get("stride");
 743                 DictValue kernel = layerParams.get("kernel_size");
 744
 745                 String padMode;
 746                 std::vector<int> adjust_pads;
 747                 if (layerParams.has("pad_mode"))
 748                 {
 749                     padMode = toUpperCase(layerParams.get<String>("pad_mode"));
 750                     if (padMode != "SAME" && padMode != "VALID")
 751                         CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
 752
 753                     for (int i = 0; i < strides.size(); i++)
 754                     {
 755                         int sz = outShape.get<int>(2 + i);
 756                         int stride = strides.get<int>(i);
 757                         adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride :
 758                                                                  (sz - kernel.get<int>(i)) % stride);
 759                     }
 760                     layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size()));
 761                 }
 762             }
 763             else if (layerParams.has("output_padding"))
 764             {
 765                 replaceLayerParam(layerParams, "output_padding", "adj");
 766             }
 767         }
 768         else if (layer_type == "Transpose")
 769         {
 770             layerParams.type = "Permute";
 771             replaceLayerParam(layerParams, "perm", "order");
 772
 773             CV_Assert(node_proto.input_size() == 1);
 774             if (constBlobs.find(node_proto.input(0)) != constBlobs.end())
 775             {
 776                 std::vector<Mat> inputs(1, getBlob(node_proto, constBlobs, 0)), transposed;
 777                 runLayer(layerParams, inputs, transposed);
 778                 CV_Assert(transposed.size() == 1);
 779                 constBlobs.insert(std::make_pair(layerParams.name, transposed[0]));
 780                 continue;
 781             }
 782         }
 783         else if (layer_type == "ReduceL2")
 784         {
 785             CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes"));
 786             CV_Assert(graph_proto.node_size() > li + 1 && graph_proto.node(li + 1).op_type() == "Div");
 787             ++li;
 788             node_proto = graph_proto.node(li);
 789             layerParams.name = node_proto.output(0);
 790             layerParams.type = "Normalize";
 791
 792             DictValue axes_dict = layerParams.get("axes");
 793             if (axes_dict.size() != 1)
 794                 CV_Error(Error::StsNotImplemented, "Multidimensional reduceL2");
 795             int axis = axes_dict.getIntValue(0);
 796             layerParams.set("axis",axis);
 797             layerParams.set("end_axis", axis);
 798         }
 799         else if (layer_type == "Squeeze")
 800         {
 801             CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes"));
 802             DictValue axes_dict = layerParams.get("axes");
 803             if (axes_dict.size() != 1)
 804                 CV_Error(Error::StsNotImplemented, "Multidimensional squeeze");
 805
 806             int axis = axes_dict.getIntValue(0);
 807             layerParams.set("axis", axis - 1);
 808             layerParams.set("end_axis", axis);
 809             layerParams.type = "Flatten";
 810         }
 811         else if (layer_type == "Unsqueeze")
 812         {
 813             CV_Assert(node_proto.input_size() == 1);
 814             DictValue axes = layerParams.get("axes");
 815             if (constBlobs.find(node_proto.input(0)) != constBlobs.end())
 816             {
 817                 // Constant input.
 818                 Mat input = getBlob(node_proto, constBlobs, 0);
 819
 820                 std::vector<int> dims;
 821                 for (int j = 0; j < input.dims; j++) {
 822                     dims.push_back(input.size[j]);
 823                 }
 824                 CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size());
 825                 for (int j = 0; j < axes.size(); j++) {
 826                     dims.insert(dims.begin() + axes.getIntValue(j), 1);
 827                 }
 828
 829                 Mat out = input.reshape(0, dims);
 830                 constBlobs.insert(std::make_pair(layerParams.name, out));
 831                 continue;
 832             }
 833
 834             // Variable input.
 835             if (axes.size() != 1)
 836                 CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze");
 837
 838             MatShape inpShape = outShapes[node_proto.input(0)];
 839             int axis = axes.getIntValue(0);
 840             CV_Assert(0 <= axis && axis <= inpShape.size());
 841             std::vector<int> outShape = inpShape;
 842             outShape.insert(outShape.begin() + axis, 1);
 843             layerParams.type = "Reshape";
 844             layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size()));
 845         }
 846         else if (layer_type == "Reshape")
 847         {
 848             CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape"));
 849
 850             if (node_proto.input_size() == 2) {
 851                 Mat blob = getBlob(node_proto, constBlobs, 1);
 852                 CV_Assert(blob.type() == CV_32SC1);
 853
 854                 layerParams.set("dim", DictValue::arrayInt<int*>(
 855                             blob.ptr<int>(), blob.total() ));
 856
 857                 if (layer_id.find(node_proto.input(0)) == layer_id.end()) {
 858                     std::vector<Mat> inputs(1, getBlob(node_proto, constBlobs, 0)), outputs;
 859                     runLayer(layerParams, inputs, outputs);
 860                     constBlobs.insert(std::make_pair(layerParams.name, outputs[0]));
 861                     continue;
 862                 }
 863             }
 864             else {
 865                 DictValue shape = layerParams.get("shape");
 866                 std::vector<int> dim;
 867                 for (int j = 0; j < shape.size(); j++) {
 868                     dim.push_back(shape.getIntValue(j));
 869                 }
 870
 871                 if (layer_id.find(node_proto.input(0)) == layer_id.end()) {
 872                     Mat input = getBlob(node_proto, constBlobs, 0);
 873                     Mat out = input.reshape(0, dim);
 874                     constBlobs.insert(std::make_pair(layerParams.name, out));
 875                     continue;
 876                 }
 877                 replaceLayerParam(layerParams, "shape", "dim");
 878             }
 879         }
 880         else if (layer_type == "Pad")
 881         {
 882             layerParams.type = "Padding";
 883         }
 884         else if (layer_type == "Shape")
 885         {
 886             CV_Assert(node_proto.input_size() == 1);
 887             shapeIt = outShapes.find(node_proto.input(0));
 888             CV_Assert(shapeIt != outShapes.end());
 889             MatShape inpShape = shapeIt->second;
 890
 891             Mat shapeMat(inpShape.size(), 1, CV_32S);
 892             for (int j = 0; j < inpShape.size(); ++j)
 893                 shapeMat.at<int>(j) = inpShape[j];
 894             shapeMat.dims = 1;
 895
 896             constBlobs.insert(std::make_pair(layerParams.name, shapeMat));
 897             continue;
 898         }
 899         else if (layer_type == "Gather")
 900         {
 901             CV_Assert(node_proto.input_size() == 2);
 902             CV_Assert(layerParams.has("axis"));
 903             Mat input = getBlob(node_proto, constBlobs, 0);
 904             Mat indexMat = getBlob(node_proto, constBlobs, 1);
 905             CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1);
 906             int index = indexMat.at<int>(0);
 907             int axis = layerParams.get<int>("axis");
 908
 909             std::vector<cv::Range> ranges(input.dims, Range::all());
 910             ranges[axis] = Range(index, index + 1);
 911
 912             Mat out = input(ranges);
 913             constBlobs.insert(std::make_pair(layerParams.name, out));
 914             continue;
 915         }
 916         else if (layer_type == "Concat")
 917         {
 918             bool hasVariableInps = false;
 919             for (int i = 0; i < node_proto.input_size(); ++i)
 920             {
 921                 if (layer_id.find(node_proto.input(i)) != layer_id.end())
 922                 {
 923                     hasVariableInps = true;
 924                     break;
 925                 }
 926             }
 927
 928             if (!hasVariableInps)
 929             {
 930                 std::vector<Mat> inputs(node_proto.input_size()), concatenated;
 931                 for (size_t i = 0; i < inputs.size(); ++i)
 932                 {
 933                     inputs[i] = getBlob(node_proto, constBlobs, i);
 934                 }
 935                 runLayer(layerParams, inputs, concatenated);
 936
 937                 CV_Assert(concatenated.size() == 1);
 938                 constBlobs.insert(std::make_pair(layerParams.name, concatenated[0]));
 939                 continue;
 940             }
 941         }
 942         else if (layer_type == "Upsample")
 943         {
 944             layerParams.type = "Resize";
 945             if (layerParams.has("scales"))
 946             {
 947                 // Pytorch layer
 948                 DictValue scales = layerParams.get("scales");
 949                 CV_Assert(scales.size() == 4);
 950                 layerParams.set("zoom_factor_y", scales.getIntValue(2));
 951                 layerParams.set("zoom_factor_x", scales.getIntValue(3));
 952             }
 953             else
 954             {
 955                 // Caffe2 layer
 956                 replaceLayerParam(layerParams, "height_scale", "zoom_factor_y");
 957                 replaceLayerParam(layerParams, "width_scale", "zoom_factor_x");
 958             }
 959             replaceLayerParam(layerParams, "mode", "interpolation");
 960
 961             if (layerParams.get<String>("interpolation") == "linear" && framework_name == "pytorch") {
 962                 layerParams.type = "Resize";
 963                 Mat scales = getBlob(node_proto, constBlobs, 1);
 964                 CV_Assert(scales.total() == 4);
 965                 layerParams.set("interpolation", "opencv_linear");
 966                 layerParams.set("zoom_factor_y", scales.at<float>(2));
 967                 layerParams.set("zoom_factor_x", scales.at<float>(3));
 968             }
 969         }
 970         else if (layer_type == "LogSoftmax")
 971         {
 972             layerParams.type = "Softmax";
 973             layerParams.set("log_softmax", true);
 974         }
 975         else
 976         {
 977             for (int j = 0; j < node_proto.input_size(); j++) {
 978                 if (layer_id.find(node_proto.input(j)) == layer_id.end())
 979                     layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
 980             }
 981         }
 982
 983         int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams);
 984         for (int i = 0; i < node_proto.output_size(); ++i)
 985         {
 986             layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i)));
 987         }
 988
 989         std::vector<MatShape> layerInpShapes, layerOutShapes, layerInternalShapes;
 990         for (int j = 0; j < node_proto.input_size(); j++) {
 991             layerId = layer_id.find(node_proto.input(j));
 992             if (layerId != layer_id.end()) {
 993                 dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, j);
 994                 // Collect input shapes.
 995                 shapeIt = outShapes.find(node_proto.input(j));
 996                 CV_Assert(shapeIt != outShapes.end());
 997                 layerInpShapes.push_back(shapeIt->second);
 998             }
 999         }
1000
1001         // Compute shape of output blob for this layer.
1002         Ptr<Layer> layer = dstNet.getLayer(id);
1003         layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes);
1004         for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i)
1005         {
1006             outShapes[node_proto.output(i)] = layerOutShapes[i];
1007         }
1008     }
1009 }
1010
1011 Net readNetFromONNX(const String& onnxFile)
1012 {
1013     ONNXImporter onnxImporter(onnxFile.c_str());
1014     Net net;
1015     onnxImporter.populateNet(net);
1016     return net;
1017 }
1018
1019 Net readNetFromONNX(const char* buffer, size_t sizeBuffer)
1020 {
1021     ONNXImporter onnxImporter(buffer, sizeBuffer);
1022     Net net;
1023     onnxImporter.populateNet(net);
1024     return net;
1025 }
1026
1027 Net readNetFromONNX(const std::vector<uchar>& buffer)
1028 {
1029     return readNetFromONNX(reinterpret_cast<const char*>(buffer.data()), buffer.size());
1030 }
1031
1032 Mat readTensorFromONNX(const String& path)
1033 {
1034     opencv_onnx::TensorProto tensor_proto = opencv_onnx::TensorProto();
1035     std::fstream input(path.c_str(), std::ios::in | std::ios::binary);
1036     if (!tensor_proto.ParseFromIstream(&input)) {
1037         CV_Error(Error::StsUnsupportedFormat, "Failed to parse data");
1038     }
1039     Mat mat = getMatFromTensor(tensor_proto);
1040     releaseONNXTensor(tensor_proto);
1041     return mat;
1042 }
1043
1044 CV__DNN_EXPERIMENTAL_NS_END
1045 }} // namespace
1046
1047 #endif