1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2018, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
8 #include "../precomp.hpp"
9 #include <opencv2/dnn/shape_utils.hpp>
20 #if defined(__GNUC__) && __GNUC__ >= 5
21 #pragma GCC diagnostic push
22 #pragma GCC diagnostic ignored "-Wsuggest-override"
24 #include "opencv-onnx.pb.h"
25 #if defined(__GNUC__) && __GNUC__ >= 5
26 #pragma GCC diagnostic pop
31 CV__DNN_INLINE_NS_BEGIN
36 opencv_onnx::ModelProto model_proto;
40 LayerInfo(int _layerId, int _outputId) : layerId(_layerId), outputId(_outputId) {}
43 std::map<std::string, Mat> getGraphTensors(
44 const opencv_onnx::GraphProto& graph_proto);
45 Mat getBlob(const opencv_onnx::NodeProto& node_proto, const std::map<std::string, Mat>& constBlobs, int index);
47 LayerParams getLayerParams(const opencv_onnx::NodeProto& node_proto);
48 bool isCeilMode(const LayerParams& layerParams);
52 ONNXImporter(const char *onnxFile)
54 std::fstream input(onnxFile, std::ios::in | std::ios::binary);
56 if (!model_proto.ParseFromIstream(&input))
57 CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model");
60 ONNXImporter(const char* buffer, size_t sizeBuffer)
62 struct _Buf : public std::streambuf
64 _Buf(const char* buffer, size_t sizeBuffer)
66 char* p = const_cast<char*>(buffer);
67 setg(p, p, p + sizeBuffer);
71 _Buf buf(buffer, sizeBuffer);
72 std::istream input(&buf);
74 if (!model_proto.ParseFromIstream(&input))
75 CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array.");
78 void populateNet(Net dstNet);
81 inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey)
83 if (layerParams.has(oldKey)) {
84 layerParams.set(newKey, layerParams.get(oldKey));
85 layerParams.erase(oldKey);
89 void releaseONNXTensor(opencv_onnx::TensorProto& tensor_proto)
91 if (!tensor_proto.raw_data().empty()) {
92 delete tensor_proto.release_raw_data();
96 template<typename T1, typename T2>
97 void convertInt64ToInt32(const T1& src, T2& dst, int size)
99 for (int i = 0; i < size; i++) {
100 if (src[i] < std::numeric_limits<int32_t>::min() || src[i] > std::numeric_limits<int32_t>::max()) {
101 CV_Error(Error::StsOutOfRange, "Input is out of OpenCV 32S range");
103 dst[i] = saturate_cast<int32_t>(src[i]);
107 Mat getMatFromTensor(opencv_onnx::TensorProto& tensor_proto)
109 CV_Assert(!tensor_proto.raw_data().empty() || !tensor_proto.float_data().empty()
110 || !tensor_proto.double_data().empty() || !tensor_proto.int64_data().empty());
112 opencv_onnx::TensorProto_DataType datatype = tensor_proto.data_type();
114 std::vector<int> sizes;
115 for (int i = 0; i < tensor_proto.dims_size(); i++) {
116 sizes.push_back(tensor_proto.dims(i));
120 if (datatype == opencv_onnx::TensorProto_DataType_FLOAT) {
122 if (!tensor_proto.float_data().empty()) {
123 const ::google::protobuf::RepeatedField<float> field = tensor_proto.float_data();
124 Mat(sizes, CV_32FC1, (void*)field.data()).copyTo(blob);
127 char* val = const_cast<char*>(tensor_proto.raw_data().c_str());
128 Mat(sizes, CV_32FC1, val).copyTo(blob);
131 else if (datatype == opencv_onnx::TensorProto_DataType_DOUBLE)
133 const ::google::protobuf::RepeatedField<double> field = tensor_proto.double_data();
134 CV_Assert(!field.empty());
135 Mat(sizes, CV_64FC1, (void*)field.data()).convertTo(blob, CV_32FC1);
137 else if (datatype == opencv_onnx::TensorProto_DataType_INT64)
139 blob.create(sizes, CV_32SC1);
140 int32_t* dst = reinterpret_cast<int32_t*>(blob.data);
142 if (!tensor_proto.int64_data().empty()) {
143 ::google::protobuf::RepeatedField< ::google::protobuf::int64> src = tensor_proto.int64_data();
144 convertInt64ToInt32(src, dst, blob.total());
148 char* val = const_cast<char*>(tensor_proto.raw_data().c_str());
149 int64_t* src = reinterpret_cast<int64_t*>(val);
150 convertInt64ToInt32(src, dst, blob.total());
154 CV_Error(Error::StsUnsupportedFormat, "Unsupported data type: " +
155 opencv_onnx::TensorProto_DataType_Name(datatype));
156 if (tensor_proto.dims_size() == 0)
157 blob.dims = 1; // To force 1-dimensional cv::Mat for scalars.
161 void runLayer(LayerParams& params, const std::vector<Mat>& inputs,
162 std::vector<Mat>& outputs)
164 Ptr<Layer> layer = LayerFactory::createLayerInstance(params.type, params);
165 CV_Assert((bool)layer);
167 std::vector<MatShape> inpShapes(inputs.size());
169 for (size_t i = 0; i < inputs.size(); ++i)
171 inpShapes[i] = shape(inputs[i]);
172 if (i > 0 && ddepth != inputs[i].depth())
173 CV_Error(Error::StsNotImplemented, "Mixed input data types.");
174 ddepth = inputs[i].depth();
177 std::vector<MatShape> outShapes, internalShapes;
178 layer->getMemoryShapes(inpShapes, 0, outShapes, internalShapes);
180 std::vector<Mat> internals(internalShapes.size());
181 outputs.resize(outShapes.size());
182 for (size_t i = 0; i < outShapes.size(); ++i)
183 outputs[i].create(outShapes[i], ddepth);
184 for (size_t i = 0; i < internalShapes.size(); ++i)
185 internals[i].create(internalShapes[i], ddepth);
187 layer->finalize(inputs, outputs);
188 layer->forward(inputs, outputs, internals);
191 std::map<std::string, Mat> ONNXImporter::getGraphTensors(
192 const opencv_onnx::GraphProto& graph_proto)
194 opencv_onnx::TensorProto tensor_proto;
195 std::map<std::string, Mat> layers_weights;
197 for (int i = 0; i < graph_proto.initializer_size(); i++)
199 tensor_proto = graph_proto.initializer(i);
200 Mat mat = getMatFromTensor(tensor_proto);
201 releaseONNXTensor(tensor_proto);
202 layers_weights.insert(std::make_pair(tensor_proto.name(), mat));
204 return layers_weights;
207 static DictValue parse(const ::google::protobuf::RepeatedField< ::google::protobuf::int64>& src) {
208 std::vector<int32_t> dst(src.size());
209 convertInt64ToInt32(src, dst, src.size());
210 return DictValue::arrayInt(&dst[0], src.size());
213 LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_proto)
216 for(int i = 0; i < node_proto.attribute_size(); i++)
218 opencv_onnx::AttributeProto attribute_proto = node_proto.attribute(i);
219 std::string attribute_name = attribute_proto.name();
221 if(attribute_name == "kernel_shape")
223 CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
224 lp.set("kernel_size", parse(attribute_proto.ints()));
226 else if(attribute_name == "strides")
228 CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
229 lp.set("stride", parse(attribute_proto.ints()));
231 else if(attribute_name == "pads")
233 if (node_proto.op_type() == "Pad")
236 // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN.
237 // We need to shuffle it to begin0, end0, begin1, end1, ...
238 CV_Assert(attribute_proto.ints_size() % 2 == 0);
239 const int dims = attribute_proto.ints_size() / 2;
240 std::vector<int32_t> paddings;
241 paddings.reserve(attribute_proto.ints_size());
242 for (int i = 0; i < dims; ++i)
244 paddings.push_back(attribute_proto.ints(i));
245 paddings.push_back(attribute_proto.ints(dims + i));
247 lp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size()));
251 // Convolution or pooling.
252 CV_Assert(attribute_proto.ints_size() == 4 || attribute_proto.ints_size() == 6);
253 lp.set("pad", parse(attribute_proto.ints()));
256 else if(attribute_name == "auto_pad")
258 if (attribute_proto.s() == "SAME_UPPER" || attribute_proto.s() == "SAME_LOWER") {
259 lp.set("pad_mode", "SAME");
261 else if (attribute_proto.s() == "VALID") {
262 lp.set("pad_mode", "VALID");
265 else if(attribute_name == "dilations")
267 CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
268 lp.set("dilation", parse(attribute_proto.ints()));
270 else if (attribute_proto.has_i())
272 ::google::protobuf::int64 src = attribute_proto.i();
273 if (src < std::numeric_limits<int32_t>::min() || src > std::numeric_limits<int32_t>::max())
274 CV_Error(Error::StsOutOfRange, "Input is out of OpenCV 32S range");
276 lp.set(attribute_name, saturate_cast<int32_t>(src));
278 else if (attribute_proto.has_f())
280 lp.set(attribute_name, attribute_proto.f());
282 else if (attribute_proto.has_s())
284 lp.set(attribute_name, attribute_proto.s());
286 else if (attribute_proto.floats_size() > 0)
288 lp.set(attribute_name, DictValue::arrayReal(
289 attribute_proto.floats().data(), attribute_proto.floats_size()));
291 else if (attribute_proto.ints_size() > 0)
293 lp.set(attribute_proto.name(), parse(attribute_proto.ints()));
295 else if (attribute_proto.has_t())
297 opencv_onnx::TensorProto tensor = attribute_proto.t();
298 Mat blob = getMatFromTensor(tensor);
299 lp.blobs.push_back(blob);
301 else if (attribute_proto.has_g() || attribute_proto.strings_size() > 0 ||
302 attribute_proto.tensors_size() > 0 || attribute_proto.graphs_size() > 0)
304 CV_Error(Error::StsNotImplemented, "Unexpected attribute type");
307 CV_Error(Error::StsNotImplemented, "Unsupported attribute type");
312 Mat ONNXImporter::getBlob(const opencv_onnx::NodeProto& node_proto,
313 const std::map<std::string, Mat>& constBlobs, int index)
315 CV_Assert(index < node_proto.input_size());
316 std::map<std::string, Mat>::const_iterator constBlob;
317 constBlob = constBlobs.find(node_proto.input(index));
318 if (constBlob == constBlobs.end()) {
319 CV_Error(Error::StsObjectNotFound,
320 "Blob " + node_proto.input(index) + " not found in const blobs");
322 return constBlob->second;
325 void ONNXImporter::populateNet(Net dstNet)
327 CV_Assert(model_proto.has_graph());
328 opencv_onnx::GraphProto graph_proto = model_proto.graph();
329 std::map<std::string, Mat> constBlobs = getGraphTensors(graph_proto);
330 // List of internal blobs shapes.
331 std::map<std::string, MatShape> outShapes;
332 // Add all the inputs shapes. It includes as constant blobs as network's inputs shapes.
333 for (int i = 0; i < graph_proto.input_size(); ++i)
335 opencv_onnx::ValueInfoProto valueInfoProto = graph_proto.input(i);
336 CV_Assert(valueInfoProto.has_type());
337 opencv_onnx::TypeProto typeProto = valueInfoProto.type();
338 CV_Assert(typeProto.has_tensor_type());
339 opencv_onnx::TypeProto::Tensor tensor = typeProto.tensor_type();
340 CV_Assert(tensor.has_shape());
341 opencv_onnx::TensorShapeProto tensorShape = tensor.shape();
343 MatShape inpShape(tensorShape.dim_size());
344 for (int j = 0; j < inpShape.size(); ++j)
346 inpShape[j] = tensorShape.dim(j).dim_value();
348 outShapes[valueInfoProto.name()] = inpShape;
351 std::string framework_name;
352 if (model_proto.has_producer_name()) {
353 framework_name = model_proto.producer_name();
356 // create map with network inputs (without const blobs)
357 std::map<std::string, LayerInfo> layer_id;
358 std::map<std::string, LayerInfo>::iterator layerId;
359 std::map<std::string, MatShape>::iterator shapeIt;
360 // fill map: push layer name, layer id and output id
361 std::vector<String> netInputs;
362 for (int j = 0; j < graph_proto.input_size(); j++)
364 const std::string& name = graph_proto.input(j).name();
365 if (constBlobs.find(name) == constBlobs.end()) {
366 netInputs.push_back(name);
367 layer_id.insert(std::make_pair(name, LayerInfo(0, netInputs.size() - 1)));
370 dstNet.setInputsNames(netInputs);
372 int layersSize = graph_proto.node_size();
373 LayerParams layerParams;
374 opencv_onnx::NodeProto node_proto;
376 for(int li = 0; li < layersSize; li++)
378 node_proto = graph_proto.node(li);
379 layerParams = getLayerParams(node_proto);
380 CV_Assert(node_proto.output_size() >= 1);
381 layerParams.name = node_proto.output(0);
383 std::string layer_type = node_proto.op_type();
384 layerParams.type = layer_type;
387 if (layer_type == "MaxPool")
389 layerParams.type = "Pooling";
390 layerParams.set("pool", "MAX");
391 layerParams.set("ceil_mode", layerParams.has("pad_mode"));
393 else if (layer_type == "AveragePool")
395 layerParams.type = "Pooling";
396 layerParams.set("pool", "AVE");
397 layerParams.set("ceil_mode", layerParams.has("pad_mode"));
398 layerParams.set("ave_pool_padded_area", framework_name == "pytorch");
400 else if (layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool" || layer_type == "ReduceMean")
402 CV_Assert(node_proto.input_size() == 1);
403 layerParams.type = "Pooling";
404 layerParams.set("pool", layer_type == "GlobalMaxPool"? "MAX" : "AVE");
405 layerParams.set("global_pooling", layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool");
407 if (layer_type == "ReduceMean")
409 if (layerParams.get<int>("keepdims") == 0 || !layerParams.has("axes"))
410 CV_Error(Error::StsNotImplemented, "Unsupported mode of ReduceMean operation.");
412 MatShape inpShape = outShapes[node_proto.input(0)];
413 if (inpShape.size() != 4 && inpShape.size() != 5)
414 CV_Error(Error::StsNotImplemented, "Unsupported input shape of reduce_mean operation.");
416 DictValue axes = layerParams.get("axes");
417 CV_Assert(axes.size() <= inpShape.size() - 2);
418 std::vector<int> kernel_size(inpShape.size() - 2, 1);
419 for (int i = 0; i < axes.size(); i++) {
420 int axis = axes.get<int>(i);
421 CV_Assert_N(axis >= 2 + i, axis < inpShape.size());
422 kernel_size[axis - 2] = inpShape[axis];
425 layerParams.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size()));
428 else if (layer_type == "Slice")
430 if (layerParams.has("steps")) {
431 DictValue steps = layerParams.get("steps");
432 for (int i = 0; i < steps.size(); ++i) {
433 if (steps.get<int>(i) != 1)
434 CV_Error(Error::StsNotImplemented,
435 "Slice layer only supports steps = 1");
440 if (layerParams.has("axes")) {
441 DictValue axes = layerParams.get("axes");
442 for (int i = 1; i < axes.size(); ++i) {
443 CV_Assert(axes.get<int>(i - 1) == axes.get<int>(i) - 1);
445 axis = axes.get<int>(0);
447 layerParams.set("axis", axis);
449 DictValue starts = layerParams.get("starts");
450 DictValue ends = layerParams.get("ends");
451 CV_Assert(starts.size() == ends.size());
453 std::vector<int> begin;
454 std::vector<int> end;
456 begin.resize(axis, 0);
457 end.resize(axis, -1);
460 for (int i = 0; i < starts.size(); ++i)
462 begin.push_back(starts.get<int>(i));
463 int finish = ends.get<int>(i);
464 end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim
466 layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size()));
467 layerParams.set("end", DictValue::arrayInt(&end[0], end.size()));
469 else if (layer_type == "Split")
471 DictValue splits = layerParams.get("split");
472 const int numSplits = splits.size();
473 CV_Assert(numSplits > 1);
475 std::vector<int> slicePoints(numSplits - 1, splits.get<int>(0));
476 for (int i = 1; i < splits.size() - 1; ++i)
478 slicePoints[i] = slicePoints[i - 1] + splits.get<int>(i - 1);
480 layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size()));
481 layerParams.type = "Slice";
483 else if (layer_type == "Add" || layer_type == "Sum")
485 if (layer_id.find(node_proto.input(1)) == layer_id.end())
487 Mat blob = getBlob(node_proto, constBlobs, 1);
488 blob = blob.reshape(1, 1);
489 if (blob.total() == 1) {
490 layerParams.type = "Power";
491 layerParams.set("shift", blob.at<float>(0));
494 layerParams.type = "Scale";
495 layerParams.set("bias_term", true);
496 layerParams.blobs.push_back(blob);
500 layerParams.type = "Eltwise";
503 else if (layer_type == "Max")
505 layerParams.type = "Eltwise";
506 layerParams.set("operation", "max");
508 else if (layer_type == "Sub")
510 Mat blob = getBlob(node_proto, constBlobs, 1);
511 if (blob.total() == 1) {
512 layerParams.type = "Power";
513 layerParams.set("shift", -blob.at<float>(0));
516 layerParams.type = "Scale";
517 layerParams.set("has_bias", true);
518 layerParams.blobs.push_back(-1.0f * blob.reshape(1, 1));
521 else if (layer_type == "Div")
523 if (constBlobs.find(node_proto.input(1)) == constBlobs.end())
525 layerParams.type = "Eltwise";
526 layerParams.set("operation", "div");
530 Mat blob = getBlob(node_proto, constBlobs, 1);
531 CV_Assert_N(blob.type() == CV_32F, blob.total());
532 if (blob.total() == 1)
534 layerParams.set("scale", 1.0f / blob.at<float>(0));
535 layerParams.type = "Power";
539 layerParams.type = "Scale";
540 divide(1.0, blob, blob);
541 layerParams.blobs.push_back(blob);
542 layerParams.set("bias_term", false);
546 else if (layer_type == "Neg")
548 layerParams.type = "Power";
549 layerParams.set("scale", -1);
551 else if (layer_type == "Constant")
553 CV_Assert(node_proto.input_size() == 0);
554 CV_Assert(layerParams.blobs.size() == 1);
555 constBlobs.insert(std::make_pair(layerParams.name, layerParams.blobs[0]));
558 else if (layer_type == "ImageScaler")
560 const float scale = layerParams.has("scale") ? layerParams.get<float>("scale") : 1.0f;
561 layerParams.erase("scale");
563 if (layerParams.has("bias"))
565 layerParams.type = "Scale";
566 layerParams.blobs.push_back(
567 Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale));
569 layerParams.set("bias_term", true);
570 Mat bias(1, layerParams.get("bias").size(), CV_32FC1);
571 for (int j = 0; j < bias.total(); j++) {
572 bias.at<float>(0, j) = layerParams.get("bias").getRealValue(j);
574 layerParams.blobs.push_back(bias);
575 layerParams.erase("bias");
578 layerParams.set("scale", scale);
579 layerParams.type = "Power";
582 else if (layer_type == "Clip")
584 layerParams.type = "ReLU6";
585 replaceLayerParam(layerParams, "min", "min_value");
586 replaceLayerParam(layerParams, "max", "max_value");
589 else if (layer_type == "LeakyRelu")
591 layerParams.type = "ReLU";
592 replaceLayerParam(layerParams, "alpha", "negative_slope");
594 else if (layer_type == "LRN")
596 replaceLayerParam(layerParams, "size", "local_size");
598 else if (layer_type == "InstanceNormalization")
600 if (node_proto.input_size() != 3)
601 CV_Error(Error::StsNotImplemented,
602 "Expected input, scale, bias");
604 layerParams.blobs.resize(4);
605 layerParams.blobs[2] = getBlob(node_proto, constBlobs, 1); // weightData
606 layerParams.blobs[3] = getBlob(node_proto, constBlobs, 2); // biasData
607 layerParams.set("has_bias", true);
608 layerParams.set("has_weight", true);
610 // Get number of channels in input
611 int size = layerParams.blobs[2].total();
612 layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean
613 layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std
615 LayerParams mvnParams;
616 mvnParams.name = layerParams.name + "/MVN";
617 mvnParams.type = "MVN";
618 mvnParams.set("eps", layerParams.get<float>("epsilon"));
619 layerParams.erase("epsilon");
622 int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams);
624 layerId = layer_id.find(node_proto.input(0));
625 CV_Assert(layerId != layer_id.end());
626 dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
628 layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0)));
629 outShapes[mvnParams.name] = outShapes[node_proto.input(0)];
631 //Replace Batch Norm's input to MVN
632 node_proto.set_input(0, mvnParams.name);
633 layerParams.type = "BatchNorm";
635 else if (layer_type == "BatchNormalization")
637 if (node_proto.input_size() != 5)
638 CV_Error(Error::StsNotImplemented,
639 "Expected input, scale, bias, mean and var");
641 layerParams.type = "BatchNorm";
642 replaceLayerParam(layerParams, "epsilon", "eps");
643 replaceLayerParam(layerParams, "spatial", "use_global_stats");
645 Mat meanData = getBlob(node_proto, constBlobs, 3);
646 Mat stdData = getBlob(node_proto, constBlobs, 4);
648 layerParams.blobs.push_back(meanData);
649 layerParams.blobs.push_back(stdData);
651 if (!node_proto.input(1).empty()) {
652 layerParams.set("has_weight", true);
653 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 1)); // weightData
655 layerParams.set("has_weight", false);
658 if (!node_proto.input(2).empty()) {
659 layerParams.set("has_bias", true);
660 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 2)); // biasData
662 layerParams.set("has_bias", false);
665 else if (layer_type == "Gemm")
667 CV_Assert(node_proto.input_size() >= 2);
668 layerParams.type = "InnerProduct";
669 Mat weights = getBlob(node_proto, constBlobs, 1);
671 if (layerParams.has("transB") && !layerParams.get<int>("transB")) {
672 transpose(weights, weights);
675 layerParams.blobs.push_back(weights);
677 if (node_proto.input_size() == 3) {
678 Mat bias = getBlob(node_proto, constBlobs, 2);
679 layerParams.blobs.push_back(bias);
682 layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]);
683 layerParams.set("bias_term", node_proto.input_size() == 3);
685 else if (layer_type == "MatMul")
687 CV_Assert(node_proto.input_size() == 2);
688 layerParams.type = "InnerProduct";
689 Mat blob = getBlob(node_proto, constBlobs, 1);
690 layerParams.blobs.push_back(blob.t());
691 layerParams.set("bias_term", false);
692 layerParams.set("num_output", layerParams.blobs[0].size[0]);
694 else if (layer_type == "Mul")
696 CV_Assert(node_proto.input_size() == 2);
697 if (layer_id.find(node_proto.input(1)) == layer_id.end()) {
698 Mat blob = getBlob(node_proto, constBlobs, 1);
699 blob = blob.reshape(1, 1);
700 if (blob.total() == 1) {
701 layerParams.set("scale", blob.at<float>(0));
702 layerParams.type = "Power";
705 layerParams.blobs.push_back(blob);
706 layerParams.type = "Scale";
710 layerParams.type = "Eltwise";
711 layerParams.set("operation", "prod");
714 else if (layer_type == "Conv")
716 CV_Assert(node_proto.input_size() >= 2);
717 layerParams.type = "Convolution";
718 for (int j = 1; j < node_proto.input_size(); j++) {
719 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
721 layerParams.set("num_output", layerParams.blobs[0].size[0]);
722 layerParams.set("bias_term", node_proto.input_size() == 3);
724 else if (layer_type == "ConvTranspose")
726 CV_Assert(node_proto.input_size() >= 2);
727 layerParams.type = "Deconvolution";
728 for (int j = 1; j < node_proto.input_size(); j++) {
729 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
731 layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get<int>("group", 1));
732 layerParams.set("bias_term", node_proto.input_size() == 3);
734 if (!layerParams.has("kernel_size"))
735 CV_Error(Error::StsNotImplemented,
736 "Required attribute 'kernel_size' is not present.");
738 if (layerParams.has("output_shape"))
740 const DictValue& outShape = layerParams.get("output_shape");
741 DictValue strides = layerParams.get("stride");
742 DictValue kernel = layerParams.get("kernel_size");
745 std::vector<int> adjust_pads;
746 if (layerParams.has("pad_mode"))
748 padMode = toUpperCase(layerParams.get<String>("pad_mode"));
749 if (padMode != "SAME" && padMode != "VALID")
750 CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
752 for (int i = 0; i < strides.size(); i++)
754 int sz = outShape.get<int>(2 + i);
755 int stride = strides.get<int>(i);
756 adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride :
757 (sz - kernel.get<int>(i)) % stride);
759 layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size()));
762 else if (layerParams.has("output_padding"))
764 replaceLayerParam(layerParams, "output_padding", "adj");
767 else if (layer_type == "Transpose")
769 layerParams.type = "Permute";
770 replaceLayerParam(layerParams, "perm", "order");
772 CV_Assert(node_proto.input_size() == 1);
773 if (constBlobs.find(node_proto.input(0)) != constBlobs.end())
775 std::vector<Mat> inputs(1, getBlob(node_proto, constBlobs, 0)), transposed;
776 runLayer(layerParams, inputs, transposed);
777 CV_Assert(transposed.size() == 1);
778 constBlobs.insert(std::make_pair(layerParams.name, transposed[0]));
782 else if (layer_type == "ReduceL2")
784 CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes"));
785 CV_Assert(graph_proto.node_size() > li + 1 && graph_proto.node(li + 1).op_type() == "Div");
787 node_proto = graph_proto.node(li);
788 layerParams.name = node_proto.output(0);
789 layerParams.type = "Normalize";
791 DictValue axes_dict = layerParams.get("axes");
792 if (axes_dict.size() != 1)
793 CV_Error(Error::StsNotImplemented, "Multidimensional reduceL2");
794 int axis = axes_dict.getIntValue(0);
795 layerParams.set("axis",axis);
796 layerParams.set("end_axis", axis);
798 else if (layer_type == "Squeeze")
800 CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes"));
801 DictValue axes_dict = layerParams.get("axes");
802 if (axes_dict.size() != 1)
803 CV_Error(Error::StsNotImplemented, "Multidimensional squeeze");
805 int axis = axes_dict.getIntValue(0);
806 layerParams.set("axis", axis - 1);
807 layerParams.set("end_axis", axis);
808 layerParams.type = "Flatten";
810 else if (layer_type == "Unsqueeze")
812 CV_Assert(node_proto.input_size() == 1);
813 DictValue axes = layerParams.get("axes");
814 if (constBlobs.find(node_proto.input(0)) != constBlobs.end())
817 Mat input = getBlob(node_proto, constBlobs, 0);
819 std::vector<int> dims;
820 for (int j = 0; j < input.dims; j++) {
821 dims.push_back(input.size[j]);
823 CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size());
824 for (int j = 0; j < axes.size(); j++) {
825 dims.insert(dims.begin() + axes.getIntValue(j), 1);
828 Mat out = input.reshape(0, dims);
829 constBlobs.insert(std::make_pair(layerParams.name, out));
834 if (axes.size() != 1)
835 CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze");
837 MatShape inpShape = outShapes[node_proto.input(0)];
838 int axis = axes.getIntValue(0);
839 CV_Assert(0 <= axis && axis <= inpShape.size());
840 std::vector<int> outShape = inpShape;
841 outShape.insert(outShape.begin() + axis, 1);
842 layerParams.type = "Reshape";
843 layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size()));
845 else if (layer_type == "Reshape")
847 CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape"));
849 if (node_proto.input_size() == 2) {
850 Mat blob = getBlob(node_proto, constBlobs, 1);
851 CV_Assert(blob.type() == CV_32SC1);
853 layerParams.set("dim", DictValue::arrayInt<int*>(
854 blob.ptr<int>(), blob.total() ));
856 if (layer_id.find(node_proto.input(0)) == layer_id.end()) {
857 std::vector<Mat> inputs(1, getBlob(node_proto, constBlobs, 0)), outputs;
858 runLayer(layerParams, inputs, outputs);
859 constBlobs.insert(std::make_pair(layerParams.name, outputs[0]));
864 DictValue shape = layerParams.get("shape");
865 std::vector<int> dim;
866 for (int j = 0; j < shape.size(); j++) {
867 dim.push_back(shape.getIntValue(j));
870 if (layer_id.find(node_proto.input(0)) == layer_id.end()) {
871 Mat input = getBlob(node_proto, constBlobs, 0);
872 Mat out = input.reshape(0, dim);
873 constBlobs.insert(std::make_pair(layerParams.name, out));
876 replaceLayerParam(layerParams, "shape", "dim");
879 else if (layer_type == "Pad")
881 layerParams.type = "Padding";
883 else if (layer_type == "Shape")
885 CV_Assert(node_proto.input_size() == 1);
886 shapeIt = outShapes.find(node_proto.input(0));
887 CV_Assert(shapeIt != outShapes.end());
888 MatShape inpShape = shapeIt->second;
890 Mat shapeMat(inpShape.size(), 1, CV_32S);
891 for (int j = 0; j < inpShape.size(); ++j)
892 shapeMat.at<int>(j) = inpShape[j];
895 constBlobs.insert(std::make_pair(layerParams.name, shapeMat));
898 else if (layer_type == "Gather")
900 CV_Assert(node_proto.input_size() == 2);
901 CV_Assert(layerParams.has("axis"));
902 Mat input = getBlob(node_proto, constBlobs, 0);
903 Mat indexMat = getBlob(node_proto, constBlobs, 1);
904 CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1);
905 int index = indexMat.at<int>(0);
906 int axis = layerParams.get<int>("axis");
908 std::vector<cv::Range> ranges(input.dims, Range::all());
909 ranges[axis] = Range(index, index + 1);
911 Mat out = input(ranges);
912 constBlobs.insert(std::make_pair(layerParams.name, out));
915 else if (layer_type == "Concat")
917 bool hasVariableInps = false;
918 for (int i = 0; i < node_proto.input_size(); ++i)
920 if (layer_id.find(node_proto.input(i)) != layer_id.end())
922 hasVariableInps = true;
927 if (!hasVariableInps)
929 std::vector<Mat> inputs(node_proto.input_size()), concatenated;
930 for (size_t i = 0; i < inputs.size(); ++i)
932 inputs[i] = getBlob(node_proto, constBlobs, i);
934 runLayer(layerParams, inputs, concatenated);
936 CV_Assert(concatenated.size() == 1);
937 constBlobs.insert(std::make_pair(layerParams.name, concatenated[0]));
941 else if (layer_type == "Upsample")
943 layerParams.type = "Resize";
944 if (layerParams.has("scales"))
947 DictValue scales = layerParams.get("scales");
948 CV_Assert(scales.size() == 4);
949 layerParams.set("zoom_factor_y", scales.getIntValue(2));
950 layerParams.set("zoom_factor_x", scales.getIntValue(3));
955 replaceLayerParam(layerParams, "height_scale", "zoom_factor_y");
956 replaceLayerParam(layerParams, "width_scale", "zoom_factor_x");
958 replaceLayerParam(layerParams, "mode", "interpolation");
960 else if (layer_type == "LogSoftmax")
962 layerParams.type = "Softmax";
963 layerParams.set("log_softmax", true);
967 for (int j = 0; j < node_proto.input_size(); j++) {
968 if (layer_id.find(node_proto.input(j)) == layer_id.end())
969 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
973 int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams);
974 for (int i = 0; i < node_proto.output_size(); ++i)
976 layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i)));
979 std::vector<MatShape> layerInpShapes, layerOutShapes, layerInternalShapes;
980 for (int j = 0; j < node_proto.input_size(); j++) {
981 layerId = layer_id.find(node_proto.input(j));
982 if (layerId != layer_id.end()) {
983 dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, j);
984 // Collect input shapes.
985 shapeIt = outShapes.find(node_proto.input(j));
986 CV_Assert(shapeIt != outShapes.end());
987 layerInpShapes.push_back(shapeIt->second);
991 // Compute shape of output blob for this layer.
992 Ptr<Layer> layer = dstNet.getLayer(id);
993 layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes);
994 for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i)
996 outShapes[node_proto.output(i)] = layerOutShapes[i];
1001 Net readNetFromONNX(const String& onnxFile)
1003 ONNXImporter onnxImporter(onnxFile.c_str());
1005 onnxImporter.populateNet(net);
1009 Net readNetFromONNX(const char* buffer, size_t sizeBuffer)
1011 ONNXImporter onnxImporter(buffer, sizeBuffer);
1013 onnxImporter.populateNet(net);
1017 Net readNetFromONNX(const std::vector<uchar>& buffer)
1019 return readNetFromONNX(reinterpret_cast<const char*>(buffer.data()), buffer.size());
1022 Mat readTensorFromONNX(const String& path)
1024 opencv_onnx::TensorProto tensor_proto = opencv_onnx::TensorProto();
1025 std::fstream input(path.c_str(), std::ios::in | std::ios::binary);
1026 if (!tensor_proto.ParseFromIstream(&input)) {
1027 CV_Error(Error::StsUnsupportedFormat, "Failed to parse data");
1029 Mat mat = getMatFromTensor(tensor_proto);
1030 releaseONNXTensor(tensor_proto);
1034 CV__DNN_INLINE_NS_END