1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2018, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
8 #include "../precomp.hpp"
9 #include <opencv2/dnn/shape_utils.hpp>
20 #if defined(__GNUC__) && __GNUC__ >= 5
21 #pragma GCC diagnostic push
22 #pragma GCC diagnostic ignored "-Wsuggest-override"
24 #include "opencv-onnx.pb.h"
25 #if defined(__GNUC__) && __GNUC__ >= 5
26 #pragma GCC diagnostic pop
29 #include "onnx_graph_simplifier.hpp"
33 CV__DNN_INLINE_NS_BEGIN
38 opencv_onnx::ModelProto model_proto;
42 LayerInfo(int _layerId, int _outputId) : layerId(_layerId), outputId(_outputId) {}
45 std::map<std::string, Mat> getGraphTensors(
46 const opencv_onnx::GraphProto& graph_proto);
47 Mat getBlob(const opencv_onnx::NodeProto& node_proto, const std::map<std::string, Mat>& constBlobs, int index);
49 LayerParams getLayerParams(const opencv_onnx::NodeProto& node_proto);
50 bool isCeilMode(const LayerParams& layerParams);
54 ONNXImporter(const char *onnxFile)
56 std::fstream input(onnxFile, std::ios::in | std::ios::binary);
58 if (!model_proto.ParseFromIstream(&input))
59 CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model");
62 ONNXImporter(const char* buffer, size_t sizeBuffer)
64 struct _Buf : public std::streambuf
66 _Buf(const char* buffer, size_t sizeBuffer)
68 char* p = const_cast<char*>(buffer);
69 setg(p, p, p + sizeBuffer);
73 _Buf buf(buffer, sizeBuffer);
74 std::istream input(&buf);
76 if (!model_proto.ParseFromIstream(&input))
77 CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array.");
80 void populateNet(Net dstNet);
83 inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey)
85 if (layerParams.has(oldKey)) {
86 layerParams.set(newKey, layerParams.get(oldKey));
87 layerParams.erase(oldKey);
91 void releaseONNXTensor(opencv_onnx::TensorProto& tensor_proto)
93 if (!tensor_proto.raw_data().empty()) {
94 delete tensor_proto.release_raw_data();
98 void runLayer(LayerParams& params, const std::vector<Mat>& inputs,
99 std::vector<Mat>& outputs)
101 Ptr<Layer> layer = LayerFactory::createLayerInstance(params.type, params);
102 CV_Assert((bool)layer);
104 std::vector<MatShape> inpShapes(inputs.size());
106 for (size_t i = 0; i < inputs.size(); ++i)
108 inpShapes[i] = shape(inputs[i]);
109 if (i > 0 && ddepth != inputs[i].depth())
110 CV_Error(Error::StsNotImplemented, "Mixed input data types.");
111 ddepth = inputs[i].depth();
114 std::vector<MatShape> outShapes, internalShapes;
115 layer->getMemoryShapes(inpShapes, 0, outShapes, internalShapes);
117 std::vector<Mat> internals(internalShapes.size());
118 outputs.resize(outShapes.size());
119 for (size_t i = 0; i < outShapes.size(); ++i)
120 outputs[i].create(outShapes[i], ddepth);
121 for (size_t i = 0; i < internalShapes.size(); ++i)
122 internals[i].create(internalShapes[i], ddepth);
124 layer->finalize(inputs, outputs);
125 layer->forward(inputs, outputs, internals);
128 std::map<std::string, Mat> ONNXImporter::getGraphTensors(
129 const opencv_onnx::GraphProto& graph_proto)
131 opencv_onnx::TensorProto tensor_proto;
132 std::map<std::string, Mat> layers_weights;
134 for (int i = 0; i < graph_proto.initializer_size(); i++)
136 tensor_proto = graph_proto.initializer(i);
137 Mat mat = getMatFromTensor(tensor_proto);
138 releaseONNXTensor(tensor_proto);
139 layers_weights.insert(std::make_pair(tensor_proto.name(), mat));
141 return layers_weights;
144 static DictValue parse(const ::google::protobuf::RepeatedField< ::google::protobuf::int64>& src) {
145 std::vector<int32_t> dst(src.size());
146 convertInt64ToInt32(src, dst, src.size());
147 return DictValue::arrayInt(&dst[0], src.size());
150 LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_proto)
153 for(int i = 0; i < node_proto.attribute_size(); i++)
155 opencv_onnx::AttributeProto attribute_proto = node_proto.attribute(i);
156 std::string attribute_name = attribute_proto.name();
158 if(attribute_name == "kernel_shape")
160 CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
161 lp.set("kernel_size", parse(attribute_proto.ints()));
163 else if(attribute_name == "strides")
165 CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
166 lp.set("stride", parse(attribute_proto.ints()));
168 else if(attribute_name == "pads")
170 if (node_proto.op_type() == "Pad")
173 // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN.
174 // We need to shuffle it to begin0, end0, begin1, end1, ...
175 CV_Assert(attribute_proto.ints_size() % 2 == 0);
176 const int dims = attribute_proto.ints_size() / 2;
177 std::vector<int32_t> paddings;
178 paddings.reserve(attribute_proto.ints_size());
179 for (int i = 0; i < dims; ++i)
181 paddings.push_back(attribute_proto.ints(i));
182 paddings.push_back(attribute_proto.ints(dims + i));
184 lp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size()));
188 // Convolution or pooling.
189 CV_Assert(attribute_proto.ints_size() == 4 || attribute_proto.ints_size() == 6);
190 lp.set("pad", parse(attribute_proto.ints()));
193 else if(attribute_name == "auto_pad")
195 if (attribute_proto.s() == "SAME_UPPER" || attribute_proto.s() == "SAME_LOWER") {
196 lp.set("pad_mode", "SAME");
198 else if (attribute_proto.s() == "VALID") {
199 lp.set("pad_mode", "VALID");
202 else if(attribute_name == "dilations")
204 CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3);
205 lp.set("dilation", parse(attribute_proto.ints()));
207 else if (attribute_proto.has_i())
209 ::google::protobuf::int64 src = attribute_proto.i();
210 if (src < std::numeric_limits<int32_t>::min() || src > std::numeric_limits<int32_t>::max())
211 CV_Error(Error::StsOutOfRange, "Input is out of OpenCV 32S range");
213 lp.set(attribute_name, saturate_cast<int32_t>(src));
215 else if (attribute_proto.has_f())
217 lp.set(attribute_name, attribute_proto.f());
219 else if (attribute_proto.has_s())
221 lp.set(attribute_name, attribute_proto.s());
223 else if (attribute_proto.floats_size() > 0)
225 lp.set(attribute_name, DictValue::arrayReal(
226 attribute_proto.floats().data(), attribute_proto.floats_size()));
228 else if (attribute_proto.ints_size() > 0)
230 lp.set(attribute_proto.name(), parse(attribute_proto.ints()));
232 else if (attribute_proto.has_t())
234 opencv_onnx::TensorProto tensor = attribute_proto.t();
235 Mat blob = getMatFromTensor(tensor);
236 lp.blobs.push_back(blob);
238 else if (attribute_proto.has_g() || attribute_proto.strings_size() > 0 ||
239 attribute_proto.tensors_size() > 0 || attribute_proto.graphs_size() > 0)
241 CV_Error(Error::StsNotImplemented, "Unexpected attribute type");
244 CV_Error(Error::StsNotImplemented, "Unsupported attribute type");
249 Mat ONNXImporter::getBlob(const opencv_onnx::NodeProto& node_proto,
250 const std::map<std::string, Mat>& constBlobs, int index)
252 CV_Assert(index < node_proto.input_size());
253 std::map<std::string, Mat>::const_iterator constBlob;
254 constBlob = constBlobs.find(node_proto.input(index));
255 if (constBlob == constBlobs.end()) {
256 CV_Error(Error::StsObjectNotFound,
257 "Blob " + node_proto.input(index) + " not found in const blobs");
259 return constBlob->second;
262 void ONNXImporter::populateNet(Net dstNet)
264 CV_Assert(model_proto.has_graph());
265 opencv_onnx::GraphProto graph_proto = model_proto.graph();
267 simplifySubgraphs(graph_proto);
269 std::map<std::string, Mat> constBlobs = getGraphTensors(graph_proto);
270 // List of internal blobs shapes.
271 std::map<std::string, MatShape> outShapes;
272 // Add all the inputs shapes. It includes as constant blobs as network's inputs shapes.
273 for (int i = 0; i < graph_proto.input_size(); ++i)
275 opencv_onnx::ValueInfoProto valueInfoProto = graph_proto.input(i);
276 CV_Assert(valueInfoProto.has_type());
277 opencv_onnx::TypeProto typeProto = valueInfoProto.type();
278 CV_Assert(typeProto.has_tensor_type());
279 opencv_onnx::TypeProto::Tensor tensor = typeProto.tensor_type();
280 CV_Assert(tensor.has_shape());
281 opencv_onnx::TensorShapeProto tensorShape = tensor.shape();
283 MatShape inpShape(tensorShape.dim_size());
284 for (int j = 0; j < inpShape.size(); ++j)
286 inpShape[j] = tensorShape.dim(j).dim_value();
288 outShapes[valueInfoProto.name()] = inpShape;
291 std::string framework_name;
292 if (model_proto.has_producer_name()) {
293 framework_name = model_proto.producer_name();
296 // create map with network inputs (without const blobs)
297 std::map<std::string, LayerInfo> layer_id;
298 std::map<std::string, LayerInfo>::iterator layerId;
299 std::map<std::string, MatShape>::iterator shapeIt;
300 // fill map: push layer name, layer id and output id
301 std::vector<String> netInputs;
302 for (int j = 0; j < graph_proto.input_size(); j++)
304 const std::string& name = graph_proto.input(j).name();
305 if (constBlobs.find(name) == constBlobs.end()) {
306 netInputs.push_back(name);
307 layer_id.insert(std::make_pair(name, LayerInfo(0, netInputs.size() - 1)));
310 dstNet.setInputsNames(netInputs);
312 int layersSize = graph_proto.node_size();
313 LayerParams layerParams;
314 opencv_onnx::NodeProto node_proto;
316 for(int li = 0; li < layersSize; li++)
318 node_proto = graph_proto.node(li);
319 layerParams = getLayerParams(node_proto);
320 CV_Assert(node_proto.output_size() >= 1);
321 layerParams.name = node_proto.output(0);
323 std::string layer_type = node_proto.op_type();
324 layerParams.type = layer_type;
327 if (layer_type == "MaxPool")
329 layerParams.type = "Pooling";
330 layerParams.set("pool", "MAX");
331 layerParams.set("ceil_mode", layerParams.has("pad_mode"));
333 else if (layer_type == "AveragePool")
335 layerParams.type = "Pooling";
336 layerParams.set("pool", "AVE");
337 layerParams.set("ceil_mode", layerParams.has("pad_mode"));
338 layerParams.set("ave_pool_padded_area", framework_name == "pytorch");
340 else if (layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool" || layer_type == "ReduceMean")
342 CV_Assert(node_proto.input_size() == 1);
343 layerParams.type = "Pooling";
344 layerParams.set("pool", layer_type == "GlobalMaxPool"? "MAX" : "AVE");
345 layerParams.set("global_pooling", layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool");
347 if (layer_type == "ReduceMean")
349 if (layerParams.get<int>("keepdims") == 0 || !layerParams.has("axes"))
350 CV_Error(Error::StsNotImplemented, "Unsupported mode of ReduceMean operation.");
352 MatShape inpShape = outShapes[node_proto.input(0)];
353 if (inpShape.size() != 4 && inpShape.size() != 5)
354 CV_Error(Error::StsNotImplemented, "Unsupported input shape of reduce_mean operation.");
356 DictValue axes = layerParams.get("axes");
357 CV_Assert(axes.size() <= inpShape.size() - 2);
358 std::vector<int> kernel_size(inpShape.size() - 2, 1);
359 for (int i = 0; i < axes.size(); i++) {
360 int axis = axes.get<int>(i);
361 CV_Assert_N(axis >= 2 + i, axis < inpShape.size());
362 kernel_size[axis - 2] = inpShape[axis];
365 layerParams.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size()));
368 else if (layer_type == "Slice")
370 if (layerParams.has("steps")) {
371 DictValue steps = layerParams.get("steps");
372 for (int i = 0; i < steps.size(); ++i) {
373 if (steps.get<int>(i) != 1)
374 CV_Error(Error::StsNotImplemented,
375 "Slice layer only supports steps = 1");
380 if (layerParams.has("axes")) {
381 DictValue axes = layerParams.get("axes");
382 for (int i = 1; i < axes.size(); ++i) {
383 CV_Assert(axes.get<int>(i - 1) == axes.get<int>(i) - 1);
385 axis = axes.get<int>(0);
387 layerParams.set("axis", axis);
389 DictValue starts = layerParams.get("starts");
390 DictValue ends = layerParams.get("ends");
391 CV_Assert(starts.size() == ends.size());
393 std::vector<int> begin;
394 std::vector<int> end;
396 begin.resize(axis, 0);
397 end.resize(axis, -1);
400 for (int i = 0; i < starts.size(); ++i)
402 begin.push_back(starts.get<int>(i));
403 int finish = ends.get<int>(i);
404 end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim
406 layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size()));
407 layerParams.set("end", DictValue::arrayInt(&end[0], end.size()));
409 else if (layer_type == "Split")
411 if (layerParams.has("split"))
413 DictValue splits = layerParams.get("split");
414 const int numSplits = splits.size();
415 CV_Assert(numSplits > 1);
417 std::vector<int> slicePoints(numSplits - 1, splits.get<int>(0));
418 for (int i = 1; i < splits.size() - 1; ++i)
420 slicePoints[i] = slicePoints[i - 1] + splits.get<int>(i - 1);
422 layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size()));
426 layerParams.set("num_split", node_proto.output_size());
428 layerParams.type = "Slice";
430 else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub")
432 bool isSub = layer_type == "Sub";
433 CV_CheckEQ(node_proto.input_size(), 2, "");
434 if (layer_id.find(node_proto.input(1)) == layer_id.end())
436 Mat blob = getBlob(node_proto, constBlobs, 1);
437 blob = blob.reshape(1, 1);
438 if (blob.total() == 1) {
439 layerParams.type = "Power";
440 layerParams.set("shift", (isSub ? -1 : 1) * blob.at<float>(0));
443 layerParams.type = "Scale";
444 layerParams.set("bias_term", true);
445 layerParams.blobs.push_back((isSub ? -1 : 1) * blob);
448 else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)])
450 layerParams.type = "Eltwise";
453 static float subCoeffs[] = {1.f, -1.f};
454 layerParams.set("coeff", DictValue::arrayReal<float*>(subCoeffs, 2));
461 LayerParams powerParams;
462 powerParams.name = layerParams.name + "/neg";
463 powerParams.type = "Power";
464 powerParams.set("scale", -1);
467 int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams);
469 layerId = layer_id.find(node_proto.input(1));
470 CV_Assert(layerId != layer_id.end());
471 dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
473 layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0)));
474 outShapes[powerParams.name] = outShapes[node_proto.input(1)];
476 //Replace input to Power
477 node_proto.set_input(1, powerParams.name);
479 layerParams.type = "Scale";
480 layerParams.set("bias_term", true);
483 else if (layer_type == "Max")
485 layerParams.type = "Eltwise";
486 layerParams.set("operation", "max");
488 else if (layer_type == "Neg")
490 layerParams.type = "Power";
491 layerParams.set("scale", -1);
493 else if (layer_type == "Constant")
495 CV_Assert(node_proto.input_size() == 0);
496 CV_Assert(layerParams.blobs.size() == 1);
497 constBlobs.insert(std::make_pair(layerParams.name, layerParams.blobs[0]));
500 else if (layer_type == "ImageScaler")
502 const float scale = layerParams.has("scale") ? layerParams.get<float>("scale") : 1.0f;
503 layerParams.erase("scale");
505 if (layerParams.has("bias"))
507 layerParams.type = "Scale";
508 layerParams.blobs.push_back(
509 Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale));
511 layerParams.set("bias_term", true);
512 Mat bias(1, layerParams.get("bias").size(), CV_32FC1);
513 for (int j = 0; j < bias.total(); j++) {
514 bias.at<float>(0, j) = layerParams.get("bias").getRealValue(j);
516 layerParams.blobs.push_back(bias);
517 layerParams.erase("bias");
520 layerParams.set("scale", scale);
521 layerParams.type = "Power";
524 else if (layer_type == "Clip")
526 layerParams.type = "ReLU6";
527 replaceLayerParam(layerParams, "min", "min_value");
528 replaceLayerParam(layerParams, "max", "max_value");
531 else if (layer_type == "LeakyRelu")
533 layerParams.type = "ReLU";
534 replaceLayerParam(layerParams, "alpha", "negative_slope");
536 else if (layer_type == "LRN")
538 replaceLayerParam(layerParams, "size", "local_size");
540 else if (layer_type == "InstanceNormalization")
542 if (node_proto.input_size() != 3)
543 CV_Error(Error::StsNotImplemented,
544 "Expected input, scale, bias");
546 layerParams.blobs.resize(4);
547 layerParams.blobs[2] = getBlob(node_proto, constBlobs, 1); // weightData
548 layerParams.blobs[3] = getBlob(node_proto, constBlobs, 2); // biasData
549 layerParams.set("has_bias", true);
550 layerParams.set("has_weight", true);
552 // Get number of channels in input
553 int size = layerParams.blobs[2].total();
554 layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean
555 layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std
557 LayerParams mvnParams;
558 mvnParams.name = layerParams.name + "/MVN";
559 mvnParams.type = "MVN";
560 mvnParams.set("eps", layerParams.get<float>("epsilon"));
561 layerParams.erase("epsilon");
564 int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams);
566 layerId = layer_id.find(node_proto.input(0));
567 CV_Assert(layerId != layer_id.end());
568 dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
570 layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0)));
571 outShapes[mvnParams.name] = outShapes[node_proto.input(0)];
573 //Replace Batch Norm's input to MVN
574 node_proto.set_input(0, mvnParams.name);
575 layerParams.type = "BatchNorm";
577 else if (layer_type == "BatchNormalization")
579 if (node_proto.input_size() != 5)
580 CV_Error(Error::StsNotImplemented,
581 "Expected input, scale, bias, mean and var");
583 layerParams.type = "BatchNorm";
584 replaceLayerParam(layerParams, "epsilon", "eps");
585 replaceLayerParam(layerParams, "spatial", "use_global_stats");
587 Mat meanData = getBlob(node_proto, constBlobs, 3);
588 Mat stdData = getBlob(node_proto, constBlobs, 4);
590 layerParams.blobs.push_back(meanData);
591 layerParams.blobs.push_back(stdData);
593 if (!node_proto.input(1).empty()) {
594 layerParams.set("has_weight", true);
595 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 1)); // weightData
597 layerParams.set("has_weight", false);
600 if (!node_proto.input(2).empty()) {
601 layerParams.set("has_bias", true);
602 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 2)); // biasData
604 layerParams.set("has_bias", false);
607 else if (layer_type == "Gemm")
609 CV_Assert(node_proto.input_size() >= 2);
610 layerParams.type = "InnerProduct";
611 Mat weights = getBlob(node_proto, constBlobs, 1);
613 if (layerParams.has("transB") && !layerParams.get<int>("transB")) {
614 transpose(weights, weights);
617 layerParams.blobs.push_back(weights);
619 if (node_proto.input_size() == 3) {
620 Mat bias = getBlob(node_proto, constBlobs, 2);
621 layerParams.blobs.push_back(bias);
624 layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]);
625 layerParams.set("bias_term", node_proto.input_size() == 3);
627 else if (layer_type == "MatMul")
629 CV_Assert(node_proto.input_size() == 2);
630 layerParams.type = "InnerProduct";
631 Mat blob = getBlob(node_proto, constBlobs, 1);
632 layerParams.blobs.push_back(blob.t());
633 layerParams.set("bias_term", false);
634 layerParams.set("num_output", layerParams.blobs[0].size[0]);
636 else if (layer_type == "Mul" || layer_type == "Div")
638 CV_Assert(node_proto.input_size() == 2);
640 bool isDiv = layer_type == "Div";
642 bool haveVariables = false;
643 for (int i = 0; i < 2; ++i)
645 if (constBlobs.find(node_proto.input(i)) != constBlobs.end())
648 haveVariables = true;
650 if (constId != -1 && haveVariables)
652 Mat blob = getBlob(node_proto, constBlobs, constId);
653 blob = blob.reshape(1, 1);
654 if (blob.total() == 1) {
655 float coeff = isDiv ? 1.0 / blob.at<float>(0) : blob.at<float>(0);
656 layerParams.set("scale", coeff);
657 layerParams.type = "Power";
661 divide(1.0, blob, blob);
662 layerParams.blobs.push_back(blob);
663 layerParams.type = "Scale";
666 else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)])
668 layerParams.type = "Eltwise";
669 layerParams.set("operation", isDiv ? "div" : "prod");
675 LayerParams powerParams;
676 powerParams.name = layerParams.name + "/inv";
677 powerParams.type = "Power";
678 powerParams.set("power", -1);
681 int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams);
683 layerId = layer_id.find(node_proto.input(1));
684 CV_Assert(layerId != layer_id.end());
685 dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
687 layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0)));
688 outShapes[powerParams.name] = outShapes[node_proto.input(1)];
690 //Replace input to Power
691 node_proto.set_input(1, powerParams.name);
693 layerParams.type = "Scale";
698 Mat inp0 = getBlob(node_proto, constBlobs, 0);
699 Mat inp1 = getBlob(node_proto, constBlobs, 1);
700 if (inp0.size != inp1.size)
701 CV_Error(Error::StsNotImplemented, "Constant multiply with different shapes");
705 divide(inp0, inp1, out);
707 multiply(inp0, inp1, out);
709 out = out.reshape(1, inp0.dims, inp0.size);
710 out.dims = inp0.dims; // to workaround dims == 1
711 constBlobs.insert(std::make_pair(layerParams.name, out));
715 else if (layer_type == "Conv")
717 CV_Assert(node_proto.input_size() >= 2);
718 layerParams.type = "Convolution";
719 for (int j = 1; j < node_proto.input_size(); j++) {
720 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
722 layerParams.set("num_output", layerParams.blobs[0].size[0]);
723 layerParams.set("bias_term", node_proto.input_size() == 3);
725 else if (layer_type == "ConvTranspose")
727 CV_Assert(node_proto.input_size() >= 2);
728 layerParams.type = "Deconvolution";
729 for (int j = 1; j < node_proto.input_size(); j++) {
730 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
732 layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get<int>("group", 1));
733 layerParams.set("bias_term", node_proto.input_size() == 3);
735 if (!layerParams.has("kernel_size"))
736 CV_Error(Error::StsNotImplemented,
737 "Required attribute 'kernel_size' is not present.");
739 if (layerParams.has("output_shape"))
741 const DictValue& outShape = layerParams.get("output_shape");
742 DictValue strides = layerParams.get("stride");
743 DictValue kernel = layerParams.get("kernel_size");
746 std::vector<int> adjust_pads;
747 if (layerParams.has("pad_mode"))
749 padMode = toUpperCase(layerParams.get<String>("pad_mode"));
750 if (padMode != "SAME" && padMode != "VALID")
751 CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
753 for (int i = 0; i < strides.size(); i++)
755 int sz = outShape.get<int>(2 + i);
756 int stride = strides.get<int>(i);
757 adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride :
758 (sz - kernel.get<int>(i)) % stride);
760 layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size()));
763 else if (layerParams.has("output_padding"))
765 replaceLayerParam(layerParams, "output_padding", "adj");
768 else if (layer_type == "Transpose")
770 layerParams.type = "Permute";
771 replaceLayerParam(layerParams, "perm", "order");
773 CV_Assert(node_proto.input_size() == 1);
774 if (constBlobs.find(node_proto.input(0)) != constBlobs.end())
776 std::vector<Mat> inputs(1, getBlob(node_proto, constBlobs, 0)), transposed;
777 runLayer(layerParams, inputs, transposed);
778 CV_Assert(transposed.size() == 1);
779 constBlobs.insert(std::make_pair(layerParams.name, transposed[0]));
783 else if (layer_type == "ReduceL2")
785 CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes"));
786 CV_Assert(graph_proto.node_size() > li + 1 && graph_proto.node(li + 1).op_type() == "Div");
788 node_proto = graph_proto.node(li);
789 layerParams.name = node_proto.output(0);
790 layerParams.type = "Normalize";
792 DictValue axes_dict = layerParams.get("axes");
793 if (axes_dict.size() != 1)
794 CV_Error(Error::StsNotImplemented, "Multidimensional reduceL2");
795 int axis = axes_dict.getIntValue(0);
796 layerParams.set("axis",axis);
797 layerParams.set("end_axis", axis);
799 else if (layer_type == "Squeeze")
801 CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes"));
802 DictValue axes_dict = layerParams.get("axes");
803 if (axes_dict.size() != 1)
804 CV_Error(Error::StsNotImplemented, "Multidimensional squeeze");
806 int axis = axes_dict.getIntValue(0);
807 layerParams.set("axis", axis - 1);
808 layerParams.set("end_axis", axis);
809 layerParams.type = "Flatten";
811 else if (layer_type == "Unsqueeze")
813 CV_Assert(node_proto.input_size() == 1);
814 DictValue axes = layerParams.get("axes");
815 if (constBlobs.find(node_proto.input(0)) != constBlobs.end())
818 Mat input = getBlob(node_proto, constBlobs, 0);
820 std::vector<int> dims;
821 for (int j = 0; j < input.dims; j++) {
822 dims.push_back(input.size[j]);
824 CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size());
825 for (int j = 0; j < axes.size(); j++) {
826 dims.insert(dims.begin() + axes.getIntValue(j), 1);
829 Mat out = input.reshape(0, dims);
830 constBlobs.insert(std::make_pair(layerParams.name, out));
835 if (axes.size() != 1)
836 CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze");
838 MatShape inpShape = outShapes[node_proto.input(0)];
839 int axis = axes.getIntValue(0);
840 CV_Assert(0 <= axis && axis <= inpShape.size());
841 std::vector<int> outShape = inpShape;
842 outShape.insert(outShape.begin() + axis, 1);
843 layerParams.type = "Reshape";
844 layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size()));
846 else if (layer_type == "Reshape")
848 CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape"));
850 if (node_proto.input_size() == 2) {
851 Mat blob = getBlob(node_proto, constBlobs, 1);
852 CV_Assert(blob.type() == CV_32SC1);
854 layerParams.set("dim", DictValue::arrayInt<int*>(
855 blob.ptr<int>(), blob.total() ));
857 if (layer_id.find(node_proto.input(0)) == layer_id.end()) {
858 std::vector<Mat> inputs(1, getBlob(node_proto, constBlobs, 0)), outputs;
859 runLayer(layerParams, inputs, outputs);
860 constBlobs.insert(std::make_pair(layerParams.name, outputs[0]));
865 DictValue shape = layerParams.get("shape");
866 std::vector<int> dim;
867 for (int j = 0; j < shape.size(); j++) {
868 dim.push_back(shape.getIntValue(j));
871 if (layer_id.find(node_proto.input(0)) == layer_id.end()) {
872 Mat input = getBlob(node_proto, constBlobs, 0);
873 Mat out = input.reshape(0, dim);
874 constBlobs.insert(std::make_pair(layerParams.name, out));
877 replaceLayerParam(layerParams, "shape", "dim");
880 else if (layer_type == "Pad")
882 layerParams.type = "Padding";
884 else if (layer_type == "Shape")
886 CV_Assert(node_proto.input_size() == 1);
887 shapeIt = outShapes.find(node_proto.input(0));
888 CV_Assert(shapeIt != outShapes.end());
889 MatShape inpShape = shapeIt->second;
891 Mat shapeMat(inpShape.size(), 1, CV_32S);
892 for (int j = 0; j < inpShape.size(); ++j)
893 shapeMat.at<int>(j) = inpShape[j];
896 constBlobs.insert(std::make_pair(layerParams.name, shapeMat));
899 else if (layer_type == "Gather")
901 CV_Assert(node_proto.input_size() == 2);
902 CV_Assert(layerParams.has("axis"));
903 Mat input = getBlob(node_proto, constBlobs, 0);
904 Mat indexMat = getBlob(node_proto, constBlobs, 1);
905 CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1);
906 int index = indexMat.at<int>(0);
907 int axis = layerParams.get<int>("axis");
909 std::vector<cv::Range> ranges(input.dims, Range::all());
910 ranges[axis] = Range(index, index + 1);
912 Mat out = input(ranges);
913 constBlobs.insert(std::make_pair(layerParams.name, out));
916 else if (layer_type == "Concat")
918 bool hasVariableInps = false;
919 for (int i = 0; i < node_proto.input_size(); ++i)
921 if (layer_id.find(node_proto.input(i)) != layer_id.end())
923 hasVariableInps = true;
928 if (!hasVariableInps)
930 std::vector<Mat> inputs(node_proto.input_size()), concatenated;
931 for (size_t i = 0; i < inputs.size(); ++i)
933 inputs[i] = getBlob(node_proto, constBlobs, i);
935 runLayer(layerParams, inputs, concatenated);
937 CV_Assert(concatenated.size() == 1);
938 constBlobs.insert(std::make_pair(layerParams.name, concatenated[0]));
942 else if (layer_type == "Upsample")
944 layerParams.type = "Resize";
945 if (layerParams.has("scales"))
948 DictValue scales = layerParams.get("scales");
949 CV_Assert(scales.size() == 4);
950 layerParams.set("zoom_factor_y", scales.getIntValue(2));
951 layerParams.set("zoom_factor_x", scales.getIntValue(3));
956 replaceLayerParam(layerParams, "height_scale", "zoom_factor_y");
957 replaceLayerParam(layerParams, "width_scale", "zoom_factor_x");
959 replaceLayerParam(layerParams, "mode", "interpolation");
961 if (layerParams.get<String>("interpolation") == "linear" && framework_name == "pytorch") {
962 layerParams.type = "Resize";
963 Mat scales = getBlob(node_proto, constBlobs, 1);
964 CV_Assert(scales.total() == 4);
965 layerParams.set("interpolation", "opencv_linear");
966 layerParams.set("zoom_factor_y", scales.at<float>(2));
967 layerParams.set("zoom_factor_x", scales.at<float>(3));
970 else if (layer_type == "LogSoftmax")
972 layerParams.type = "Softmax";
973 layerParams.set("log_softmax", true);
977 for (int j = 0; j < node_proto.input_size(); j++) {
978 if (layer_id.find(node_proto.input(j)) == layer_id.end())
979 layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j));
983 int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams);
984 for (int i = 0; i < node_proto.output_size(); ++i)
986 layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i)));
989 std::vector<MatShape> layerInpShapes, layerOutShapes, layerInternalShapes;
990 for (int j = 0; j < node_proto.input_size(); j++) {
991 layerId = layer_id.find(node_proto.input(j));
992 if (layerId != layer_id.end()) {
993 dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, j);
994 // Collect input shapes.
995 shapeIt = outShapes.find(node_proto.input(j));
996 CV_Assert(shapeIt != outShapes.end());
997 layerInpShapes.push_back(shapeIt->second);
1001 // Compute shape of output blob for this layer.
1002 Ptr<Layer> layer = dstNet.getLayer(id);
1003 layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes);
1004 for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i)
1006 outShapes[node_proto.output(i)] = layerOutShapes[i];
1011 Net readNetFromONNX(const String& onnxFile)
1013 ONNXImporter onnxImporter(onnxFile.c_str());
1015 onnxImporter.populateNet(net);
1019 Net readNetFromONNX(const char* buffer, size_t sizeBuffer)
1021 ONNXImporter onnxImporter(buffer, sizeBuffer);
1023 onnxImporter.populateNet(net);
1027 Net readNetFromONNX(const std::vector<uchar>& buffer)
1029 return readNetFromONNX(reinterpret_cast<const char*>(buffer.data()), buffer.size());
1032 Mat readTensorFromONNX(const String& path)
1034 opencv_onnx::TensorProto tensor_proto = opencv_onnx::TensorProto();
1035 std::fstream input(path.c_str(), std::ios::in | std::ios::binary);
1036 if (!tensor_proto.ParseFromIstream(&input)) {
1037 CV_Error(Error::StsUnsupportedFormat, "Failed to parse data");
1039 Mat mat = getMatFromTensor(tensor_proto);
1040 releaseONNXTensor(tensor_proto);
1044 CV__DNN_INLINE_NS_END