From e59793cc75ed5476c7d85a61c539c5cf8944a54c Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 24 Sep 2020 05:08:35 +0000 Subject: [PATCH] dnn: improve debugging of ONNX parsing errors --- modules/dnn/src/layers/resize_layer.cpp | 2 +- modules/dnn/src/onnx/onnx_importer.cpp | 460 ++++++++++++++++++++------------ 2 files changed, 283 insertions(+), 179 deletions(-) diff --git a/modules/dnn/src/layers/resize_layer.cpp b/modules/dnn/src/layers/resize_layer.cpp index 7fc2115..370679c 100644 --- a/modules/dnn/src/layers/resize_layer.cpp +++ b/modules/dnn/src/layers/resize_layer.cpp @@ -39,7 +39,7 @@ public: CV_Assert(params.has("zoom_factor_x") && params.has("zoom_factor_y")); } interpolation = params.get("interpolation"); - CV_Assert(interpolation == "nearest" || interpolation == "opencv_linear" || interpolation == "bilinear"); + CV_Check(interpolation, interpolation == "nearest" || interpolation == "opencv_linear" || interpolation == "bilinear", ""); alignCorners = params.get("align_corners", false); } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index c6f8b65..56683f4 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -8,6 +8,11 @@ #include "../precomp.hpp" #include +#include +#undef CV_LOG_STRIP_LEVEL +#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_DEBUG + 1 +#include + #ifdef HAVE_PROTOBUF #include @@ -44,28 +49,43 @@ class ONNXImporter std::map getGraphTensors( const opencv_onnx::GraphProto& graph_proto); - Mat getBlob(const opencv_onnx::NodeProto& node_proto, const std::map& constBlobs, int index); + Mat getBlob(const opencv_onnx::NodeProto& node_proto, int index); + Mat getBlob(const std::string& input_name); LayerParams getLayerParams(const opencv_onnx::NodeProto& node_proto); bool isCeilMode(const LayerParams& layerParams); - void addLayer(Net& dstNet, LayerParams& layerParams, - const opencv_onnx::NodeProto& node_proto, - std::map& layer_id, - std::map& outShapes); + void addConstant(const std::string& name, const Mat& blob); + void addLayer(LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto); public: - ONNXImporter(const char *onnxFile) + ONNXImporter(Net& net, const char *onnxFile) + : dstNet(net) { + CV_Assert(onnxFile); + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing ONNX model from file: " << onnxFile); + std::fstream input(onnxFile, std::ios::in | std::ios::binary); + if (!input) + { + CV_Error(Error::StsBadArg, cv::format("Can't read ONNX file: %s", onnxFile)); + } if (!model_proto.ParseFromIstream(&input)) - CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model"); + { + CV_Error(Error::StsUnsupportedFormat, cv::format("Failed to parse ONNX model: %s", onnxFile)); + } + + populateNet(); } - ONNXImporter(const char* buffer, size_t sizeBuffer) + ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) + : dstNet(net) { + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); + struct _Buf : public std::streambuf { _Buf(const char* buffer, size_t sizeBuffer) @@ -80,9 +100,27 @@ public: if (!model_proto.ParseFromIstream(&input)) CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array."); + + populateNet(); } - void populateNet(Net dstNet); + void populateNet(); + +protected: + Net& dstNet; + + opencv_onnx::GraphProto graph_proto; + std::string framework_name; + + std::map constBlobs; + + std::map outShapes; // List of internal blobs shapes. + typedef std::map::iterator IterShape_t; + + std::map layer_id; + typedef std::map::iterator IterLayerId_t; + + void handleNode(const opencv_onnx::NodeProto& node_proto); }; inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) @@ -232,7 +270,7 @@ LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_prot } else if (attribute_proto.ints_size() > 0) { - lp.set(attribute_proto.name(), parse(attribute_proto.ints())); + lp.set(attribute_name, parse(attribute_proto.ints())); } else if (attribute_proto.has_t()) { @@ -240,38 +278,63 @@ LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_prot Mat blob = getMatFromTensor(tensor); lp.blobs.push_back(blob); } - else if (attribute_proto.has_g() || attribute_proto.strings_size() > 0 || - attribute_proto.tensors_size() > 0 || attribute_proto.graphs_size() > 0) + else if (attribute_proto.has_g()) + { + CV_Error(Error::StsNotImplemented, cv::format("DNN/ONNX/Attribute[%s]: 'Graph' is not supported", attribute_name.c_str())); + } + else if (attribute_proto.graphs_size() > 0) + { + CV_Error(Error::StsNotImplemented, + cv::format("DNN/ONNX/Attribute[%s]: 'Graphs' (%d) in attributes is not supported", + attribute_name.c_str(), attribute_proto.graphs_size()) + ); + } + else if (attribute_proto.strings_size() > 0) + { + std::string msg = cv::format("DNN/ONNX/Attribute[%s]: 'Strings' (%d) are not supported", + attribute_name.c_str(), attribute_proto.strings_size()); + CV_LOG_ERROR(NULL, msg); + for (int i = 0; i < attribute_proto.strings_size(); i++) + { + CV_LOG_ERROR(NULL, " Attribute[" << attribute_name << "].string(" << i << ") = '" << attribute_proto.strings(i) << "'"); + } + CV_Error(Error::StsNotImplemented, msg); + } + else if (attribute_proto.tensors_size() > 0) { - CV_Error(Error::StsNotImplemented, "Unexpected attribute type"); + CV_Error(Error::StsNotImplemented, + cv::format("DNN/ONNX/Attribute[%s]: 'Tensors' (%d) in attributes are not supported", + attribute_name.c_str(), attribute_proto.tensors_size()) + ); } else - CV_Error(Error::StsNotImplemented, "Unsupported attribute type"); + { + CV_Error(Error::StsNotImplemented, cv::format("DNN/ONNX/Attribute[%s]: unsupported attribute format", attribute_name.c_str())); + } } return lp; } -Mat ONNXImporter::getBlob(const opencv_onnx::NodeProto& node_proto, - const std::map& constBlobs, int index) +Mat ONNXImporter::getBlob(const opencv_onnx::NodeProto& node_proto, int index) { CV_Assert(index < node_proto.input_size()); - std::map::const_iterator constBlob; - constBlob = constBlobs.find(node_proto.input(index)); - if (constBlob == constBlobs.end()) { - CV_Error(Error::StsObjectNotFound, - "Blob " + node_proto.input(index) + " not found in const blobs"); + const std::string& input_name = node_proto.input(index); + return getBlob(input_name); +} + +Mat ONNXImporter::getBlob(const std::string& input_name) +{ + std::map::const_iterator constBlob = constBlobs.find(input_name); + if (constBlob == constBlobs.end()) + { + CV_Error(Error::StsBadArg, std::string("Blob ") + input_name + " not found in const blobs"); } return constBlob->second; } -void ONNXImporter::addLayer(Net& dstNet, LayerParams& layerParams, - const opencv_onnx::NodeProto& node_proto, - std::map& layer_id, - std::map& outShapes) +void ONNXImporter::addLayer(LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto) { - std::map::iterator layerId; - std::map::iterator shapeIt; - int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); for (int i = 0; i < node_proto.output_size(); ++i) { @@ -280,19 +343,21 @@ void ONNXImporter::addLayer(Net& dstNet, LayerParams& layerParams, std::vector layerInpShapes, layerOutShapes, layerInternalShapes; int inpNum = 0; - for (int j = 0; j < node_proto.input_size(); j++) { - layerId = layer_id.find(node_proto.input(j)); + for (int j = 0; j < node_proto.input_size(); j++) + { + const std::string& input_name = node_proto.input(j); + IterLayerId_t layerId = layer_id.find(input_name); if (layerId != layer_id.end()) { dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); ++inpNum; // Collect input shapes. - shapeIt = outShapes.find(node_proto.input(j)); + IterShape_t shapeIt = outShapes.find(input_name); CV_Assert(shapeIt != outShapes.end()); layerInpShapes.push_back(shapeIt->second); } } // Compute shape of output blob for this layer. - Ptr layer = dstNet.getLayer(id); + Ptr layer = dstNet.getLayer(id); // FIXIT: avoid instantiation of layers during the import stage layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes); for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i) { @@ -300,29 +365,43 @@ void ONNXImporter::addLayer(Net& dstNet, LayerParams& layerParams, } } -static void addConstant(const std::string& name, - const Mat& blob, - std::map& constBlobs, - std::map& outShapes) +void ONNXImporter::addConstant(const std::string& name, const Mat& blob) { constBlobs.insert(std::make_pair(name, blob)); outShapes.insert(std::make_pair(name, shape(blob))); } -void ONNXImporter::populateNet(Net dstNet) +void ONNXImporter::populateNet() { CV_Assert(model_proto.has_graph()); - opencv_onnx::GraphProto graph_proto = model_proto.graph(); + graph_proto = model_proto.graph(); + + std::string framework_version; + if (model_proto.has_producer_name()) + framework_name = model_proto.producer_name(); + if (model_proto.has_producer_version()) + framework_version = model_proto.producer_version(); + + CV_LOG_INFO(NULL, "DNN/ONNX: loading ONNX" + << (model_proto.has_ir_version() ? cv::format(" v%d", (int)model_proto.ir_version()) : cv::String()) + << " model produced by '" << framework_name << "'" + << (framework_version.empty() ? cv::String() : cv::format(":%s", framework_version.c_str())) + << ". Number of nodes = " << graph_proto.node_size() + << ", inputs = " << graph_proto.input_size() + << ", outputs = " << graph_proto.output_size() + ); simplifySubgraphs(graph_proto); - std::map constBlobs = getGraphTensors(graph_proto); - // List of internal blobs shapes. - std::map outShapes; + const int layersSize = graph_proto.node_size(); + CV_LOG_DEBUG(NULL, "DNN/ONNX: graph simplified to " << layersSize << " nodes"); + + constBlobs = getGraphTensors(graph_proto); // Add all the inputs shapes. It includes as constant blobs as network's inputs shapes. for (int i = 0; i < graph_proto.input_size(); ++i) { - opencv_onnx::ValueInfoProto valueInfoProto = graph_proto.input(i); + const opencv_onnx::ValueInfoProto& valueInfoProto = graph_proto.input(i); + CV_Assert(valueInfoProto.has_name()); CV_Assert(valueInfoProto.has_type()); opencv_onnx::TypeProto typeProto = valueInfoProto.type(); CV_Assert(typeProto.has_tensor_type()); @@ -342,15 +421,7 @@ void ONNXImporter::populateNet(Net dstNet) outShapes[valueInfoProto.name()] = inpShape; } - std::string framework_name; - if (model_proto.has_producer_name()) { - framework_name = model_proto.producer_name(); - } - // create map with network inputs (without const blobs) - std::map layer_id; - std::map::iterator layerId; - std::map::iterator shapeIt; // fill map: push layer name, layer id and output id std::vector netInputs; for (int j = 0; j < graph_proto.input_size(); j++) @@ -363,20 +434,33 @@ void ONNXImporter::populateNet(Net dstNet) } dstNet.setInputsNames(netInputs); - int layersSize = graph_proto.node_size(); - LayerParams layerParams; - opencv_onnx::NodeProto node_proto; - for(int li = 0; li < layersSize; li++) { - node_proto = graph_proto.node(li); - layerParams = getLayerParams(node_proto); - CV_Assert(node_proto.output_size() >= 1); - layerParams.name = node_proto.output(0); + const opencv_onnx::NodeProto& node_proto = graph_proto.node(li); + handleNode(node_proto); + } - std::string layer_type = node_proto.op_type(); - layerParams.type = layer_type; + CV_LOG_DEBUG(NULL, "DNN/ONNX: import completed!"); +} + +void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; // TODO FIXIT + CV_Assert(node_proto.output_size() >= 1); + std::string name = node_proto.output(0); + std::string layer_type = node_proto.op_type(); + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " + << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) + ); + + try + { + // FIXIT not all cases can be repacked into "LayerParams". Importer should handle such cases directly for each "layer_type" + LayerParams layerParams = getLayerParams(node_proto); + + layerParams.name = name; + layerParams.type = layer_type; if (layer_type == "MaxPool") { @@ -437,7 +521,7 @@ void ONNXImporter::populateNet(Net dstNet) opencv_onnx::NodeProto proto; proto.add_input(node_proto.input(0)); proto.add_output(reshapeLp.name); - addLayer(dstNet, reshapeLp, proto, layer_id, outShapes); + addLayer(reshapeLp, proto); LayerParams avgLp; avgLp.name = layerParams.name + "/avg"; @@ -458,7 +542,7 @@ void ONNXImporter::populateNet(Net dstNet) node_proto.set_input(0, reshapeLp.name); node_proto.set_output(0, avgLp.name); - addLayer(dstNet, avgLp, node_proto, layer_id, outShapes); + addLayer(avgLp, node_proto); } else { @@ -478,7 +562,7 @@ void ONNXImporter::populateNet(Net dstNet) poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); node_proto.set_output(0, poolLp.name); - addLayer(dstNet, poolLp, node_proto, layer_id, outShapes); + addLayer(poolLp, node_proto); } layerParams.type = "Reshape"; @@ -500,7 +584,7 @@ void ONNXImporter::populateNet(Net dstNet) opencv_onnx::NodeProto proto; proto.add_input(node_proto.input(0)); proto.add_output(reshapeLp.name); - addLayer(dstNet, reshapeLp, proto, layer_id, outShapes); + addLayer(reshapeLp, proto); LayerParams poolLp = layerParams; poolLp.name = layerParams.name + "/pool"; @@ -508,7 +592,7 @@ void ONNXImporter::populateNet(Net dstNet) node_proto.set_input(0, reshapeLp.name); node_proto.set_output(0, poolLp.name); - addLayer(dstNet, poolLp, node_proto, layer_id, outShapes); + addLayer(poolLp, node_proto); layerParams.type = "Reshape"; int targetShape[] = {1}; @@ -564,12 +648,12 @@ void ONNXImporter::populateNet(Net dstNet) for (int i = 1; i < inp_size; i++) { CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); } - Mat start_blob = getBlob(node_proto, constBlobs, 1); - Mat end_blob = getBlob(node_proto, constBlobs, 2); + Mat start_blob = getBlob(node_proto, 1); + Mat end_blob = getBlob(node_proto, 2); CV_Assert(start_blob.total() == end_blob.total()); if (inp_size > 3) { - Mat axes_blob = getBlob(node_proto, constBlobs, 3); + Mat axes_blob = getBlob(node_proto, 3); const int* axes = (int*)axes_blob.data; for (int i = 1; i < axes_blob.total(); ++i) { CV_Assert(axes[i - 1] == axes[i] - 1); @@ -592,7 +676,7 @@ void ONNXImporter::populateNet(Net dstNet) if (inp_size == 5) { CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); - Mat step_blob = getBlob(node_proto, constBlobs, 4); + Mat step_blob = getBlob(node_proto, 4); // Very strange application for Slice op with tensor reversing. // We just workaround it for 2d constants. @@ -601,13 +685,13 @@ void ONNXImporter::populateNet(Net dstNet) start_blob.at(0) == -1 && step_blob.at(0) == -1 && end_blob.at(0) == std::numeric_limits::min()) { - Mat inp = getBlob(node_proto, constBlobs, 0); + Mat inp = getBlob(node_proto, 0); if (inp.dims == 2) { Mat flipped; flip(inp, flipped, 0); - addConstant(layerParams.name, flipped, constBlobs, outShapes); - continue; + addConstant(layerParams.name, flipped); + return; } } CV_CheckEQ(countNonZero(step_blob != 1), 0, "Slice layer only supports steps = 1"); @@ -619,13 +703,13 @@ void ONNXImporter::populateNet(Net dstNet) if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { - Mat inp = getBlob(node_proto, constBlobs, 0); + Mat inp = getBlob(node_proto, 0); std::vector inputs, sliced; inputs.push_back(inp); runLayer(layerParams, inputs, sliced); CV_Assert(sliced.size() == 1); - addConstant(layerParams.name, sliced[0], constBlobs, outShapes); - continue; + addConstant(layerParams.name, sliced[0]); + return; } } else if (layer_type == "Split") @@ -657,17 +741,17 @@ void ONNXImporter::populateNet(Net dstNet) bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); if (is_const_0 && is_const_1) { - Mat blob_0 = getBlob(node_proto, constBlobs, 0); - Mat blob_1 = getBlob(node_proto, constBlobs, 1); + Mat blob_0 = getBlob(node_proto, 0); + Mat blob_1 = getBlob(node_proto, 1); CV_Assert(blob_0.size == blob_1.size); Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); - addConstant(layerParams.name, output, constBlobs, outShapes); - continue; + addConstant(layerParams.name, output); + return; } else if (is_const_0 || is_const_1) { int const_blob_id = is_const_0 ? 0 : 1; - Mat blob = getBlob(node_proto, constBlobs, const_blob_id); + Mat blob = getBlob(node_proto, const_blob_id); int blob_total = blob.total(); if (blob_total == 1) { layerParams.type = "Power"; @@ -729,7 +813,7 @@ void ONNXImporter::populateNet(Net dstNet) //Create Power layer int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); //Connect to input - layerId = layer_id.find(node_proto.input(1)); + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); CV_Assert(layerId != layer_id.end()); dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape @@ -748,7 +832,7 @@ void ONNXImporter::populateNet(Net dstNet) if (layer_id.find(node_proto.input(1)) != layer_id.end()) CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); - Mat blob = getBlob(node_proto, constBlobs, 1); + Mat blob = getBlob(node_proto, 1); if (blob.total() != 1) CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); @@ -770,8 +854,8 @@ void ONNXImporter::populateNet(Net dstNet) { CV_Assert(node_proto.input_size() == 0); CV_Assert(layerParams.blobs.size() == 1); - addConstant(layerParams.name, layerParams.blobs[0], constBlobs, outShapes); - continue; + addConstant(layerParams.name, layerParams.blobs[0]); + return; } else if (layer_type == "LSTM") { @@ -780,11 +864,11 @@ void ONNXImporter::populateNet(Net dstNet) // https://pytorch.org/docs/stable/nn.html#lstm CV_Assert(node_proto.input_size() == 7); - Mat Wx = getBlob(node_proto, constBlobs, 1); - Mat Wh = getBlob(node_proto, constBlobs, 2); - Mat b = getBlob(node_proto, constBlobs, 3); - CV_CheckEQ(countNonZero(getBlob(node_proto, constBlobs, 5)), 0, "Unsupported non zero initial_h"); - CV_CheckEQ(countNonZero(getBlob(node_proto, constBlobs, 6)), 0, "Unsupported non zero initial_c"); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + CV_CheckEQ(countNonZero(getBlob(node_proto, 5)), 0, "Unsupported non zero initial_h"); + CV_CheckEQ(countNonZero(getBlob(node_proto, 6)), 0, "Unsupported non zero initial_c"); b = b.reshape(1, b.size[0]); const int numHidden = lstmParams.get("hidden_size"); @@ -825,7 +909,7 @@ void ONNXImporter::populateNet(Net dstNet) lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name - addLayer(dstNet, lstmParams, node_proto, layer_id, outShapes); + addLayer(lstmParams, node_proto); MatShape lstmShape = outShapes[node_proto.output(0)]; @@ -888,7 +972,7 @@ void ONNXImporter::populateNet(Net dstNet) else if (layer_type == "PRelu") { layerParams.type = "PReLU"; - layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 1)); + layerParams.blobs.push_back(getBlob(node_proto, 1)); } else if (layer_type == "LRN") { @@ -901,8 +985,8 @@ void ONNXImporter::populateNet(Net dstNet) "Expected input, scale, bias"); layerParams.blobs.resize(4); - layerParams.blobs[2] = getBlob(node_proto, constBlobs, 1); // weightData - layerParams.blobs[3] = getBlob(node_proto, constBlobs, 2); // biasData + layerParams.blobs[2] = getBlob(node_proto, 1); // weightData + layerParams.blobs[3] = getBlob(node_proto, 2); // biasData layerParams.set("has_bias", true); layerParams.set("has_weight", true); @@ -920,7 +1004,7 @@ void ONNXImporter::populateNet(Net dstNet) //Create MVN layer int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); //Connect to input - layerId = layer_id.find(node_proto.input(0)); + IterLayerId_t layerId = layer_id.find(node_proto.input(0)); CV_Assert(layerId != layer_id.end()); dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape @@ -941,22 +1025,22 @@ void ONNXImporter::populateNet(Net dstNet) replaceLayerParam(layerParams, "epsilon", "eps"); replaceLayerParam(layerParams, "spatial", "use_global_stats"); - Mat meanData = getBlob(node_proto, constBlobs, 3); - Mat stdData = getBlob(node_proto, constBlobs, 4); + Mat meanData = getBlob(node_proto, 3); + Mat stdData = getBlob(node_proto, 4); layerParams.blobs.push_back(meanData); layerParams.blobs.push_back(stdData); if (!node_proto.input(1).empty()) { layerParams.set("has_weight", true); - layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 1)); // weightData + layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData } else { layerParams.set("has_weight", false); } if (!node_proto.input(2).empty()) { layerParams.set("has_bias", true); - layerParams.blobs.push_back(getBlob(node_proto, constBlobs, 2)); // biasData + layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData } else { layerParams.set("has_bias", false); } @@ -965,7 +1049,7 @@ void ONNXImporter::populateNet(Net dstNet) { CV_Assert(node_proto.input_size() >= 2); layerParams.type = "InnerProduct"; - Mat weights = getBlob(node_proto, constBlobs, 1); + Mat weights = getBlob(node_proto, 1); int ind_num_out = 0; if (layerParams.has("transB") && !layerParams.get("transB")) { transpose(weights, weights); @@ -974,12 +1058,12 @@ void ONNXImporter::populateNet(Net dstNet) layerParams.blobs.push_back(weights); if (node_proto.input_size() == 3) { - Mat bias = getBlob(node_proto, constBlobs, 2); + Mat bias = getBlob(node_proto, 2); layerParams.blobs.push_back(bias); } if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { - Mat inputBuf = getBlob(node_proto, constBlobs, 0); + Mat inputBuf = getBlob(node_proto, 0); LayerParams constParams; constParams.name = node_proto.input(0); @@ -988,7 +1072,7 @@ void ONNXImporter::populateNet(Net dstNet) opencv_onnx::NodeProto proto; proto.add_output(constParams.name); - addLayer(dstNet, constParams, proto, layer_id, outShapes); + addLayer(constParams, proto); } layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); @@ -1005,7 +1089,7 @@ void ONNXImporter::populateNet(Net dstNet) if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) { - Mat blob = getBlob(node_proto, constBlobs, 1); + Mat blob = getBlob(node_proto, 1); secondInpDims = blob.dims; layerParams.blobs.push_back(blob.t()); layerParams.set("num_output", layerParams.blobs[0].size[0]); @@ -1030,7 +1114,7 @@ void ONNXImporter::populateNet(Net dstNet) } if (constId != -1 && haveVariables) { - Mat blob = getBlob(node_proto, constBlobs, constId); + Mat blob = getBlob(node_proto, constId); blob = blob.reshape(1, 1); if (blob.total() == 1) { float coeff = isDiv ? 1.0 / blob.at(0) : blob.at(0); @@ -1071,7 +1155,7 @@ void ONNXImporter::populateNet(Net dstNet) //Create Power layer int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); //Connect to input - layerId = layer_id.find(node_proto.input(1)); + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); CV_Assert(layerId != layer_id.end()); dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape @@ -1086,16 +1170,16 @@ void ONNXImporter::populateNet(Net dstNet) if (!haveVariables) { - Mat inp0 = getBlob(node_proto, constBlobs, 0); - Mat inp1 = getBlob(node_proto, constBlobs, 1); + Mat inp0 = getBlob(node_proto, 0); + Mat inp1 = getBlob(node_proto, 1); if (inp0.size != inp1.size && inp1.total() != 1) CV_Error(Error::StsNotImplemented, "Constant multiply with different shapes"); Mat out = isDiv ? inp0 / inp1 : inp0.mul(inp1); out = out.reshape(1, inp0.dims, inp0.size); out.dims = inp0.dims; // to workaround dims == 1 - addConstant(layerParams.name, out, constBlobs, outShapes); - continue; + addConstant(layerParams.name, out); + return; } } else if (layer_type == "Conv") @@ -1105,7 +1189,7 @@ void ONNXImporter::populateNet(Net dstNet) for (int j = 1; j < node_proto.input_size(); j++) { if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) { - layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j)); + layerParams.blobs.push_back(getBlob(node_proto, j)); } } int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; @@ -1116,7 +1200,7 @@ void ONNXImporter::populateNet(Net dstNet) CV_Assert(node_proto.input_size() >= 2); layerParams.type = "Deconvolution"; for (int j = 1; j < node_proto.input_size(); j++) { - layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j)); + layerParams.blobs.push_back(getBlob(node_proto, j)); } layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); layerParams.set("bias_term", node_proto.input_size() == 3); @@ -1162,11 +1246,11 @@ void ONNXImporter::populateNet(Net dstNet) CV_Assert(node_proto.input_size() == 1); if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { - std::vector inputs(1, getBlob(node_proto, constBlobs, 0)), transposed; + std::vector inputs(1, getBlob(node_proto, 0)), transposed; runLayer(layerParams, inputs, transposed); CV_Assert(transposed.size() == 1); - addConstant(layerParams.name, transposed[0], constBlobs, outShapes); - continue; + addConstant(layerParams.name, transposed[0]); + return; } } else if (layer_type == "Squeeze") @@ -1198,11 +1282,11 @@ void ONNXImporter::populateNet(Net dstNet) if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { - Mat inp = getBlob(node_proto, constBlobs, 0); + Mat inp = getBlob(node_proto, 0); Mat out = inp.reshape(1, outShape); out.dims = outShape.size(); // to workaround dims == 1 - addConstant(layerParams.name, out, constBlobs, outShapes); - continue; + addConstant(layerParams.name, out); + return; } } else if (layer_type == "Flatten") @@ -1210,14 +1294,14 @@ void ONNXImporter::populateNet(Net dstNet) CV_CheckEQ(node_proto.input_size(), 1, ""); if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { - Mat input = getBlob(node_proto, constBlobs, 0); + Mat input = getBlob(node_proto, 0); int axis = clamp(layerParams.get("axis", 1), input.dims); std::vector out_size(&input.size[0], &input.size[0] + axis); out_size.push_back(input.total(axis)); Mat output = input.reshape(1, out_size); - addConstant(layerParams.name, output, constBlobs, outShapes); - continue; + addConstant(layerParams.name, output); + return; } } else if (layer_type == "Unsqueeze") @@ -1227,7 +1311,7 @@ void ONNXImporter::populateNet(Net dstNet) if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { // Constant input. - Mat input = getBlob(node_proto, constBlobs, 0); + Mat input = getBlob(node_proto, 0); std::vector dims; for (int j = 0; j < input.dims; j++) { @@ -1239,8 +1323,8 @@ void ONNXImporter::populateNet(Net dstNet) } Mat out = input.reshape(0, dims); - addConstant(layerParams.name, out, constBlobs, outShapes); - continue; + addConstant(layerParams.name, out); + return; } // Variable input. @@ -1258,24 +1342,25 @@ void ONNXImporter::populateNet(Net dstNet) else if (layer_type == "Expand") { CV_CheckEQ(node_proto.input_size(), 2, ""); - CV_Assert(constBlobs.find(node_proto.input(1)) != constBlobs.end()); - Mat newShapeMat = getBlob(node_proto, constBlobs, 1); + const std::string& input0 = node_proto.input(0); + const std::string& input1 = node_proto.input(1); + Mat newShapeMat = getBlob(input1); MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); MatShape inpShape; - bool haveVariables = constBlobs.find(node_proto.input(0)) == constBlobs.end(); + bool haveVariables = constBlobs.find(input0) == constBlobs.end(); if (haveVariables) { - shapeIt = outShapes.find(node_proto.input(0)); + IterShape_t shapeIt = outShapes.find(input0); CV_Assert(shapeIt != outShapes.end()); inpShape = shapeIt->second; } else { - inpShape = shape(getBlob(node_proto, constBlobs, 0)); + inpShape = shape(getBlob(input0)); } - String srcName = node_proto.input(0); + String srcName = input0; // Unsqueeze and repeat along new axis if (targetShape.size() == inpShape.size() + 1) { @@ -1297,7 +1382,7 @@ void ONNXImporter::populateNet(Net dstNet) opencv_onnx::NodeProto proto; proto.add_input(node_proto.input(0)); proto.add_output(reshapeLp.name); - addLayer(dstNet, reshapeLp, proto, layer_id, outShapes); + addLayer(reshapeLp, proto); srcName = reshapeLp.name; } } @@ -1320,12 +1405,12 @@ void ONNXImporter::populateNet(Net dstNet) if (broadcast_axes.size() != 1) CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); - Mat input = getBlob(node_proto, constBlobs, 0); + Mat input = getBlob(node_proto, 0); input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); output = output.reshape(0, targetShape); - addConstant(layerParams.name, output, constBlobs, outShapes); - continue; + addConstant(layerParams.name, output); + return; } if (broadcast_axes.size() == 2 && @@ -1341,12 +1426,12 @@ void ONNXImporter::populateNet(Net dstNet) opencv_onnx::NodeProto proto; proto.add_output(constParams.name); - addLayer(dstNet, constParams, proto, layer_id, outShapes); + addLayer(constParams, proto); layerParams.type = "Scale"; layerParams.set("bias_term", false); node_proto.set_input(0, constParams.name); - node_proto.set_input(1, shapeIt->first); + node_proto.set_input(1, srcName); } else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) { @@ -1364,7 +1449,7 @@ void ONNXImporter::populateNet(Net dstNet) node_proto.set_input(0, srcName); node_proto.set_output(0, copyLP.name); - addLayer(dstNet, copyLP, node_proto, layer_id, outShapes); + addLayer(copyLP, node_proto); } node_proto.clear_input(); for (int i = 0; i < input_names.size(); i++) @@ -1383,17 +1468,17 @@ void ONNXImporter::populateNet(Net dstNet) CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); if (node_proto.input_size() == 2) { - Mat blob = getBlob(node_proto, constBlobs, 1); + Mat blob = getBlob(node_proto, 1); CV_Assert(blob.type() == CV_32SC1); layerParams.set("dim", DictValue::arrayInt( blob.ptr(), blob.total() )); if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - std::vector inputs(1, getBlob(node_proto, constBlobs, 0)), outputs; + std::vector inputs(1, getBlob(node_proto, 0)), outputs; runLayer(layerParams, inputs, outputs); - addConstant(layerParams.name, outputs[0], constBlobs, outShapes); - continue; + addConstant(layerParams.name, outputs[0]); + return; } } else { @@ -1404,10 +1489,10 @@ void ONNXImporter::populateNet(Net dstNet) } if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - Mat input = getBlob(node_proto, constBlobs, 0); + Mat input = getBlob(node_proto, 0); Mat out = input.reshape(0, dim); - addConstant(layerParams.name, out, constBlobs, outShapes); - continue; + addConstant(layerParams.name, out); + return; } replaceLayerParam(layerParams, "shape", "dim"); } @@ -1420,13 +1505,13 @@ void ONNXImporter::populateNet(Net dstNet) { // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. // We need to shuffle it to begin0, end0, begin1, end1, ... - Mat paddings = getBlob(node_proto, constBlobs, 1).reshape(1, 2); + Mat paddings = getBlob(node_proto, 1).reshape(1, 2); paddings = paddings.t(); layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); if (node_proto.input_size() == 3) { - Mat value = getBlob(node_proto, constBlobs, 2); + Mat value = getBlob(node_proto, 2); layerParams.set("value", value.at(0)); } } @@ -1434,23 +1519,23 @@ void ONNXImporter::populateNet(Net dstNet) else if (layer_type == "Shape") { CV_Assert(node_proto.input_size() == 1); - shapeIt = outShapes.find(node_proto.input(0)); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); CV_Assert(shapeIt != outShapes.end()); - MatShape inpShape = shapeIt->second; + const MatShape& inpShape = shapeIt->second; Mat shapeMat(inpShape.size(), 1, CV_32S); for (int j = 0; j < inpShape.size(); ++j) shapeMat.at(j) = inpShape[j]; shapeMat.dims = 1; - addConstant(layerParams.name, shapeMat, constBlobs, outShapes); - continue; + addConstant(layerParams.name, shapeMat); + return; } else if (layer_type == "Cast") { if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { - Mat blob = getBlob(node_proto, constBlobs, 0); + Mat blob = getBlob(node_proto, 0); int type; switch (layerParams.get("to")) { @@ -1467,8 +1552,8 @@ void ONNXImporter::populateNet(Net dstNet) Mat dst; blob.convertTo(dst, type); dst.dims = blob.dims; - addConstant(layerParams.name, dst, constBlobs, outShapes); - continue; + addConstant(layerParams.name, dst); + return; } else layerParams.type = "Identity"; @@ -1488,24 +1573,24 @@ void ONNXImporter::populateNet(Net dstNet) else fill_value = layerParams.get("value", 0); - MatShape inpShape = getBlob(node_proto, constBlobs, 0); + MatShape inpShape = getBlob(node_proto, 0); for (int i = 0; i < inpShape.size(); i++) CV_CheckGT(inpShape[i], 0, ""); Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); - addConstant(layerParams.name, tensor, constBlobs, outShapes); - continue; + addConstant(layerParams.name, tensor); + return; } else if (layer_type == "Gather") { CV_Assert(node_proto.input_size() == 2); - Mat indexMat = getBlob(node_proto, constBlobs, 1); + Mat indexMat = getBlob(node_proto, 1); CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); int index = indexMat.at(0); int axis = layerParams.get("axis", 0); if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) { - Mat input = getBlob(node_proto, constBlobs, 0); + Mat input = getBlob(node_proto, 0); Mat out; std::vector ranges(input.dims, Range::all()); ranges[axis] = Range(index, index + 1); @@ -1519,12 +1604,12 @@ void ONNXImporter::populateNet(Net dstNet) } else { out.dims = 1; } - addConstant(layerParams.name, out, constBlobs, outShapes); - continue; + addConstant(layerParams.name, out); + return; } else { - shapeIt = outShapes.find(node_proto.input(0)); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); CV_Assert(shapeIt != outShapes.end()); MatShape inpShape = shapeIt->second; @@ -1546,7 +1631,7 @@ void ONNXImporter::populateNet(Net dstNet) opencv_onnx::NodeProto proto; proto.add_input(node_proto.input(0)); proto.add_output(sliceLp.name); - addLayer(dstNet, sliceLp, proto, layer_id, outShapes); + addLayer(sliceLp, proto); inpShape.erase(inpShape.begin() + axis); layerParams.type = "Reshape"; @@ -1577,13 +1662,13 @@ void ONNXImporter::populateNet(Net dstNet) std::vector inputs(node_proto.input_size()), concatenated; for (size_t i = 0; i < inputs.size(); ++i) { - inputs[i] = getBlob(node_proto, constBlobs, i); + inputs[i] = getBlob(node_proto, i); } runLayer(layerParams, inputs, concatenated); CV_Assert(concatenated.size() == 1); - addConstant(layerParams.name, concatenated[0], constBlobs, outShapes); - continue; + addConstant(layerParams.name, concatenated[0]); + return; } } else if (layer_type == "Resize") @@ -1595,7 +1680,7 @@ void ONNXImporter::populateNet(Net dstNet) CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); layerParams.set("align_corners", interp_mode == "align_corners"); - Mat shapes = getBlob(node_proto, constBlobs, node_proto.input_size() - 1); + Mat shapes = getBlob(node_proto, node_proto.input_size() - 1); CV_CheckEQ(shapes.size[0], 4, ""); CV_CheckEQ(shapes.size[1], 1, ""); CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); @@ -1605,7 +1690,7 @@ void ONNXImporter::populateNet(Net dstNet) int width = shapes.at(3); if (node_proto.input_size() == 3) { - shapeIt = outShapes.find(node_proto.input(0)); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); CV_Assert(shapeIt != outShapes.end()); MatShape scales = shapeIt->second; height *= scales[2]; @@ -1656,7 +1741,7 @@ void ONNXImporter::populateNet(Net dstNet) else { // scales as input - Mat scales = getBlob(node_proto, constBlobs, 1); + Mat scales = getBlob(node_proto, 1); CV_Assert(scales.total() == 4); layerParams.set("zoom_factor_y", scales.at(2)); layerParams.set("zoom_factor_x", scales.at(3)); @@ -1673,7 +1758,7 @@ void ONNXImporter::populateNet(Net dstNet) CV_CheckEQ(node_proto.input_size(), 3, ""); if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) { - Mat priors = getBlob(node_proto, constBlobs, 2); + Mat priors = getBlob(node_proto, 2); LayerParams constParams; constParams.name = layerParams.name + "/priors"; @@ -1682,7 +1767,7 @@ void ONNXImporter::populateNet(Net dstNet) opencv_onnx::NodeProto priorsProto; priorsProto.add_output(constParams.name); - addLayer(dstNet, constParams, priorsProto, layer_id, outShapes); + addLayer(constParams, priorsProto); node_proto.set_input(2, constParams.name); } @@ -1691,26 +1776,39 @@ void ONNXImporter::populateNet(Net dstNet) { for (int j = 0; j < node_proto.input_size(); j++) { if (layer_id.find(node_proto.input(j)) == layer_id.end()) - layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j)); + layerParams.blobs.push_back(getBlob(node_proto, j)); } } - addLayer(dstNet, layerParams, node_proto, layer_id, outShapes); + addLayer(layerParams, node_proto); + } + catch (const cv::Exception& e) + { + CV_LOG_ERROR(NULL, "DNN/ONNX: ERROR during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " + << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) + ); + for (int i = 0; i < node_proto.input_size(); i++) + { + CV_LOG_INFO(NULL, " Input[" << i << "] = '" << node_proto.input(i) << "'"); + } + for (int i = 0; i < node_proto.output_size(); i++) + { + CV_LOG_INFO(NULL, " Output[" << i << "] = '" << node_proto.output(i) << "'"); + } + CV_Error(Error::StsError, cv::format("Node [%s]:(%s) parse error: %s", layer_type.c_str(), name.c_str(), e.what())); } } Net readNetFromONNX(const String& onnxFile) { - ONNXImporter onnxImporter(onnxFile.c_str()); Net net; - onnxImporter.populateNet(net); + ONNXImporter onnxImporter(net, onnxFile.c_str()); return net; } Net readNetFromONNX(const char* buffer, size_t sizeBuffer) { - ONNXImporter onnxImporter(buffer, sizeBuffer); Net net; - onnxImporter.populateNet(net); + ONNXImporter onnxImporter(net, buffer, sizeBuffer); return net; } @@ -1721,10 +1819,16 @@ Net readNetFromONNX(const std::vector& buffer) Mat readTensorFromONNX(const String& path) { - opencv_onnx::TensorProto tensor_proto = opencv_onnx::TensorProto(); std::fstream input(path.c_str(), std::ios::in | std::ios::binary); - if (!tensor_proto.ParseFromIstream(&input)) { - CV_Error(Error::StsUnsupportedFormat, "Failed to parse data"); + if (!input) + { + CV_Error(Error::StsBadArg, cv::format("Can't read ONNX file: %s", path.c_str())); + } + + opencv_onnx::TensorProto tensor_proto = opencv_onnx::TensorProto(); + if (!tensor_proto.ParseFromIstream(&input)) + { + CV_Error(Error::StsUnsupportedFormat, cv::format("Failed to parse ONNX data: %s", path.c_str())); } Mat mat = getMatFromTensor(tensor_proto); releaseONNXTensor(tensor_proto); -- 2.7.4