From 9544bd490627ef17ce372966199510091451644d Mon Sep 17 00:00:00 2001 From: =?utf8?q?=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A2=D0=B8=D1=89?= =?utf8?q?=D0=B5=D0=BD=D0=BA=D0=BE/AI=20Tools=20Lab=20/SRR/Staff=20Enginee?= =?utf8?q?r/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Thu, 29 Nov 2018 13:49:30 +0300 Subject: [PATCH] [nnc] ConstantOp, shape inference, model analyze, serialize, first artifact initial versions were implemented. (#2394) New ConstantOp was added: at the moment it's used for input constant tensors like weights, etc. Shape inference, model analyzing, etc, are working enough to be able to create the first artifact. Signed-off-by: Andrew V. Tischenko --- contrib/nnc/core/modelIR/Graph.cpp | 3 + contrib/nnc/core/modelIR/IrDotDumper.cpp | 8 + contrib/nnc/core/modelIR/Operation.cpp | 1 + contrib/nnc/core/modelIR/Shape.cpp | 14 +- contrib/nnc/core/modelIR/ShapeInference.cpp | 7 +- contrib/nnc/include/core/modelIR/Graph.h | 6 + contrib/nnc/include/core/modelIR/IrDotDumper.h | 2 + contrib/nnc/include/core/modelIR/Shape.h | 4 +- contrib/nnc/include/core/modelIR/ShapeInference.h | 1 + .../include/core/modelIR/operations/ConstantOp.h | 41 ++++ .../nnc/include/core/modelIR/operations/PoolOp.h | 5 +- .../core/modelIR/operations/operations.lst.h | 1 + .../passes/acl_soft_backend/AclCppOpGenerator.h | 1 + .../nnc/include/passes/interpreter/Interpreter.h | 1 + .../passes/acl_soft_backend/AclCppOpGenerator.cpp | 6 + contrib/nnc/passes/interpreter/Interpreter.cpp | 6 + contrib/nnc/passes/onnx_frontend/ONNXImporter.cpp | 2 - .../nnc/passes/onnx_frontend/ONNXImporterImpl.cpp | 224 +++++++++++---------- .../nnc/passes/onnx_frontend/ONNXImporterImpl.h | 35 ++-- contrib/nnc/passes/onnx_frontend/ONNXOpCreator.cpp | 136 +++++-------- contrib/nnc/passes/onnx_frontend/ONNXOpCreator.h | 19 +- contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp | 44 ++-- contrib/nnc/passes/soft_backend/ModelAnalyzer.h | 7 +- contrib/nnc/passes/soft_backend/SBSerializer.cpp | 6 + contrib/nnc/passes/soft_backend/SBSerializer.h | 1 + 25 files changed, 325 insertions(+), 256 deletions(-) create mode 100644 contrib/nnc/include/core/modelIR/operations/ConstantOp.h diff --git a/contrib/nnc/core/modelIR/Graph.cpp b/contrib/nnc/core/modelIR/Graph.cpp index aa5981b..4e18187 100644 --- a/contrib/nnc/core/modelIR/Graph.cpp +++ b/contrib/nnc/core/modelIR/Graph.cpp @@ -120,6 +120,9 @@ std::vector Graph::collectInputs() const { for (auto& e : _inputs) { res.emplace_back(e.second); } + for (auto c : _constants) { + res.emplace_back(c); + } return res; } diff --git a/contrib/nnc/core/modelIR/IrDotDumper.cpp b/contrib/nnc/core/modelIR/IrDotDumper.cpp index c1f0986..df6c0ab 100644 --- a/contrib/nnc/core/modelIR/IrDotDumper.cpp +++ b/contrib/nnc/core/modelIR/IrDotDumper.cpp @@ -143,6 +143,14 @@ void IrDotDumper::visit(ops::VariableOp& op) { dotBuilder.updateWithOp(&op, nodeInfo); } +void IrDotDumper::visit(ops::ConstantOp& op) { + auto node_info = DotIrNodeInfo().withType("Constant", op.getName()) + .withInShapes(getInputShapes(op)) + .withOutShapes(getOutputShapes(op)); + + dotBuilder.updateWithOp(&op, node_info); +} + void IrDotDumper::visit(ops::BatchNormOp& op) { auto nodeInfo = DotIrNodeInfo().withType("BatchNorm", op.getName()) .withInShapes(getInputShapes(op)) diff --git a/contrib/nnc/core/modelIR/Operation.cpp b/contrib/nnc/core/modelIR/Operation.cpp index 84a65b0..8936bca 100644 --- a/contrib/nnc/core/modelIR/Operation.cpp +++ b/contrib/nnc/core/modelIR/Operation.cpp @@ -19,6 +19,7 @@ #include "core/modelIR/operations/SoftmaxOp.h" #include "core/modelIR/operations/CappedReluOp.h" #include "core/modelIR/operations/DepthwiseConv2DOp.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/operations/Conv2DOp.h" #include "core/modelIR/operations/Deconv2DOp.h" #include "core/modelIR/operations/PoolOp.h" diff --git a/contrib/nnc/core/modelIR/Shape.cpp b/contrib/nnc/core/modelIR/Shape.cpp index 0ddb7f9..358e3c0 100644 --- a/contrib/nnc/core/modelIR/Shape.cpp +++ b/contrib/nnc/core/modelIR/Shape.cpp @@ -38,14 +38,16 @@ Shape &Shape::resize(int32_t size) return *this; } -int32_t &Shape::dim(int32_t axis) -{ - return _dims.at((axis < 0) ? (_dims.size() + axis) : axis); +int32_t Shape::dim(int32_t axis) const { + auto dim = (axis < 0) ? (_dims.size() + axis) : axis; + assert(dim < _dims.size()); + return _dims.at(dim); } -int32_t Shape::dim(int32_t axis) const -{ - return _dims.at((axis < 0) ? (_dims.size() + axis) : axis); +int32_t& Shape::dim(int32_t axis) { + auto dim = (axis < 0) ? (_dims.size() + axis) : axis; + assert(dim < _dims.size()); + return _dims.at(dim); } int32_t Shape::numElements() const diff --git a/contrib/nnc/core/modelIR/ShapeInference.cpp b/contrib/nnc/core/modelIR/ShapeInference.cpp index 98d98a1..1252404 100644 --- a/contrib/nnc/core/modelIR/ShapeInference.cpp +++ b/contrib/nnc/core/modelIR/ShapeInference.cpp @@ -23,6 +23,7 @@ #include "core/modelIR/operations/SoftmaxOp.h" #include "core/modelIR/operations/CappedReluOp.h" #include "core/modelIR/operations/DepthwiseConv2DOp.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/operations/Conv2DOp.h" #include "core/modelIR/operations/Deconv2DOp.h" #include "core/modelIR/operations/PoolOp.h" @@ -131,11 +132,13 @@ void ShapeInference::visit(ops::Conv2DOp& op) { op.setOutputShape(0, outShape); } -void ShapeInference::visit(ops::VariableOp& op) { - (void)op; +void ShapeInference::visit(ops::VariableOp&) { // No need to do anything for inputs. These should be set by user } +void ShapeInference::visit(ops::ConstantOp&) { +} + void ShapeInference::fillInputShapes(Operation& op) { size_t i = 0; for (auto &in : op.getPrevNodes()) diff --git a/contrib/nnc/include/core/modelIR/Graph.h b/contrib/nnc/include/core/modelIR/Graph.h index 36b48d9..95b10c0 100644 --- a/contrib/nnc/include/core/modelIR/Graph.h +++ b/contrib/nnc/include/core/modelIR/Graph.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "core/modelIR/Operation.h" #include "core/modelIR/operations/VariableOp.h" @@ -81,6 +82,10 @@ class Graph { */ ops::VariableOp* replaceWithInputNode(const Operation* op); + void setConstants(std::set consts) { + _constants = consts; + } + /** * @brief Change graph inputs to nodes with names in newInputs * @param new_inputs names of nodes to be made into input nodes @@ -115,6 +120,7 @@ class Graph { size_t _lastNodeId = 0; std::unordered_map _inputs; std::unordered_map _outputs; + std::set _constants; }; } // namespace mir diff --git a/contrib/nnc/include/core/modelIR/IrDotDumper.h b/contrib/nnc/include/core/modelIR/IrDotDumper.h index 567cc82..829c321 100644 --- a/contrib/nnc/include/core/modelIR/IrDotDumper.h +++ b/contrib/nnc/include/core/modelIR/IrDotDumper.h @@ -21,6 +21,7 @@ #include "core/modelIR/operations/FullyConnectedOp.h" #include "core/modelIR/operations/SoftmaxOp.h" #include "core/modelIR/operations/CappedReluOp.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/operations/DepthwiseConv2DOp.h" #include "core/modelIR/operations/Conv2DOp.h" #include "core/modelIR/operations/Deconv2DOp.h" @@ -54,6 +55,7 @@ namespace mir class IrDotDumper : public IVisitor { public: void visit(ops::ConcatOp& op) override; + void visit(ops::ConstantOp& op) override; void visit(ops::ReluOp& op) override; void visit(ops::Conv2DOp& op) override; void visit(ops::DepthwiseConv2DOp& op) override; diff --git a/contrib/nnc/include/core/modelIR/Shape.h b/contrib/nnc/include/core/modelIR/Shape.h index 6c245e1..e9fcd5d 100644 --- a/contrib/nnc/include/core/modelIR/Shape.h +++ b/contrib/nnc/include/core/modelIR/Shape.h @@ -40,8 +40,8 @@ public: Shape &resize(int32_t size); - int32_t &dim(int32_t axis); - int32_t dim(int32_t axis) const; + int32_t& dim(int32_t axis); + int32_t dim(int32_t axis) const; int32_t numElements() const; diff --git a/contrib/nnc/include/core/modelIR/ShapeInference.h b/contrib/nnc/include/core/modelIR/ShapeInference.h index 17e9f4a..e30746f 100644 --- a/contrib/nnc/include/core/modelIR/ShapeInference.h +++ b/contrib/nnc/include/core/modelIR/ShapeInference.h @@ -29,6 +29,7 @@ namespace mir class ShapeInference : public IVisitor { public: void visit(ops::ConcatOp& op) override; + void visit(ops::ConstantOp& op) override; void visit(ops::Conv2DOp& op) override; void visit(ops::DepthwiseConv2DOp& op) override; void visit(ops::ReluOp& op) override; diff --git a/contrib/nnc/include/core/modelIR/operations/ConstantOp.h b/contrib/nnc/include/core/modelIR/operations/ConstantOp.h new file mode 100644 index 0000000..1ce3458 --- /dev/null +++ b/contrib/nnc/include/core/modelIR/operations/ConstantOp.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _NNC_CORE_IR_MODEL_CONSTANT_H_ +#define _NNC_CORE_IR_MODEL_CONSTANT_H_ + +#include "core/modelIR/Operation.h" + +namespace nnc { +namespace mir { +namespace ops { + +class ConstantOp : public Operation { +public: + explicit ConstantOp(TensorVariant* value) : Operation(Type::constant, {}), + _value(*value) {} + + const TensorVariant &getValue() const {return _value;} + +private: + TensorVariant _value; +}; + +} // namespace ops +} // namespace mir +} // namespace nnc + +#endif //_NNC_CORE_IR_MODEL_CONSTANT_H_ diff --git a/contrib/nnc/include/core/modelIR/operations/PoolOp.h b/contrib/nnc/include/core/modelIR/operations/PoolOp.h index 578f210..6e860aa 100644 --- a/contrib/nnc/include/core/modelIR/operations/PoolOp.h +++ b/contrib/nnc/include/core/modelIR/operations/PoolOp.h @@ -57,7 +57,10 @@ public: int32_t getPadding(int32_t dim) const { return _pads[dim]; } - void setPadding(int32_t dim, int32_t pad) { _pads[dim] = pad; } + void setPadding(int32_t dim, int32_t pad) { + assert(dim < (int32_t)_pads.size()); + _pads[dim] = pad; + } private: PaddingType _padding; diff --git a/contrib/nnc/include/core/modelIR/operations/operations.lst.h b/contrib/nnc/include/core/modelIR/operations/operations.lst.h index c89e716..b1455b6 100644 --- a/contrib/nnc/include/core/modelIR/operations/operations.lst.h +++ b/contrib/nnc/include/core/modelIR/operations/operations.lst.h @@ -27,6 +27,7 @@ HANDLE_OP(fullyConnected, FullyConnectedOp) HANDLE_OP(cappedReLU, CappedReluOp) HANDLE_OP(biasAdd, BiasAddOp) HANDLE_OP(variable, VariableOp) +HANDLE_OP(constant, ConstantOp) HANDLE_OP(ReLU, ReluOp) HANDLE_OP(reshape, ReshapeOp) HANDLE_OP(resizeIm, ResizeOp) diff --git a/contrib/nnc/include/passes/acl_soft_backend/AclCppOpGenerator.h b/contrib/nnc/include/passes/acl_soft_backend/AclCppOpGenerator.h index ff0037c..906bf93 100644 --- a/contrib/nnc/include/passes/acl_soft_backend/AclCppOpGenerator.h +++ b/contrib/nnc/include/passes/acl_soft_backend/AclCppOpGenerator.h @@ -48,6 +48,7 @@ public: * @param op */ void visit(mir::ops::ConcatOp& op) override; + void visit(mir::ops::ConstantOp& op) override; void visit(mir::ops::Conv2DOp& op) override; void visit(mir::ops::DepthwiseConv2DOp& op) override; void visit(mir::ops::SoftmaxOp& op) override; diff --git a/contrib/nnc/include/passes/interpreter/Interpreter.h b/contrib/nnc/include/passes/interpreter/Interpreter.h index de0742a..ed973ee 100644 --- a/contrib/nnc/include/passes/interpreter/Interpreter.h +++ b/contrib/nnc/include/passes/interpreter/Interpreter.h @@ -37,6 +37,7 @@ public: explicit NNInterpreter() = default; void visit(ops::ConcatOp& op) override; + void visit(ops::ConstantOp& op) override; void visit(ops::Conv2DOp& op) override; void visit(ops::DepthwiseConv2DOp& op) override; void visit(ops::ReluOp& op) override; diff --git a/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.cpp b/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.cpp index c8fe372..0e8439c 100644 --- a/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.cpp +++ b/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.cpp @@ -8,6 +8,7 @@ #include "core/modelIR/operations/VariableOp.h" #include "core/modelIR/operations/SoftmaxOp.h" #include "core/modelIR/operations/Conv2DOp.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/operations/ScaleOp.h" #include "core/modelIR/operations/BatchNormOp.h" #include "core/modelIR/operations/DropoutOp.h" @@ -299,6 +300,11 @@ void AclCppOpGenerator::visit(ops::VariableOp& op) { allocate(tensor); } +void AclCppOpGenerator::visit(ops::ConstantOp& op) { + // TODO: NIY + assert(false); +} + void AclCppOpGenerator::visit(ops::ReluOp& op) { genActivation(op, "RELU"); } diff --git a/contrib/nnc/passes/interpreter/Interpreter.cpp b/contrib/nnc/passes/interpreter/Interpreter.cpp index c2e9611..cf0378b 100644 --- a/contrib/nnc/passes/interpreter/Interpreter.cpp +++ b/contrib/nnc/passes/interpreter/Interpreter.cpp @@ -25,6 +25,7 @@ #include "core/modelIR/operations/SoftmaxOp.h" #include "core/modelIR/operations/CappedReluOp.h" #include "core/modelIR/operations/DepthwiseConv2DOp.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/operations/Conv2DOp.h" #include "core/modelIR/operations/Deconv2DOp.h" #include "core/modelIR/operations/PoolOp.h" @@ -76,6 +77,11 @@ void NNInterpreter::visit(ops::VariableOp& op) { var(op.getId()) = {it->second}; } +void NNInterpreter::visit(ops::ConstantOp& op) { + // TODO: NIY + assert(false); +} + std::vector &NNInterpreter::getResult(Operation* op) { auto res = vars.find(op->getId()); if (res != vars.end()) diff --git a/contrib/nnc/passes/onnx_frontend/ONNXImporter.cpp b/contrib/nnc/passes/onnx_frontend/ONNXImporter.cpp index 72f7524..e439f88 100644 --- a/contrib/nnc/passes/onnx_frontend/ONNXImporter.cpp +++ b/contrib/nnc/passes/onnx_frontend/ONNXImporter.cpp @@ -15,14 +15,12 @@ */ #include "option/Options.h" -#include "pass/PassException.h" #include "passes/onnx_frontend/ONNXImporter.h" #include "ONNXImporterImpl.h" namespace nnc { PassData ONNXImporter::run(PassData data) { ONNXImporterImpl importer{cli::inputFile}; - importer.import(); return importer.createIR(); } } // namespace nnc diff --git a/contrib/nnc/passes/onnx_frontend/ONNXImporterImpl.cpp b/contrib/nnc/passes/onnx_frontend/ONNXImporterImpl.cpp index c2c53dd..5d4e65e 100644 --- a/contrib/nnc/passes/onnx_frontend/ONNXImporterImpl.cpp +++ b/contrib/nnc/passes/onnx_frontend/ONNXImporterImpl.cpp @@ -17,47 +17,75 @@ #include #include #include +#include +#include "core/modelIR/IrDotDumper.h" +#include "core/modelIR/ShapeInference.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/Operation.h" +#include "core/modelIR/Shape.h" +#include "core/modelIR/TensorVariant.h" #include "core/modelIR/operations/Conv2DOp.h" #include "core/modelIR/operations/ElementwiseOp.h" #include "core/modelIR/operations/VariableOp.h" -#include "core/modelIR/TensorVariant.h" #include "onnx/onnx_pb.h" #include "onnx/proto_utils.h" #include "passes/common_frontend/model_allocation.h" #include "passes/common_frontend/shape_helper.h" #include "pass/PassException.h" + #include "ONNXImporterImpl.h" #include "ONNXPerfectHash.h" -#include namespace nnc { -void ONNXImporterImpl::import() { - GOOGLE_PROTOBUF_VERIFY_VERSION; +static void collectUnsupportedOps(std::unique_ptr& model) { + std::set problems_op_set; - ModelAllocation ma(_modelFilename); - size_t size = ma.getNumBytes(); - if (!size) - throw PassException("Could not load model: " + _modelFilename + "\n"); - _model.reset(new onnx::ModelProto()); - bool result = onnx::ParseProtoFromBytes(_model.get(), (const char*)ma.getDataPnt(), size); + for (int i = 0; i < model->graph().node_size(); i++) { + auto *onnx_node = &(model->graph().node(i)); + assert(onnx_node->has_op_type()); + auto op_type = onnx_node->op_type().c_str(); + auto *ir_op_type = ONNXPerfectHash::getONNXOpType(op_type, onnx_node->op_type().size()); + + switch (ir_op_type->opCode) { + case ONNXOpCode::opAdd: + case ONNXOpCode::opAveragePool: + case ONNXOpCode::opBatchNormalization: + case ONNXOpCode::opConcat: + case ONNXOpCode::opConv: + case ONNXOpCode::opDropout: + case ONNXOpCode::opGemm: + case ONNXOpCode::opMax: + case ONNXOpCode::opMaxPool: + case ONNXOpCode::opMul: + case ONNXOpCode::opRelu: + case ONNXOpCode::opReshape: + case ONNXOpCode::opScale: + case ONNXOpCode::opSoftmax: + case ONNXOpCode::opSum: + break; + default: + problems_op_set.insert(op_type); + } + } + if (!problems_op_set.empty()) { + std::cerr << "The following operators are not supported:\n"; + for (auto op : problems_op_set) + std::cerr << op << std::endl; + throw PassException("Unsupported operators found"); + } } - static std::shared_ptr createTensor(const onnx::TensorProto *tensor) { - mir::TensorVariant::DTYPE type = mir::TensorVariant::DTYPE::FLOAT; +static std::shared_ptr createTensor(const onnx::TensorProto *tensor, + mir::Shape input_shape) { + assert(tensor); + mir::DTYPE type = mir::DTYPE::FLOAT32; size_t element_size; size_t buffer_size; const char* src_data; - char data[] = "DATA buffer"; - if (tensor == nullptr){ - // It's 'data' input: create special kind of tensor - element_size = sizeof(float); - buffer_size = sizeof(data); - src_data = reinterpret_cast(data); - } else if (tensor->float_data_size() != 0) { + if (tensor->float_data_size() != 0) { element_size = sizeof(float); buffer_size = tensor->float_data_size() * element_size; src_data = reinterpret_cast(tensor->float_data().data()); @@ -66,46 +94,45 @@ void ONNXImporterImpl::import() { element_size = sizeof(double); buffer_size = tensor->double_data_size() * element_size; src_data = reinterpret_cast(tensor->double_data().data()); - std::cerr << "WARNING: We don't support double tensors yet, investigate\n"; + throw PassException("WARNING: We don't support double tensors yet, investigate\n"); } else if (tensor->int32_data_size() != 0) { element_size = sizeof(int32_t); buffer_size = tensor->int32_data_size() * element_size; src_data = reinterpret_cast(tensor->int32_data().data()); - std::cerr << "WARNING: We don't support int32 tensors yet, investigate\n"; + throw PassException("WARNING: We don't support int32 tensors yet, investigate\n"); } else if (tensor->int64_data_size() != 0) { element_size = sizeof(int64_t); buffer_size = tensor->int64_data_size() * element_size; src_data = reinterpret_cast(tensor->int64_data().data()); - std::cerr << "WARNING: We don't support int64 tensors yet, investigate\n"; + throw PassException("WARNING: We don't support int64 tensors yet, investigate\n"); + } else if (tensor->raw_data().size() != 0) { + switch ((tensor->data_type())) { + case onnx::TensorProto_DataType_FLOAT: + element_size = sizeof(float); + buffer_size = tensor->raw_data().size(); + src_data = reinterpret_cast(tensor->raw_data().data()); + break; + default: + throw PassException("Don't support this tensor type yet, investigate"); + } } else { throw PassException("Invalid data in Proto file, investigate"); } - // Create untyped tensor. Note, tensor contents will be *copied* here. - std::shared_ptr tensor_buffer_copy(new char[buffer_size], std::default_delete()); - char* dst_data = tensor_buffer_copy.get(); - memcpy(dst_data, src_data, buffer_size); - - mir::Shape tensor_shape; - if (tensor == nullptr) - tensor_shape = ShapeHelper::createShape(std::vector(), 0); - else - tensor_shape = ShapeHelper::createShape( - tensor->dims(), static_cast(tensor->dims_size())); - - auto mir_tensor = std::make_shared(tensor_shape, tensor_buffer_copy, type, - element_size); - return mir_tensor; + std::shared_ptr shared_buffer (new char[buffer_size], std::default_delete()); + memcpy(shared_buffer.get(), src_data, buffer_size); + return std::make_shared(input_shape, shared_buffer, type, element_size); } void ONNXImporterImpl::createGraphInputs() { - auto graph = _model->graph(); - auto initializer = graph.initializer(); - auto value_info = graph.value_info(); + auto& graph = _model->graph(); + auto& initializer = graph.initializer(); + auto& value_info = graph.value_info(); auto init_size = graph.initializer_size(); auto val_size = graph.value_info_size(); auto inp_size = graph.input_size(); - std::map onnx_tensors; + std::map onnx_tensors; + std::set constants; // Collect all initializers of the given graph for (int i = 0; i < graph.initializer_size(); i++) { @@ -114,76 +141,93 @@ void ONNXImporterImpl::createGraphInputs() { onnx_tensors[tensor.name()] = &tensor; } - for (auto input : graph.input()) { + for (auto& input : graph.input()) { assert(input.has_name()); auto name = input.name(); - // Every VariableOp relates to one graph input if (onnx_tensors.find(name) != onnx_tensors.end()) { const onnx::TensorProto* onnx_tensor = onnx_tensors[name]; - _inputTensors[name] = createTensor(onnx_tensor); mir::Shape input_shape = ShapeHelper::createShape(onnx_tensor->dims(), static_cast(onnx_tensor->dims_size())); + _inputTensors[name] = createTensor(onnx_tensor, input_shape); + auto constant = _graph->create(name, _inputTensors[name].get()); + _tensorNameToPrevMirOp[name] = constant; + constants.insert(constant); + constant->setOutputShape(0, input_shape); } else { - // Here we're dealing with graph input node that's why we're creating VariableOp - _inputTensors[name] = createTensor(nullptr); - // TODO: should we update node with special shape? - assert(input.has_type() && input.type().has_tensor_type() && - input.type().tensor_type().has_shape()); - // Make model IR input shape from onnx input shape + // We're dealing with graph input auto onnx_input_shape = input.type().tensor_type().shape(); std::vector shape_vector(onnx_input_shape.dim_size()); for (int i = 0; i < onnx_input_shape.dim_size(); i++) { assert(onnx_input_shape.dim(i).has_dim_value()); shape_vector[i] = onnx_input_shape.dim(i).dim_value(); } - mir::Shape input_shape (shape_vector); + mir::Shape input_shape(shape_vector); + ShapeHelper::cutOffBatchDim(input_shape); + // TODO: Temporary solution! Assuming that every 4D input will be used for a convolution, + // so we change every 4D input from ONNX NCHW to Model IR HWC (batch is cut off earlier). + // TODO: Implement a more consistent way of handling shapes within the model. + // FIXME: it works for 2D pictures only + if (input_shape.rank() == 3) + input_shape = mir::Shape{input_shape.dim(1), input_shape.dim(2), input_shape.dim(0)}; - // VariableOp supports 1 output only that's why we're using index 0 here + // TODO: Temporary solution! auto node = _graph->create(name, input_shape); _tensorNameToPrevMirOp[name] = node; } } + if (!constants.empty()) + _graph->setConstants(constants); } mir::Graph *ONNXImporterImpl::createIR() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + ModelAllocation ma(_modelFilename); + size_t size = ma.getNumBytes(); + if (!size) + throw PassException("Could not load model: " + _modelFilename + "\n"); + _model.reset(new onnx::ModelProto()); + bool result = onnx::ParseProtoFromBytes(_model.get(), (const char*)ma.getDataPnt(), size); + if (!result) + throw PassException("Could not parse proto file: " + _modelFilename + "\n"); + + collectUnsupportedOps(_model); createGraphInputs(); - std::set problems_op_set; // for all nodes in onnx graph - for (int i = 0; i < _model->graph().node_size(); i++) { - auto* onnx_node = &(_model->graph().node(i)); - assert(onnx_node->has_op_type()); - auto op_type = onnx_node->op_type().c_str(); + for (auto& onnx_node : _model->graph().node()) { + assert(onnx_node.has_op_type()); + auto op_type = onnx_node.op_type().c_str(); // Fill inputs of the given node - std::vector input_nodes(onnx_node->input_size()); - for (int i = 0; i < onnx_node->input_size(); i++) { - auto name = onnx_node->input(i); + std::vector input_nodes(onnx_node.input_size()); + for (int i = 0; i < onnx_node.input_size(); i++) { + auto& name = onnx_node.input(i); assert(_tensorNameToPrevMirOp.find(name) != _tensorNameToPrevMirOp.end()); input_nodes[i] = _tensorNameToPrevMirOp[name]; } std::vector outputs; - auto *onnx_op_type = ONNXPerfectHash::getONNXOpType(op_type, onnx_node->op_type().size()); + auto* onnx_op_type = ONNXPerfectHash::getONNXOpType(op_type, onnx_node.op_type().size()); switch (onnx_op_type->opCode) { //case ONNXOpCode::opIdentity: // TOD: We simply remove the operation because it does nothing. Is it OK? // break; case ONNXOpCode::opConv: - outputs = _opCreator.convertConv2D(input_nodes, onnx_node, &_inputTensors); + outputs = _opCreator.convertConv2D(input_nodes, onnx_node); break; case ONNXOpCode::opAdd: - outputs = _opCreator.convertElementwise(input_nodes, mir::ops::ElementwiseOp::OpType::sum); + outputs = _opCreator.convertElementwise(input_nodes, mir::ops::ElementwiseOp::OpType::add); break; case ONNXOpCode::opGemm: outputs = _opCreator.convertGemm(input_nodes, onnx_node); break; case ONNXOpCode::opSum: - outputs = _opCreator.convertElementwise(input_nodes, mir::ops::ElementwiseOp::OpType::sum); + outputs = _opCreator.convertElementwise(input_nodes, mir::ops::ElementwiseOp::OpType::add); break; case ONNXOpCode::opMul: - outputs = _opCreator.convertElementwise(input_nodes, mir::ops::ElementwiseOp::OpType::prod); + outputs = _opCreator.convertElementwise(input_nodes, mir::ops::ElementwiseOp::OpType::mul); break; case ONNXOpCode::opMax: outputs = _opCreator.convertElementwise(input_nodes, mir::ops::ElementwiseOp::OpType::max); @@ -210,53 +254,27 @@ mir::Graph *ONNXImporterImpl::createIR() { outputs = _opCreator.convertScale(input_nodes, onnx_node); break; case ONNXOpCode::opBatchNormalization: - outputs = _opCreator.convertBatchNorm(input_nodes, onnx_node, &_inputTensors); + outputs = _opCreator.convertBatchNorm(input_nodes, onnx_node, _inputTensors); break; case ONNXOpCode::opDropout: outputs = _opCreator.convertDropout(input_nodes, onnx_node); break; default: - problems_op_set.insert(op_type); - } - if (!outputs.size()) { - // FIXME: it's for debugging only - for (auto name : onnx_node->output()) { - auto node = _graph->create(name, mir::Shape{}); - std::cout << "....Operation '" << op_type << "' was replaced with VariableOp name '" - << name << "'\n"; - outputs.push_back(node); - problems_op_set.insert(onnx_node->op_type()); - } - } else { - for (int i = 0; i < outputs.size(); i++){ - outputs[i]->setName(onnx_node->output(i)); - } - } - // These outputs could be usefull as inputs for following operators - for (auto item : outputs) { - if (_tensorNameToPrevMirOp.find(item->getName()) == _tensorNameToPrevMirOp.end()) { - _tensorNameToPrevMirOp[item->getName()] = item; - } else { - // TODO: Exception??? - std::cerr << "Name duplication: " << item->getName() << std::endl; - } + throw PassException("Invalid ONNXOpCode" + std::to_string((int)onnx_op_type->opCode)); } - std::cout << "\tThe following inputs were used:\n"; - for (auto name: onnx_node->input()) { - std::cout << "\t\t" << name << "\n"; + // Set outputs' names + for (int i = 0; i < outputs.size(); i++){ + outputs[i]->setName(onnx_node.output(i)); + auto result = _tensorNameToPrevMirOp.emplace(outputs[i]->getName(), outputs[i]); + if(!result.second) + throw PassException("Name duplication: " + outputs[i]->getName()); } - if (outputs.size()) - // FIXME: it should be done properly via the given graph outputs - _graphOutputs.assign(outputs.begin(), outputs.end()); - } - if (!problems_op_set.empty()) { - std::string msg("There are the following unsupported operations:\n"); - for (auto op : problems_op_set) - msg.append(op + "\n"); - std::cout << msg; - //throw PassException(msg); + assert (outputs.size()); + // FIXME: it should be done properly via the given graph outputs + _graphOutputs.assign(outputs.begin(), outputs.end()); } // set graph outputs + // TODO: it should be done with onnx graph outputs for (auto& output_idx : _graphOutputs) _graph->markOutput(output_idx); diff --git a/contrib/nnc/passes/onnx_frontend/ONNXImporterImpl.h b/contrib/nnc/passes/onnx_frontend/ONNXImporterImpl.h index ed2e06e..67667fb 100644 --- a/contrib/nnc/passes/onnx_frontend/ONNXImporterImpl.h +++ b/contrib/nnc/passes/onnx_frontend/ONNXImporterImpl.h @@ -31,25 +31,26 @@ namespace nnc { class ONNXImporterImpl : public NNImporter { public: - explicit ONNXImporterImpl(std::string filename) { - _modelFilename = std::move(filename); - _graph = new mir::Graph(); - _opCreator.setMirGraph(_graph); - } - void import() override; - mir::Graph *createIR() override; + explicit ONNXImporterImpl(std::string filename) { + _modelFilename = filename; + _graph = new mir::Graph(); + _opCreator.setMirGraph(_graph); + } + + void import() {}; + mir::Graph *createIR() override; private: - void createGraphInputs(); - // This map maps onnx tensor names to MIR operations/nodes - std::map _tensorNameToPrevMirOp; - // This map keeps named tensors used as graph input initializers. - std::map> _inputTensors; - std::vector _graphOutputs; - std::string _modelFilename; - std::unique_ptr _model; - mir::Graph* _graph; - ONNXOpCreator _opCreator; + void createGraphInputs(); + // This map maps onnx tensor names to MIR operations/nodes + std::map _tensorNameToPrevMirOp; + // This map keeps named tensors used as graph input initializers. + std::map> _inputTensors; + std::vector _graphOutputs; + std::string _modelFilename; + std::unique_ptr _model; + mir::Graph* _graph; + ONNXOpCreator _opCreator; }; } // namespace nnc diff --git a/contrib/nnc/passes/onnx_frontend/ONNXOpCreator.cpp b/contrib/nnc/passes/onnx_frontend/ONNXOpCreator.cpp index 1aad389..9d90c35 100644 --- a/contrib/nnc/passes/onnx_frontend/ONNXOpCreator.cpp +++ b/contrib/nnc/passes/onnx_frontend/ONNXOpCreator.cpp @@ -26,6 +26,7 @@ #include "core/modelIR/operations/BiasAddOp.h" #include "core/modelIR/operations/CappedReluOp.h" #include "core/modelIR/operations/ConcatOp.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/operations/Conv2DOp.h" #include "core/modelIR/operations/DepthwiseConv2DOp.h" #include "core/modelIR/operations/DropoutOp.h" @@ -44,9 +45,9 @@ namespace nnc { using namespace mir; -static const onnx::AttributeProto* findAttribute(const onnx::NodeProto* onnx_node, +static const onnx::AttributeProto* findAttribute(const onnx::NodeProto& onnx_node, std::string name) { - for (auto& att : onnx_node->attribute()) { + for (auto& att : onnx_node.attribute()) { if (!att.name().compare(name)) { return &att; } @@ -54,9 +55,9 @@ static const onnx::AttributeProto* findAttribute(const onnx::NodeProto* onnx_nod return nullptr; } -static std::pair getIntAttribute(const onnx::NodeProto* onnx_node, +static std::pair getIntAttribute(const onnx::NodeProto& onnx_node, std::string name = "axis") { - for (auto att : onnx_node->attribute()) { + for (auto att : onnx_node.attribute()) { if (!att.name().compare(name)) { assert(att.type() == onnx::AttributeProto_AttributeType::AttributeProto_AttributeType_INT); return {true, att.i()}; @@ -65,9 +66,9 @@ static std::pair getIntAttribute(const onnx::NodeProto* onnx_node, return {false, 0}; } -static std::pair getFloatAttribute(const onnx::NodeProto* onnx_node, +static std::pair getFloatAttribute(const onnx::NodeProto& onnx_node, std::string name) { - for (auto att : onnx_node->attribute()) { + for (auto att : onnx_node.attribute()) { if (!att.name().compare(name)) { assert(att.type() == onnx::AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT); return {true, att.f()}; @@ -77,48 +78,30 @@ static std::pair getFloatAttribute(const onnx::NodeProto* onnx_node } static const mir::TensorVariant* createTensor(float data) { - mir::TensorVariant::DTYPE type = mir::TensorVariant::DTYPE::FLOAT; + mir::DTYPE type = mir::DTYPE::FLOAT32; size_t element_size = sizeof(float); size_t buffer_size = sizeof(float); const char* src_data = reinterpret_cast(&data); - auto tensor_buffer_copy = std::shared_ptr(new char[buffer_size]); - char* dst_data = tensor_buffer_copy.get(); - memcpy(dst_data, src_data, buffer_size); - Shape tensor_shape = Shape({1}); - auto mir_tensor = new mir::TensorVariant(tensor_shape, tensor_buffer_copy, type, element_size); - return mir_tensor; -} -static const mir::TensorVariant* createTensor(Shape kernelShape, const int64_t* tensor_data) { - mir::TensorVariant::DTYPE type = mir::TensorVariant::DTYPE::INT; - size_t element_size = sizeof(int64_t); - size_t buffer_size = element_size * kernelShape.rank(); - const char* src_data = reinterpret_cast(tensor_data); - auto tensor_buffer_copy = std::shared_ptr(new char[buffer_size]); - char* dst_data = tensor_buffer_copy.get(); - memcpy(dst_data, src_data, buffer_size); - auto mir_tensor = new mir::TensorVariant(kernelShape, tensor_buffer_copy, type, element_size); + std::shared_ptr shared_buffer (new char[buffer_size], std::default_delete()); + memcpy(shared_buffer.get(), src_data, buffer_size); + Shape tensor_shape = Shape({1}); + auto mir_tensor = new mir::TensorVariant(tensor_shape, shared_buffer, type, element_size); return mir_tensor; } std::vector ONNXOpCreator::convertConv2D(InputOps& inputs, - const onnx::NodeProto* onnx_node, - InputTensors* input_tensors) { + const onnx::NodeProto& onnx_node) { assert(inputs.size() >= 2); - int value; - bool found; - std::tie(found, value) = getIntAttribute(onnx_node, "group"); - int group = found ? value : 1; - auto* kshape = findAttribute(onnx_node, "kernel_shape"); - // FIXME: kernel_shape attribute could miss and in this case it should be inferred from input W - assert(kshape && kshape->ints_size()); auto* strides = findAttribute(onnx_node, "strides"); assert(strides && strides->ints_size()); - Shape kernelShape = ShapeHelper::createShape(kshape->ints(), kshape->ints_size()); - const mir::TensorVariant* kernel_tensor = createTensor(kernelShape, kshape->ints().data()); - Shape stridesShape = ShapeHelper::createShape(strides->ints(), strides->ints_size()); - + Shape onnx_strides_shape = ShapeHelper::createShape(strides->ints(), strides->ints_size()); + // FIXME: it's a hack + Shape strides_shape = {onnx_strides_shape.dim(0), onnx_strides_shape.dim(1), 1}; + auto* in_weights = dynamic_cast(inputs[1]); + assert(in_weights); + auto in_weights_tensor = in_weights->getValue(); // TODO: we don't support padding at the moment auto pad_type = ops::PaddingType::Valid; Operation* input_bias; @@ -127,15 +110,13 @@ std::vector ONNXOpCreator::convertConv2D(InputOps& inputs, inputs.resize(1); std::vector outputs; - outputs = createOp(inputs[0]->getOutput(0), *kernel_tensor, stridesShape, pad_type); + outputs = createOp(inputs[0]->getOutput(0), in_weights_tensor, strides_shape, pad_type); // TODO: there could be bias tensor as inputs[2]. - if (input_bias) - std::cout << "WARNING: We have bias for Convolution\n"; return outputs; } std::vector ONNXOpCreator::convertConcat(InputOps& inputs, - const onnx::NodeProto* onnx_node) { + const onnx::NodeProto& onnx_node) { bool found; int axis; std::tie (found, axis) = getIntAttribute(onnx_node); @@ -148,13 +129,13 @@ std::vector ONNXOpCreator::convertConcat(InputOps& inputs, } std::vector ONNXOpCreator::convertPool(InputOps& inputs, ONNXOpCode op_code, - const onnx::NodeProto* onnx_node) { + const onnx::NodeProto& onnx_node) { auto* kshape = findAttribute(onnx_node, "kernel_shape"); assert(kshape && kshape->ints_size()); auto* strides = findAttribute(onnx_node, "strides"); assert(strides && strides->ints_size()); - Shape kernel_shape = ShapeHelper::createShape(kshape->ints(), kshape->ints_size()); - Shape strides_shape = ShapeHelper::createShape(strides->ints(), strides->ints_size()); + Shape onnx_kernel_shape = ShapeHelper::createShape(kshape->ints(), kshape->ints_size()); + Shape onnx_strides_shape = ShapeHelper::createShape(strides->ints(), strides->ints_size()); ops::PoolOp::BorderType border_type; ops::PoolOp::PoolingType pool_type; @@ -172,14 +153,19 @@ std::vector ONNXOpCreator::convertPool(InputOps& inputs, ONNXOpCode default: assert(false); } + // FIXME: it's a hack to be compatible with current implementation of PoolOp - we expect + // 3 dims for 2D picture + Shape window_shape ({onnx_kernel_shape.dim(0), onnx_kernel_shape.dim(1), 1}); + // FIXME: it's another hack identical to the above one + Shape strides_shape ({onnx_strides_shape.dim(0), onnx_strides_shape.dim(1), 1}); // TODO: ONNX has more parameters for pooling. We should use them. - auto pooling = createOp(inputs[0]->getOutput(0), kernel_shape, strides_shape, pool_type, + auto pooling = createOp(inputs[0]->getOutput(0), window_shape, strides_shape, pool_type, pad_type, border_type); return pooling; } std::vector ONNXOpCreator::convertSoftmax(InputOps& inputs, - const onnx::NodeProto* onnx_node) { + const onnx::NodeProto& onnx_node) { int axis; bool found; std::tie (found, axis) = getIntAttribute(onnx_node); @@ -188,7 +174,8 @@ std::vector ONNXOpCreator::convertSoftmax(InputOps& inputs, } std::vector ONNXOpCreator::convertReshape(Operation* inputData, Shape outputShape) { - return createOp(inputData->getOutput(0), outputShape); + auto outputs = createOp(inputData->getOutput(0), outputShape); + return outputs; } std::vector ONNXOpCreator::convertRelu(InputOps& inputs) { @@ -205,8 +192,8 @@ std::vector ONNXOpCreator::convertElementwise(InputOps& inputs, } std::vector ONNXOpCreator::convertBatchNorm(InputOps& inputs, - const onnx::NodeProto* onnx_node, - InputTensors* input_tensors) { + const onnx::NodeProto& onnx_node, + InputTensors& input_tensors) { bool found; float value; @@ -218,12 +205,12 @@ std::vector ONNXOpCreator::convertBatchNorm(InputOps& inputs, //std::tie(found, value) = getFloatAttribute(onnx_node, "spatial"); float scale_factor = 0.0f; // Scale tensor - assert(input_tensors->find(inputs[1]->getName()) != input_tensors->end()); - auto ptensor = input_tensors->at(inputs[1]->getName()); - Tensor nnc_scale (*ptensor.get()); + assert(input_tensors.find(inputs[1]->getName()) != input_tensors.end()); + auto ptensor = input_tensors.at(inputs[1]->getName()); + Tensor nnc_scale (*ptensor); // Bias tensor - assert(input_tensors->find(inputs[2]->getName()) != input_tensors->end()); - auto nnc_bias = input_tensors->at(inputs[2]->getName()); + assert(input_tensors.find(inputs[2]->getName()) != input_tensors.end()); + auto nnc_bias = input_tensors.at(inputs[2]->getName()); // TODO: there are 2 training tensors in the inputs inputs.resize(1); @@ -240,7 +227,7 @@ std::vector ONNXOpCreator::convertBatchNorm(InputOps& inputs, } std::vector ONNXOpCreator::convertDropout(InputOps& inputs, - const onnx::NodeProto* onnx_node) { + const onnx::NodeProto& onnx_node) { bool found; float value; std::tie(found, value) = getFloatAttribute(onnx_node, "ratio"); @@ -249,7 +236,7 @@ std::vector ONNXOpCreator::convertDropout(InputOps& inputs, } std::vector ONNXOpCreator::convertScale(InputOps& inputs, - const onnx::NodeProto* onnx_node) { + const onnx::NodeProto& onnx_node) { bool found; float value; std::tie(found, value) = getFloatAttribute(onnx_node, "scale"); @@ -259,41 +246,8 @@ std::vector ONNXOpCreator::convertScale(InputOps& inputs, } std::vector ONNXOpCreator::convertGemm(InputOps& inputs, - const onnx::NodeProto* onnx_node) { - bool found; - float value; - int ivalue; - - std::tie(found, value) = getFloatAttribute(onnx_node, "alpha"); - float alpha = found ? value : 1.0; - std::tie(found, value) = getFloatAttribute(onnx_node, "beta"); - float beta = found ? value : 1.0; - // A' = transpose(A) if transA else A - // B' = transpose(B) if transB else B - std::tie(found, ivalue) = getFloatAttribute(onnx_node, "tranceA"); - int transA = found ? value : 0; - std::tie(found, ivalue) = getFloatAttribute(onnx_node, "tranceB"); - int transB = found ? value : 0; - - // FIXME: we don't support transpoase at the moment - //std::shared_ptr to_tranpose = std::make_shared(tensor); - //std::shared_ptr transposed = transposeTensor(to_tranpose); - - // keep tensor C for later usage - auto tmp = inputs; - - // Compute Y = alpha * A' * B' + beta * C - inputs.resize(1); // tensor A only - auto result = createOp(inputs[0]->getOutput(0), *createTensor(alpha)); - assert (result.size() == 1); - result.push_back(inputs[1]); // add second operand of multiplication - result = convertElementwise(result, mir::ops::ElementwiseOp::OpType::prod); // multiplay - assert (result.size() == 1); - inputs[0] = tmp[2]; // tensor C as input - auto beta_C = createOp(inputs[0]->getOutput(0), *createTensor(beta)); - result.push_back(beta_C[0]); // the final addition - result = convertElementwise(result, mir::ops::ElementwiseOp::OpType::sum); - return result; + const onnx::NodeProto& onnx_node) { + // TODO: NIY + return inputs; } - } // namespace nnc diff --git a/contrib/nnc/passes/onnx_frontend/ONNXOpCreator.h b/contrib/nnc/passes/onnx_frontend/ONNXOpCreator.h index 0feb195..cf4268b 100644 --- a/contrib/nnc/passes/onnx_frontend/ONNXOpCreator.h +++ b/contrib/nnc/passes/onnx_frontend/ONNXOpCreator.h @@ -38,21 +38,20 @@ public: ONNXOpCreator() = default; void setMirGraph(mir::Graph* g) {_graph = g;}; - std::vector convertConv2D(InputOps& inputs, const onnx::NodeProto* node, - InputTensors* input_tensors); - std::vector convertConcat(InputOps& inputs, const onnx::NodeProto* onnx_node); + std::vector convertConv2D(InputOps& inputs, const onnx::NodeProto& node); + std::vector convertConcat(InputOps& inputs, const onnx::NodeProto& onnx_node); std::vector convertPool(InputOps& inputs, ONNXOpCode op_code, - const onnx::NodeProto* onnx_node); - std::vector convertSoftmax(InputOps& inputs, const onnx::NodeProto* onnx_node); + const onnx::NodeProto& onnx_node); + std::vector convertSoftmax(InputOps& inputs, const onnx::NodeProto& onnx_node); std::vector convertReshape(mir::Operation* input_data, mir::Shape output_shape); std::vector convertRelu(InputOps& inputs); std::vector convertElementwise(InputOps& inputs, mir::ops::ElementwiseOp::OpType op_type); - std::vector convertScale(InputOps& inputs, const onnx::NodeProto* node); - std::vector convertBatchNorm(InputOps& inputs, const onnx::NodeProto* node, - InputTensors* input_tensors); - std::vector convertDropout(InputOps& inputs, const onnx::NodeProto* onnx_node); - std::vector convertGemm(InputOps& inputs, const onnx::NodeProto* onnx_node); + std::vector convertScale(InputOps& inputs, const onnx::NodeProto& node); + std::vector convertBatchNorm(InputOps& inputs, const onnx::NodeProto& node, + InputTensors& input_tensors); + std::vector convertDropout(InputOps& inputs, const onnx::NodeProto& onnx_node); + std::vector convertGemm(InputOps& inputs, const onnx::NodeProto& onnx_node); private: template diff --git a/contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp b/contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp index cf0c795..c475fbe 100644 --- a/contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp +++ b/contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp @@ -25,6 +25,7 @@ #include "core/modelIR/Graph.h" #include "core/modelIR/operations/ConcatOp.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/operations/Conv2DOp.h" #include "core/modelIR/operations/Deconv2DOp.h" #include "core/modelIR/operations/DepthwiseConv2DOp.h" @@ -51,29 +52,29 @@ namespace nnc using namespace nnc::mir; -void ModelAnalyzer::addOpDescr(Operation* op, const string& opName) -{ +void ModelAnalyzer::addOpDescr(Operation* op, const string& opName) { OpDescr::Type type = OpDescr::Type::ORDINARY; vector nodeOutputs; const std::string &name = op->getName(); size_t nodeTid = INVALID_TENSOR_ID; - if (op->getPrevNodes().empty()) - { - // process input op - Shape inputShape = op->getOutputShape(0); - nodeTid = allocateTensor(name, TensorDescription::Type::IN, &inputShape); - _inputs.push_back(nodeTid); - type = OpDescr::Type::IN; - } - else if (!name.empty()) - { + if (op->getPrevNodes().empty()) { + if (auto* p2const = dynamic_cast(op)) { + type = OpDescr::Type::CONSTANT; + auto* shape = const_cast (&p2const->getOutputShape(0)); + nodeTid = allocateTensor(name, TensorDescription::Type::CONSTANT, shape); + } else { + // process input op + Shape inputShape = op->getOutputShape(0); + nodeTid = allocateTensor(name, TensorDescription::Type::IN, &inputShape); + type = OpDescr::Type::IN; + _inputs.push_back(nodeTid); + } + } else if (!name.empty()) { // process output op nodeTid = allocateTensor(name, TensorDescription::Type::OUT); _named_tensors.push_back(nodeTid); type = OpDescr::Type::OUT; - } - else - { + } else { // process ordinary op nodeTid = allocateTensor(); } @@ -81,15 +82,13 @@ void ModelAnalyzer::addOpDescr(Operation* op, const string& opName) nodeOutputs.push_back(nodeTid); // process op outputs // consider op as output if it has no consumers - if (op->getNextNodes().empty()) - { + if (op->getNextNodes().empty() && (type != OpDescr::Type::CONSTANT)) { assert(type == OpDescr::Type::OUT); _outputs.push_back(nodeTid); } // process op inputs vector nodeInputs; - for (const IODescriptor &d: op->getPrevNodes()) - { + for (const IODescriptor &d: op->getPrevNodes()) { size_t idx = d.index; Operation *op = d.op; assert(_opToDescr.find(op) != _opToDescr.end()); @@ -130,7 +129,7 @@ void ModelAnalyzer::analyze(const mir::Graph* g) { // Walk all network inputs for (Operation* in : g->collectInputs()) { - assert(dynamic_cast(in)); + assert(dynamic_cast(in) || dynamic_cast(in)); if (!visited.count(in)) { visited.insert(in); s.push({in, 0}); @@ -217,6 +216,11 @@ void ModelAnalyzer::visit(ops::VariableOp& op) { addOpDescr(&op, "in"); } +void ModelAnalyzer::visit(ops::ConstantOp& op) { + assert(op.getPrevNodes().empty()); + addOpDescr(&op, "constant"); +} + void ModelAnalyzer::visit(ops::ReluOp& op) { addOpDescr(&op, "relu"); } diff --git a/contrib/nnc/passes/soft_backend/ModelAnalyzer.h b/contrib/nnc/passes/soft_backend/ModelAnalyzer.h index 79cd275..b528011 100644 --- a/contrib/nnc/passes/soft_backend/ModelAnalyzer.h +++ b/contrib/nnc/passes/soft_backend/ModelAnalyzer.h @@ -45,7 +45,8 @@ struct TensorDescription { enum class Type { IN, OUT, - ORDINARY + ORDINARY, + CONSTANT }; size_t _id; Type _type; @@ -61,7 +62,8 @@ struct OpDescr { enum class Type { IN, OUT, - ORDINARY + ORDINARY, + CONSTANT }; Type _type; @@ -88,6 +90,7 @@ public: void analyze(const mir::Graph* g); void visit(mir::ops::ConcatOp& op) override; + void visit(mir::ops::ConstantOp& op) override; void visit(mir::ops::Conv2DOp& op) override; void visit(mir::ops::DepthwiseConv2DOp& op) override; void visit(mir::ops::SoftmaxOp& op) override; diff --git a/contrib/nnc/passes/soft_backend/SBSerializer.cpp b/contrib/nnc/passes/soft_backend/SBSerializer.cpp index 7b51dd5..9391d0f 100644 --- a/contrib/nnc/passes/soft_backend/SBSerializer.cpp +++ b/contrib/nnc/passes/soft_backend/SBSerializer.cpp @@ -21,6 +21,7 @@ #include "CommonData.def" #include "core/modelIR/operations/ConcatOp.h" +#include "core/modelIR/operations/ConstantOp.h" #include "core/modelIR/operations/Conv2DOp.h" #include "core/modelIR/operations/Deconv2DOp.h" #include "core/modelIR/operations/DepthwiseConv2DOp.h" @@ -244,6 +245,11 @@ void Serializer::visit(ops::VariableOp& op) { // no parameters to dump } +void Serializer::visit(ops::ConstantOp& op) { + _curOp->_paramStartOffset = _buffer.size(); + serializeTensor(op.getValue()); +} + void Serializer::visit(ops::ReluOp& op) { _curOp->_paramStartOffset = _buffer.size(); // no parameters to dump diff --git a/contrib/nnc/passes/soft_backend/SBSerializer.h b/contrib/nnc/passes/soft_backend/SBSerializer.h index e1c3c62..6c44fde 100644 --- a/contrib/nnc/passes/soft_backend/SBSerializer.h +++ b/contrib/nnc/passes/soft_backend/SBSerializer.h @@ -42,6 +42,7 @@ class Serializer: public mir::IVisitor { public: void visit(mir::ops::ConcatOp& op) override; + void visit(mir::ops::ConstantOp& op) override; void visit(mir::ops::Conv2DOp& op) override; void visit(mir::ops::DepthwiseConv2DOp& op) override; void visit(mir::ops::SoftmaxOp& op) override; -- 2.7.4