From a1f71acd8e317bbea809e49f403e9181321a1c6f Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=9E=A5=EC=A7=80=EC=84=AD/On-Device=20Lab=28SR=29/Enginee?= =?utf8?q?r/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Fri, 25 Oct 2019 19:29:56 +0900 Subject: [PATCH] Introduce Pack op (#8355) * Introduce Pack op This commit introduces Pack op into frontend. - Introduce PackNode into neurun - Introduce Pack op into frontend Signed-off-by: jiseob.jang * Add NYI runtime_error for pack op of base_loader This commit adds NYI runtime_error for pack op of base_loader. Signed-off-by: jiseob.jang --- .../neurun/core/include/model/Operations.Include.h | 1 + runtimes/neurun/core/include/model/Operations.lst | 1 + .../neurun/core/include/model/operation/PackNode.h | 53 ++++++++++++++++++++++ .../neurun/core/src/compiler/OperationValidator.cc | 25 ++++++++++ .../neurun/core/src/compiler/OperationValidator.h | 1 + runtimes/neurun/core/src/graph/dumper/Dumper.cc | 15 ++++++ runtimes/neurun/core/src/graph/dumper/Dumper.h | 1 + .../neurun/core/src/model/operation/PackNode.cc | 33 ++++++++++++++ runtimes/neurun/frontend/base_loader/base_loader.h | 23 ++++++++++ .../frontend/nnapi/wrapper/OperationFactory.cc | 20 ++++++++ 10 files changed, 173 insertions(+) create mode 100644 runtimes/neurun/core/include/model/operation/PackNode.h create mode 100644 runtimes/neurun/core/src/model/operation/PackNode.cc diff --git a/runtimes/neurun/core/include/model/Operations.Include.h b/runtimes/neurun/core/include/model/Operations.Include.h index 0f69194..e271cab 100644 --- a/runtimes/neurun/core/include/model/Operations.Include.h +++ b/runtimes/neurun/core/include/model/Operations.Include.h @@ -70,6 +70,7 @@ #include "operation/MeanNode.h" #include "operation/LocalResponseNormalizationNode.h" #include "operation/DepthToSpaceNode.h" +#include "operation/PackNode.h" #include "operation/ReduceMinNode.h" #include "operation/SplitNode.h" #include "operation/UnpackNode.h" diff --git a/runtimes/neurun/core/include/model/Operations.lst b/runtimes/neurun/core/include/model/Operations.lst index 15c4792..951901f 100644 --- a/runtimes/neurun/core/include/model/Operations.lst +++ b/runtimes/neurun/core/include/model/Operations.lst @@ -74,6 +74,7 @@ OP(DequantizeNode , true) OP(MeanNode , true) OP(LocalResponseNormalizationNode , true) OP(DepthToSpaceNode , true) +OP(PackNode , true) OP(ReduceMinNode , true) OP(SplitNode , true) OP(UnpackNode , true) diff --git a/runtimes/neurun/core/include/model/operation/PackNode.h b/runtimes/neurun/core/include/model/operation/PackNode.h new file mode 100644 index 0000000..ef1c10f --- /dev/null +++ b/runtimes/neurun/core/include/model/operation/PackNode.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __NEURUN_MODEL_OPERATION_PACK_NODE_H__ +#define __NEURUN_MODEL_OPERATION_PACK_NODE_H__ + +#include "model/Operation.h" + +namespace neurun +{ +namespace model +{ +namespace operation +{ +class PackNode : public model::Operation +{ +public: + struct Param + { + int32_t num; + int32_t axis; + }; + +public: + PackNode(const OperandIndexSequence &inputs, const OperandIndexSequence &outputs, + const Param ¶m); + +public: + void accept(OperationVisitor &v) const override; + std::string getName() const override { return "Pack"; } + +public: + const Param ¶m() const { return _param; } + +private: + Param _param; +}; +} // namespace operation +} // namespace model +} // namespace neurun +#endif // __NEURUN_MODEL_OPERATION_PACK_NODE_H__ diff --git a/runtimes/neurun/core/src/compiler/OperationValidator.cc b/runtimes/neurun/core/src/compiler/OperationValidator.cc index 0f72f36..1c3a2de 100644 --- a/runtimes/neurun/core/src/compiler/OperationValidator.cc +++ b/runtimes/neurun/core/src/compiler/OperationValidator.cc @@ -608,6 +608,31 @@ void OperationValidator::visit(const model::operation::DepthToSpaceNode &node) } } +void OperationValidator::visit(const model::operation::PackNode &node) +{ + const auto output_index{node.getOutputs().at(0)}; + const auto num{node.param().num}; + const auto axis{node.param().axis}; + + const auto &output_shape = _ctx.at(output_index).shape(); + const auto output_rank = static_cast(output_shape.rank()); + + const auto input1_index{node.getInputs().at(0)}; + const auto input_shape = _ctx.at(input1_index).shape(); + + UNUSED_RELEASE(num); + UNUSED_RELEASE(axis); + UNUSED_RELEASE(output_rank); + + assert(num == static_cast(node.getInputs().size())); + assert(axis >= -output_rank && axis < output_rank); + for (const auto &index : node.getInputs()) + { + UNUSED_RELEASE(index); + assert(input_shape == _ctx.at(index).shape()); + } +} + void OperationValidator::visit(const model::operation::ReduceMinNode &node) { const auto ofm_index{node.getOutputs().at(0)}; diff --git a/runtimes/neurun/core/src/compiler/OperationValidator.h b/runtimes/neurun/core/src/compiler/OperationValidator.h index c0b6356..6553d8d 100644 --- a/runtimes/neurun/core/src/compiler/OperationValidator.h +++ b/runtimes/neurun/core/src/compiler/OperationValidator.h @@ -63,6 +63,7 @@ public: void visit(const model::operation::DequantizeNode &node) override; void visit(const model::operation::MeanNode &node) override; void visit(const model::operation::DepthToSpaceNode &node) override; + void visit(const model::operation::PackNode &node) override; void visit(const model::operation::ReduceMinNode &node) override; void visit(const model::operation::LSTMNode &node) override; void visit(const model::operation::UnpackNode &node) override; diff --git a/runtimes/neurun/core/src/graph/dumper/Dumper.cc b/runtimes/neurun/core/src/graph/dumper/Dumper.cc index 315e2ce..156e005 100644 --- a/runtimes/neurun/core/src/graph/dumper/Dumper.cc +++ b/runtimes/neurun/core/src/graph/dumper/Dumper.cc @@ -338,6 +338,21 @@ void Dumper::visit(const NegNode &node) VERBOSE(LIR) << " - Output : Output(" << node.getOutputs().at(0).value() << ")" << std::endl; } +void Dumper::visit(const model::operation::PackNode &node) +{ + VERBOSE(LIR) << "* Pack" << std::endl; + std::string inputs; + const auto &input_indices = node.getInputs(); + for (auto it = std::begin(input_indices); it != std::end(input_indices); ++it) + { + inputs += std::to_string(it->value()); + if (std::next(it) != std::end(input_indices)) + inputs += ", "; + } + VERBOSE(LIR) << " - Inputs : Inputs(" << inputs << ")" << std::endl; + VERBOSE(LIR) << " - Output : Output(" << node.getOutputs().at(0).value() << ")" << std::endl; +} + void Dumper::visit(const PermuteNode &node) { std::string permute_type = "Unknown"; diff --git a/runtimes/neurun/core/src/graph/dumper/Dumper.h b/runtimes/neurun/core/src/graph/dumper/Dumper.h index 882108a..a796f09 100644 --- a/runtimes/neurun/core/src/graph/dumper/Dumper.h +++ b/runtimes/neurun/core/src/graph/dumper/Dumper.h @@ -62,6 +62,7 @@ public: void visit(const model::operation::MeanNode &) override; void visit(const model::operation::MulNode &) override; void visit(const model::operation::NegNode &) override; + void visit(const model::operation::PackNode &) override; void visit(const model::operation::PermuteNode &node) override; void visit(const model::operation::PReLUNode &) override; void visit(const model::operation::ReduceMaxNode &) override; diff --git a/runtimes/neurun/core/src/model/operation/PackNode.cc b/runtimes/neurun/core/src/model/operation/PackNode.cc new file mode 100644 index 0000000..84ff6db --- /dev/null +++ b/runtimes/neurun/core/src/model/operation/PackNode.cc @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "model/operation/PackNode.h" +#include "model/OperationVisitor.h" + +namespace neurun +{ +namespace model +{ +namespace operation +{ +void PackNode::accept(OperationVisitor &v) const { v.visit(*this); } +PackNode::PackNode(const OperandIndexSequence &inputs, const OperandIndexSequence &outputs, + const Param ¶m) + : model::Operation{OperandConstraint::createAtLeast(3u), inputs, outputs}, _param{param} +{ +} +} // namespace operation +} // namespace model +} // namespace neurun diff --git a/runtimes/neurun/frontend/base_loader/base_loader.h b/runtimes/neurun/frontend/base_loader/base_loader.h index 2d8ccc4..d2e6124 100644 --- a/runtimes/neurun/frontend/base_loader/base_loader.h +++ b/runtimes/neurun/frontend/base_loader/base_loader.h @@ -95,6 +95,7 @@ protected: void loadSub(const Operator *op); void loadMul(const Operator *op); void loadDiv(const Operator *op); + void loadPack(const Operator *op); void loadRelu(const Operator *op); void loadRelu6(const Operator *op); void loadResizeBilinear(const Operator *op); @@ -508,6 +509,25 @@ void BaseLoader::loadDiv(const Operator *op) } template +void BaseLoader::loadPack(const Operator *op) +{ + // This runtime_error will be removed if the one of backend supports this operation + throw std::runtime_error("NYI"); + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::PackNode::Param param; + const auto *options = op->builtin_options_as_PackOptions(); + param.num = options->values_count(); + param.axis = options->axis(); + + std::unique_ptr new_op(new model::operation::PackNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +template void BaseLoader::loadRelu(const Operator *op) { model::OperandIndexSequence inputs; @@ -785,6 +805,9 @@ void BaseLoader::loadOperation(const Operator *op) case BuiltinOperator::BuiltinOperator_DIV: loadDiv(op); return; + case BuiltinOperator::BuiltinOperator_PACK: + loadPack(op); + return; case BuiltinOperator::BuiltinOperator_RELU: loadRelu(op); return; diff --git a/runtimes/neurun/frontend/nnapi/wrapper/OperationFactory.cc b/runtimes/neurun/frontend/nnapi/wrapper/OperationFactory.cc index 276e8ce..77bf705 100644 --- a/runtimes/neurun/frontend/nnapi/wrapper/OperationFactory.cc +++ b/runtimes/neurun/frontend/nnapi/wrapper/OperationFactory.cc @@ -1479,6 +1479,26 @@ OperationFactory::OperationFactory() return new operation::DepthToSpaceNode{inputs, outputs, param}; }; + _map[ANEURALNETWORKS_PACK_EX] = [](const OperationFactory::Param &init_param, + neurun::model::Operands &operands) { + assert(init_param.input_count >= 3 && init_param.output_count == 1); + + OperandIndexSequence outputs{init_param.outputs[0]}; + OperandIndexSequence inputs; + for (uint32_t n = 0; n < init_param.input_count - 2; ++n) + { + inputs.append(OperandIndex{init_param.inputs[n]}); + } + + operation::PackNode::Param param; + const auto num_index = OperandIndex{init_param.inputs[init_param.input_count - 2]}; + const auto axis_index = OperandIndex{init_param.inputs[init_param.input_count - 1]}; + param.num = operands.at(num_index).asScalar(); + param.axis = operands.at(axis_index).asScalar(); + + return new operation::PackNode{inputs, outputs, param}; + }; + _map[ANEURALNETWORKS_REDUCE_MIN_EX] = [](const OperationFactory::Param &init_param, neurun::model::Operands &) { assert(init_param.input_count == 2 && init_param.output_count == 1); -- 2.7.4