From 7d2854809768cd83810e7ff964d4e9583f2f82df Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=9D=B4=ED=95=9C=EC=A2=85/On-Device=20Lab=28SR=29/Enginee?= =?utf8?q?r/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Fri, 11 Jan 2019 09:45:01 +0900 Subject: [PATCH] [neurun] Extract Operand LowerInfo from Node (#4184) * [neurun] Extract Operand LowerInfo from Node As `model::operand::Node` is pure node info so LowerInfo should not be Node class. `graph::Graph` holds them instead. Signed-off-by: Hanjoung Lee * Do not pass Graph object for DotOperandInfo Pass only `LowerInfo` instead Signed-off-by: Hanjoung Lee --- runtimes/neurun/src/compiler/Compiler.cc | 2 +- .../neurun/src/compiler/ConstantInitializer.cc | 2 +- runtimes/neurun/src/compiler/SubTensorAnalyzer.cc | 4 ++- runtimes/neurun/src/compiler/SubTensorAnalyzer.h | 4 ++- runtimes/neurun/src/compiler/operand/Context.h | 1 - runtimes/neurun/src/dumper/dot/DotDumper.cc | 6 +++-- runtimes/neurun/src/dumper/dot/DotOperandInfo.cc | 26 +++++++++---------- runtimes/neurun/src/dumper/dot/DotOperandInfo.h | 12 ++++++++- runtimes/neurun/src/frontend/execution.cc | 14 +++++----- runtimes/neurun/src/graph/Graph.cc | 30 +++++++++++++++++++--- runtimes/neurun/src/graph/Graph.h | 6 +++++ .../src/graph/pass/PermutationEliminationPass.cc | 8 +++--- .../src/graph/pass/PermutationInsertionPass.cc | 13 +++++----- runtimes/neurun/src/linear/Linear.cc | 12 +++++---- runtimes/neurun/src/model/operand/Object.cc | 9 ------- runtimes/neurun/src/model/operand/Object.h | 4 --- 16 files changed, 93 insertions(+), 60 deletions(-) diff --git a/runtimes/neurun/src/compiler/Compiler.cc b/runtimes/neurun/src/compiler/Compiler.cc index 92ec69a..af6ead5 100644 --- a/runtimes/neurun/src/compiler/Compiler.cc +++ b/runtimes/neurun/src/compiler/Compiler.cc @@ -84,7 +84,7 @@ void Compiler::compile(void) // finalize: generate tensor using subtensor info, then execute stage // Generated SubTensorInfo is in operand(Object) // for easy pass SubTensorInfo to plan builder and tensor builder - linear->accept(SubTensorAnalyzer{graph.operands()}); + linear->accept(SubTensorAnalyzer{graph}); /********************************************************** * Backend dependent analysis & optimization phase finished diff --git a/runtimes/neurun/src/compiler/ConstantInitializer.cc b/runtimes/neurun/src/compiler/ConstantInitializer.cc index 5243001..64c811d 100644 --- a/runtimes/neurun/src/compiler/ConstantInitializer.cc +++ b/runtimes/neurun/src/compiler/ConstantInitializer.cc @@ -56,7 +56,7 @@ void ConstantInitializer::operator()() VERBOSE(FillOperandData) << "Fill data for operand " << ind.value() << std::endl; auto layout = - model_obj.lower_info()->def_backends().getOnlyElement()->config()->getOperandLayout(); + _graph.getLowerInfo(ind)->def_backends().getOnlyElement()->config()->getOperandLayout(); const auto shape = model_obj.shape(); auto base = reinterpret_cast(model_obj.data().base()); auto size = model_obj.data().size(); diff --git a/runtimes/neurun/src/compiler/SubTensorAnalyzer.cc b/runtimes/neurun/src/compiler/SubTensorAnalyzer.cc index 0851b79..554a28d 100644 --- a/runtimes/neurun/src/compiler/SubTensorAnalyzer.cc +++ b/runtimes/neurun/src/compiler/SubTensorAnalyzer.cc @@ -58,7 +58,9 @@ void SubTensorAnalyzer::visit(const model::operation::ConcatNode &node) int32_t axis_point = 0; for (auto &input_index : inputs) { - auto input_shape_4D = _ctx.at(input_index).lower_info()->shape(); + auto lower_info = _graph.getLowerInfo(input_index); + assert(lower_info != nullptr); + auto input_shape_4D = lower_info->shape(); std::vector offset = {0, 0, 0, 0}; offset[axis] = axis_point; neurun::util::feature::Coordinate4D coordinate_info(offset[0], offset[1], offset[2], offset[3]); diff --git a/runtimes/neurun/src/compiler/SubTensorAnalyzer.h b/runtimes/neurun/src/compiler/SubTensorAnalyzer.h index ddfd102..689703a 100644 --- a/runtimes/neurun/src/compiler/SubTensorAnalyzer.h +++ b/runtimes/neurun/src/compiler/SubTensorAnalyzer.h @@ -24,6 +24,7 @@ #define __NEURUN_COMPILER_SUBTENSOR_ANALYZER_H__ #include "model/operation/NodeVisitor.h" +#include "graph/Graph.h" namespace neurun { @@ -51,12 +52,13 @@ public: * @brief Construct a new SubTensorAnalyzer object * @param[in] ctx Graph operand set */ - SubTensorAnalyzer(neurun::model::operand::Set &ctx) : _ctx{ctx} {} + SubTensorAnalyzer(graph::Graph &graph) : _graph{graph}, _ctx{graph.operands()} {} public: virtual void visit(const model::operation::ConcatNode &) override; private: + const neurun::graph::Graph &_graph; neurun::model::operand::Set &_ctx; }; diff --git a/runtimes/neurun/src/compiler/operand/Context.h b/runtimes/neurun/src/compiler/operand/Context.h index 3e63566..d0f83bf 100644 --- a/runtimes/neurun/src/compiler/operand/Context.h +++ b/runtimes/neurun/src/compiler/operand/Context.h @@ -19,7 +19,6 @@ #include "backend/interface/operand/IObject.h" #include "model/operand/Index.h" - #include #include diff --git a/runtimes/neurun/src/dumper/dot/DotDumper.cc b/runtimes/neurun/src/dumper/dot/DotDumper.cc index 1e53ece..0bb8412 100644 --- a/runtimes/neurun/src/dumper/dot/DotDumper.cc +++ b/runtimes/neurun/src/dumper/dot/DotDumper.cc @@ -46,7 +46,8 @@ void DotDumper::dumpIfNeeded(const std::string &tag) { using neurun::dumper::dot::DotOperandInfo; auto child = std::make_shared(output, operands.at(output), - DotOperandInfo::Type::MODEL_OUTPUT); + DotOperandInfo::Type::MODEL_OUTPUT, + _graph.getLowerInfo(output)); node_info.appendChild(child); } @@ -80,7 +81,8 @@ void DotDumper::dumpIfNeeded(const std::string &tag) return DotOperandInfo::Type::INTERNAL; }(); - neurun::dumper::dot::DotOperandInfo operand_info(index, object, type); + auto lower_info = _graph.getLowerInfo(index); + neurun::dumper::dot::DotOperandInfo operand_info(index, object, type, lower_info); for (auto operation_index : object.getUses().list()) { diff --git a/runtimes/neurun/src/dumper/dot/DotOperandInfo.cc b/runtimes/neurun/src/dumper/dot/DotOperandInfo.cc index 8f59050..5c49b8c 100644 --- a/runtimes/neurun/src/dumper/dot/DotOperandInfo.cc +++ b/runtimes/neurun/src/dumper/dot/DotOperandInfo.cc @@ -17,6 +17,7 @@ #include #include "DotOperandInfo.h" +#include "graph/Graph.h" #include "graph/operand/LowerInfo.h" #include "backend/interface/IConfig.h" #include "backend/BackendManager.h" @@ -36,14 +37,11 @@ const std::string DotOperandInfo::BG_COLOR_SCHEME = "set38"; const std::string DotOperandInfo::BG_COLORS[8] = {"4", "5", "6", "2", "7", "3", "1", "8"}; DotOperandInfo::DotOperandInfo(const neurun::model::operand::Index &index, - const neurun::model::operand::Object &object, Type type) - : _index(index), _object(object), _type(type) + const neurun::model::operand::Object &object, Type type, + const neurun::graph::operand::LowerInfo *lower_info) + : _index(index), _object(object), _type(type), _lower_info(lower_info) { - const auto &lower_info = object.lower_info(); - if (lower_info) - { - addBackendLabel(); - } + addBackendLabel(); } std::string DotOperandInfo::index_str() const @@ -87,11 +85,10 @@ std::string DotOperandInfo::bg_color_scheme() const { return BG_COLOR_SCHEME; } std::string DotOperandInfo::bg_color() const { - const auto &lower_info = _object.lower_info(); - if (!lower_info) + if (!_lower_info) return DEFAULT_BG_COLOR; - assert(lower_info != nullptr); - const auto &def_backends = lower_info->def_backends(); + + const auto &def_backends = _lower_info->def_backends(); assert(def_backends.size() == 1); std::string backend_id = def_backends.getOnlyElement()->config()->id(); @@ -112,10 +109,11 @@ std::string DotOperandInfo::bg_color() const void DotOperandInfo::addBackendLabel() { + if (!_lower_info) + return; + std::string label; - const auto &lower_info = _object.lower_info(); - assert(lower_info != nullptr); - const auto &def_backends = lower_info->def_backends(); + const auto &def_backends = _lower_info->def_backends(); assert(def_backends.size() == 1); label += "["; diff --git a/runtimes/neurun/src/dumper/dot/DotOperandInfo.h b/runtimes/neurun/src/dumper/dot/DotOperandInfo.h index c54da44..b44b598 100644 --- a/runtimes/neurun/src/dumper/dot/DotOperandInfo.h +++ b/runtimes/neurun/src/dumper/dot/DotOperandInfo.h @@ -25,6 +25,14 @@ namespace neurun { +namespace graph +{ +class Graph; +} // namespace graph +} // namespace neurun + +namespace neurun +{ namespace dumper { namespace dot @@ -50,7 +58,8 @@ public: public: DotOperandInfo(const neurun::model::operand::Index &index, - const neurun::model::operand::Object &object, Type type); + const neurun::model::operand::Object &object, Type type, + const neurun::graph::operand::LowerInfo *lower_info); public: virtual std::string index_str() const override; @@ -66,6 +75,7 @@ private: const neurun::model::operand::Index &_index; const neurun::model::operand::Object &_object; Type _type; + const neurun::graph::operand::LowerInfo *_lower_info; std::vector _labels; }; diff --git a/runtimes/neurun/src/frontend/execution.cc b/runtimes/neurun/src/frontend/execution.cc index 5f1729b..2c6b820 100644 --- a/runtimes/neurun/src/frontend/execution.cc +++ b/runtimes/neurun/src/frontend/execution.cc @@ -35,12 +35,13 @@ inline void source(ANeuralNetworksExecution *execution, const ::neurun::model::operand::DataType &type, int32_t index, const void *buffer, size_t length) { - const auto &operands = execution->plan().model().operands(); + const auto &graph = execution->plan().model(); + const auto &operands = graph.operands(); neurun::model::operand::IO::Index input_index{index}; - const auto operand_index = execution->plan().model().getInputs().at(input_index); + const auto operand_index = graph.getInputs().at(input_index); auto operand = &operands.at(operand_index); - auto operand_li = operand->lower_info(); + auto operand_li = graph.getLowerInfo(operand_index); const auto output_backend = operand_li->def_backends().getOnlyElement(); const auto output_layout = output_backend->config()->getOperandLayout(); auto input_layout = execution->plan() @@ -89,12 +90,13 @@ inline void sink(ANeuralNetworksExecution *execution, const ::neurun::model::operand::DataType &type, int32_t index, void *buffer, size_t length) { - const auto &operands = execution->plan().model().operands(); + const auto &graph = execution->plan().model(); + const auto &operands = graph.operands(); neurun::model::operand::IO::Index input_index{index}; - const auto operand_index = execution->plan().model().getOutputs().at(input_index); + const auto operand_index = graph.getOutputs().at(input_index); auto operand = &operands.at(operand_index); - auto operand_li = operand->lower_info(); + auto operand_li = graph.getLowerInfo(operand_index); const auto input_backend = operand_li->def_backends().getOnlyElement(); const auto input_layout = input_backend->config()->getOperandLayout(); auto output_layout = execution->plan() diff --git a/runtimes/neurun/src/graph/Graph.cc b/runtimes/neurun/src/graph/Graph.cc index 832e2b8..dc35efa 100644 --- a/runtimes/neurun/src/graph/Graph.cc +++ b/runtimes/neurun/src/graph/Graph.cc @@ -155,12 +155,12 @@ void Graph::lower(void) // Set LowerInfo for each operand from the operand::LowerInfo holder _model->operands.iterate([&](const model::operand::Index &index, model::operand::Object &object) { - object.lower_info(std::move(operands_lower_info[index])); + setLowerInfo(index, std::move(operands_lower_info[index])); // Dump operand LowerInfo // TODO Extract this dumping procedure to be reusable - if (!object.lower_info()->def_backends().empty() || - !object.lower_info()->use_backends().empty()) + if (!getLowerInfo(index)->def_backends().empty() || + !getLowerInfo(index)->use_backends().empty()) { auto backends_to_string = [](const operand::BackendSet &backends) { std::string str; @@ -182,7 +182,7 @@ void Graph::lower(void) return "{ " + str + "}"; }; - const auto &lower_info = object.lower_info(); + const auto lower_info = getLowerInfo(index); const auto &shape = object.shape(); const auto &lower_shape = lower_info->shape(); std::string def_ops = operation_index_to_string(object.getDef()); @@ -264,6 +264,28 @@ void Graph::setLowerInfo(const model::operation::Index &index, _operation_lower_info.insert(std::make_pair(index, std::move(lower_info))); } +const operand::LowerInfo *Graph::getLowerInfo(const model::operand::Index &index) const +{ + auto itr = _operand_lower_info.find(index); + if (itr == _operand_lower_info.end()) + return nullptr; + return itr->second.get(); +} + +operand::LowerInfo *Graph::getLowerInfo(const model::operand::Index &index) +{ + auto itr = _operand_lower_info.find(index); + if (itr == _operand_lower_info.end()) + return nullptr; + return itr->second.get(); +} + +void Graph::setLowerInfo(const model::operand::Index &index, + std::unique_ptr &&lower_info) +{ + _operand_lower_info.insert(std::make_pair(index, std::move(lower_info))); +} + } // namespace graph } // namespace neurun diff --git a/runtimes/neurun/src/graph/Graph.h b/runtimes/neurun/src/graph/Graph.h index afcfdce..edf6b2d 100644 --- a/runtimes/neurun/src/graph/Graph.h +++ b/runtimes/neurun/src/graph/Graph.h @@ -139,11 +139,17 @@ public: const operation::LowerInfo *getLowerInfo(const model::operation::Index &index) const; void setLowerInfo(const model::operation::Index &index, std::unique_ptr &&lower_info); + const operand::LowerInfo *getLowerInfo(const model::operand::Index &index) const; + operand::LowerInfo *getLowerInfo(const model::operand::Index &index); + void setLowerInfo(const model::operand::Index &index, + std::unique_ptr &&lower_info); private: std::unique_ptr _backend_resolver; std::unordered_map> _operation_lower_info; + std::unordered_map> + _operand_lower_info; }; } // namespace graph diff --git a/runtimes/neurun/src/graph/pass/PermutationEliminationPass.cc b/runtimes/neurun/src/graph/pass/PermutationEliminationPass.cc index 848f6b5..b1dc13c 100644 --- a/runtimes/neurun/src/graph/pass/PermutationEliminationPass.cc +++ b/runtimes/neurun/src/graph/pass/PermutationEliminationPass.cc @@ -143,8 +143,8 @@ bool PermutationEliminationPass::isPermuteLayerToEliminate( const model::operand::IndexSet &inp_indexes, const model::operand::IndexSet &out_indexes, bool is_for_model_input) { - auto input_def_backends = _graph.operands().at(inp_indexes.at(0)).lower_info()->def_backends(); - auto output_def_backends = _graph.operands().at(out_indexes.at(0)).lower_info()->def_backends(); + auto input_def_backends = _graph.getLowerInfo(inp_indexes.at(0))->def_backends(); + auto output_def_backends = _graph.getLowerInfo(out_indexes.at(0))->def_backends(); auto input_layout = input_def_backends.getOnlyElement()->config()->getOperandLayout(); auto output_layout = output_def_backends.getOnlyElement()->config()->getOperandLayout(); @@ -157,7 +157,7 @@ bool PermutationEliminationPass::isPermuteLayerToEliminate( // all operands' backend must be the same for (auto index : inp_indexes) { - auto op_backend_set = _graph.operands().at(index).lower_info()->def_backends(); + auto op_backend_set = _graph.getLowerInfo(index)->def_backends(); if (op_backend_set.size() != 1 || input_layout != op_backend_set.getOnlyElement()->config()->getOperandLayout()) { @@ -167,7 +167,7 @@ bool PermutationEliminationPass::isPermuteLayerToEliminate( // all operands' backend must be the same for (auto index : out_indexes) { - auto op_backend_set = _graph.operands().at(index).lower_info()->def_backends(); + auto op_backend_set = _graph.getLowerInfo(index)->def_backends(); if (op_backend_set.size() != 1 || output_layout != op_backend_set.getOnlyElement()->config()->getOperandLayout()) { diff --git a/runtimes/neurun/src/graph/pass/PermutationInsertionPass.cc b/runtimes/neurun/src/graph/pass/PermutationInsertionPass.cc index de592ce..e4632e1 100644 --- a/runtimes/neurun/src/graph/pass/PermutationInsertionPass.cc +++ b/runtimes/neurun/src/graph/pass/PermutationInsertionPass.cc @@ -40,7 +40,7 @@ namespace pass void PermutationInsertionPass::callback(const model::operand::Index &index, model::operand::Object &object) { - auto &&operand_li = object.lower_info(); + auto &&operand_li = _graph.getLowerInfo(index); assert(operand_li); // NOTE Later, constants also will have Def @@ -136,16 +136,17 @@ PermutationInsertionPass::insertPermute(const model::operand::Index &operand_ind nnfw::cpp14::make_unique(operand::asShape4D(operand.shape())); out_operand_li->addDefBackend(backend); out_operand_li->addUseBackend(backend); - out_operand.lower_info(std::move(out_operand_li)); + _graph.setLowerInfo(out_operand_index, std::move(out_operand_li)); // Update LowerInfo of input operand - operand.lower_info()->removeUseBackend(backend); - operand.lower_info()->addUseBackend(operand.lower_info()->def_backends().getOnlyElement()); + auto operand_lower_info = _graph.getLowerInfo(operand_index); + operand_lower_info->removeUseBackend(backend); + operand_lower_info->addUseBackend(operand_lower_info->def_backends().getOnlyElement()); using PermuteNode = model::operation::PermuteNode; - auto input_backend = operand.lower_info()->def_backends().getOnlyElement(); - auto output_backend = out_operand.lower_info()->def_backends().getOnlyElement(); + auto input_backend = _graph.getLowerInfo(operand_index)->def_backends().getOnlyElement(); + auto output_backend = _graph.getLowerInfo(out_operand_index)->def_backends().getOnlyElement(); // Find Permutation Type auto type = [&]() { diff --git a/runtimes/neurun/src/linear/Linear.cc b/runtimes/neurun/src/linear/Linear.cc index 6452bbd..34a90d8 100644 --- a/runtimes/neurun/src/linear/Linear.cc +++ b/runtimes/neurun/src/linear/Linear.cc @@ -65,10 +65,10 @@ backend::TensorBuilderSet Linear::planTensors() using FnOnTensorBuilder = std::function; - const auto &operands = _graph.operands(); - auto iterTensorBuilders = [&operands](const model::operand::Index &ind, FnOnTensorBuilder fn) { - const auto &obj = operands.at(ind); - for (auto backend : obj.lower_info()->def_backends()) + const auto &graph = _graph; + auto iterTensorBuilders = [&graph](const model::operand::Index &ind, FnOnTensorBuilder fn) { + const auto lower_info = graph.getLowerInfo(ind); + for (auto backend : lower_info->def_backends()) { auto tensor_builder = backend->tensor_builder(); fn(ind, tensor_builder); @@ -82,6 +82,7 @@ backend::TensorBuilderSet Linear::planTensors() _graph.operands().iterate( [&](const model::operand::Index &ind, const model::operand::Object &obj) { + const auto lower_info = graph.getLowerInfo(ind); uses_map[ind] = obj.getUses().size(); // If a tensor is a constant, increase the use of the tensor. @@ -92,7 +93,7 @@ backend::TensorBuilderSet Linear::planTensors() uses_map[ind]++; } - for (auto backend : obj.lower_info()->def_backends()) + for (auto backend : lower_info->def_backends()) { bool isSubTensor = false; auto tensor_builder = backend->tensor_builder(); @@ -151,6 +152,7 @@ backend::TensorBuilderSet Linear::planTensors() // 1. Scan USE of inputs. Decrease the USE and deallocate if the USE is 0 // 2. Scan DEF of outputs. If the DEF, allocate it VERBOSE(LINEAR) << "TENSORS" << std::endl; + const auto &operands = _graph.operands(); for (const auto op : _operations) { for (const auto &ind : op.node->getOutputs()) diff --git a/runtimes/neurun/src/model/operand/Object.cc b/runtimes/neurun/src/model/operand/Object.cc index 63cf29b..0edd352 100644 --- a/runtimes/neurun/src/model/operand/Object.cc +++ b/runtimes/neurun/src/model/operand/Object.cc @@ -105,15 +105,6 @@ void Object::removeDef(const ::neurun::model::operation::Index &idx) _def.remove(idx); } -void Object::lower_info(std::unique_ptr &&lower_info) -{ - _lower_info = std::move(lower_info); -} - -const graph::operand::LowerInfo *Object::lower_info() const { return _lower_info.get(); } - -graph::operand::LowerInfo *Object::lower_info() { return _lower_info.get(); } - void Object::parent_info(std::unique_ptr &&parent_info) { _parent_info = std::move(parent_info); diff --git a/runtimes/neurun/src/model/operand/Object.h b/runtimes/neurun/src/model/operand/Object.h index eb5f627..bd396d7 100644 --- a/runtimes/neurun/src/model/operand/Object.h +++ b/runtimes/neurun/src/model/operand/Object.h @@ -96,9 +96,6 @@ public: } public: - void lower_info(std::unique_ptr &&lower_info); - const graph::operand::LowerInfo *lower_info() const; - graph::operand::LowerInfo *lower_info(); /** * @brief Set parent information * @param[in] parent_info Parent information @@ -124,7 +121,6 @@ private: operation::IndexList _uses; operation::IndexList _def; // size is 0 (constant) or 1 (from def operation) - std::unique_ptr _lower_info; std::unique_ptr _parent_info; }; -- 2.7.4