From f7bd544d1bb47ac6d9ec6c63de8537588f79edae Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=98=A4=ED=98=95=EC=84=9D/=EB=8F=99=EC=9E=91=EC=A0=9C?= =?utf8?q?=EC=96=B4Lab=28SR=29/Staff=20Engineer/=EC=82=BC=EC=84=B1?= =?utf8?q?=EC=A0=84=EC=9E=90?= Date: Thu, 1 Nov 2018 13:36:46 +0900 Subject: [PATCH] Introduce SubTensorAnalyzer (#3410) Introduce SubTensorAnalyzer and insert subtensor analyze phase in compilation Signed-off-by: Hyeongseok Oh --- runtimes/neurun/CMakeLists.txt | 3 +- .../neurun/src/frontend/wrapper/compilation.cc | 9 ++ runtimes/neurun/src/middleend/SubTensorAnalyzer.cc | 126 +++++++++++++++++++++ runtimes/neurun/src/middleend/SubTensorAnalyzer.h | 75 ++++++++++++ 4 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 runtimes/neurun/src/middleend/SubTensorAnalyzer.cc create mode 100644 runtimes/neurun/src/middleend/SubTensorAnalyzer.h diff --git a/runtimes/neurun/CMakeLists.txt b/runtimes/neurun/CMakeLists.txt index 6379f8d..88bd126 100644 --- a/runtimes/neurun/CMakeLists.txt +++ b/runtimes/neurun/CMakeLists.txt @@ -14,6 +14,7 @@ add_subdirectory(src/backend) file(GLOB SOURCES "src/*.cc") file(GLOB_RECURSE SOURCES_FRONTEND "src/frontend/*.cc") +file(GLOB_RECURSE SOURCES_MIDDLEEND "src/middleend/*.cc") file(GLOB SOURCES_BACKEND "src/backend/*.cc") file(GLOB_RECURSE SOURCES_INTERNAL "src/internal/*.cc") file(GLOB_RECURSE SOURCES_GRAPH "src/graph/*.cc") @@ -23,7 +24,7 @@ file(GLOB_RECURSE SOURCES_DUMPER "src/dumper/*.cc") file(GLOB_RECURSE SOURCES_VERIFIER "src/verifier/*.cc") file(GLOB_RECURSE SOURCES_VERIFIER "src/util/*.cc") -set(SOURCES ${SOURCES} ${SOURCES_FRONTEND} ${SOURCES_BACKEND} ${SOURCES_INTERNAL} ${SOURCES_GRAPH} ${SOURCES_LINEAR} ${SOURCES_CODEGEN} ${SOURCES_DUMPER} ${SOURCES_VERIFIER}) +set(SOURCES ${SOURCES} ${SOURCES_FRONTEND} ${SOURCES_MIDDLEEND} ${SOURCES_BACKEND} ${SOURCES_INTERNAL} ${SOURCES_GRAPH} ${SOURCES_LINEAR} ${SOURCES_CODEGEN} ${SOURCES_DUMPER} ${SOURCES_VERIFIER}) # NOTE For now ARMCompute is necessary # TODO Remove required package below(should be optional) diff --git a/runtimes/neurun/src/frontend/wrapper/compilation.cc b/runtimes/neurun/src/frontend/wrapper/compilation.cc index be4d147..4258085 100644 --- a/runtimes/neurun/src/frontend/wrapper/compilation.cc +++ b/runtimes/neurun/src/frontend/wrapper/compilation.cc @@ -22,6 +22,7 @@ #include "codegen/IPlanBuilder.h" #include "codegen/Planner.h" #include "codegen/PlanBuilder.h" +#include "middleend/SubTensorAnalyzer.h" #include "linear/Linear.h" #include "util/EnvVar.h" @@ -54,6 +55,14 @@ int ANeuralNetworksCompilation::finish() // Dump ops linear->accept(neurun::graph::dumper::Dumper{}); + // SubTensorInfo should be generated after lower, before planner & finalize + // lower: decide backend and insert permutation + // planner: stage generate (use SubTensorInfo to return stage. prepare to optimization) + // finalize: generate tensor using subtensor info, then execute stage + // Generated SubTensorInfo is in operand(Object) + // for easy pass SubTensorInfo to plan builder and tensor builder + linear->accept(neurun::middleend::SubTensorAnalyzer{graph.operands()}); + neurun::codegen::PlanBuilder plan_builder{plan}; auto tensor_builders = linear->markTensors(); diff --git a/runtimes/neurun/src/middleend/SubTensorAnalyzer.cc b/runtimes/neurun/src/middleend/SubTensorAnalyzer.cc new file mode 100644 index 0000000..79f841b --- /dev/null +++ b/runtimes/neurun/src/middleend/SubTensorAnalyzer.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SubTensorAnalyzer.h" + +#include + +#include "nnfw/std/memory.h" +#include "internal/Convert.h" +#include "graph/operand/Set.h" +#include "codegen/IPlanBuilder.h" +#include "graph/operation/LowerInfo.h" +#include "logging.h" + +namespace neurun +{ +namespace middleend +{ + +void SubTensorAnalyzer::visit(const graph::operation::Conv2D::Implicit::Node &) +{ + // DO NOTHING +} + +void SubTensorAnalyzer::visit(const graph::operation::MaxPool2D::Implicit::Node &) +{ + // DO NOTHING +} + +void SubTensorAnalyzer::visit(const graph::operation::AvgPool2D::Implicit::Node &) +{ + // DO NOTHING +} + +void SubTensorAnalyzer::visit(const graph::operation::Concat::Node &node) +{ + // If operator is concat (or other operators related with subsumption), fill subsumption info + // TODO: if one tensor is subset of many parents or model input + // Solution 1. Handle 1st parent only, ignore others (need to invert for other childrun) + // Solution 2. Insert copy operation for other parents + auto axis_index = node.param().axis_index; + + // To prepare concat elimination, axis should be constant + if (_ctx.at(axis_index).getUsage() != graph::operand::OperandUsage::CONSTANT) + { + VERBOSE(SUBTENSOR) << "Cannot handle non-constant axis" << std::endl; + return; + } + + // NOTE This implementation assumes concat over feature depth + // TODO Remove this assumption + int32_t axis = _ctx.at(axis_index).asScalar(); + if (axis != 3) + { + VERBOSE(SUBTENSOR) << "Cannot handle axis is not channel" << std::endl; + return; + } + + auto &output_index = node.getOutputs().at(0); + auto &inputs = node.getInputs(); + + int32_t axis_point = 0; + for (auto &input_index : inputs) + { + auto input_shape_4D = _ctx.at(input_index).lower_info()->shape(); + std::vector offset = {0, 0, 0, 0}; + offset[axis] = axis_point; + graph::operand::ParentInfo::Coordinate4D coordinate_info(offset[0], offset[1], offset[2], + offset[3]); + std::unique_ptr parentInfo = + nnfw::make_unique(output_index, coordinate_info); + + // NOTD Not support multiple parent tensor yet + assert(_ctx.at(input_index).parent_info() == nullptr); + _ctx.at(input_index).parent_info(std::move(parentInfo)); + + // NOTE Only support when axis is 3(channel) + axis_point += input_shape_4D.c(); + } +} + +void SubTensorAnalyzer::visit(const graph::operation::FullyConnected::Node &) +{ + // DO NOTHING +} + +void SubTensorAnalyzer::visit(const graph::operation::Reshape::Node &) +{ + // DO NOTHING +} + +void SubTensorAnalyzer::visit(const graph::operation::Softmax::Node &) +{ + // DO NOTHING +} + +void SubTensorAnalyzer::visit(const graph::operation::NOP::Node &) +{ + // DO NOTHING +} + +void SubTensorAnalyzer::visit(const graph::operation::Permute::Node &) +{ + // DO NOTHING +} + +void SubTensorAnalyzer::visit(const graph::operation::AddNode &) +{ + // DO NOTHING +} + +} // namespace middleend +} // namespace neurun diff --git a/runtimes/neurun/src/middleend/SubTensorAnalyzer.h b/runtimes/neurun/src/middleend/SubTensorAnalyzer.h new file mode 100644 index 0000000..abe5ce9 --- /dev/null +++ b/runtimes/neurun/src/middleend/SubTensorAnalyzer.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file SubTensorAnalyzer.h + * @brief This file contains SubTensorAnalyzer to analyze tensor subsumption + * using operation visitor + */ + +#ifndef __NEURUN_MIDDLEEND_SUBTENSOR_ANALYZER_H__ +#define __NEURUN_MIDDLEEND_SUBTENSOR_ANALYZER_H__ + +#include "graph/operation/NodeVisitor.h" + +namespace neurun +{ +namespace graph +{ +namespace operand +{ +class Set; +} // namespace operation +} // namespace graph +} // namespace neurun + +namespace neurun +{ +namespace middleend +{ + +/** + * @brief Class to analyze tensor subsumption + */ +class SubTensorAnalyzer : public graph::operation::NodeVisitor +{ +public: + /** + * @brief Construct a new SubTensorAnalyzer object + * @param[in] ctx Graph operand set + */ + SubTensorAnalyzer(neurun::graph::operand::Set &ctx) : _ctx{ctx} {} + +public: + virtual void visit(const graph::operation::Conv2D::Implicit::Node &) override; + virtual void visit(const graph::operation::MaxPool2D::Implicit::Node &) override; + virtual void visit(const graph::operation::AvgPool2D::Implicit::Node &) override; + virtual void visit(const graph::operation::Concat::Node &) override; + virtual void visit(const graph::operation::Reshape::Node &) override; + virtual void visit(const graph::operation::FullyConnected::Node &) override; + virtual void visit(const graph::operation::Softmax::Node &) override; + virtual void visit(const graph::operation::NOP::Node &) override; + virtual void visit(const graph::operation::Permute::Node &) override; + virtual void visit(const graph::operation::AddNode &) override; + +private: + neurun::graph::operand::Set &_ctx; +}; + +} // namespace middleend +} // namespace neurun + +#endif // __NEURUN_MIDDLEEND_SUBTENSOR_ANALYZER_H__ -- 2.7.4