From 716be4f4460ea79e2c571db34ae5c0a9bd89d8c7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=9E=A5=EC=A7=80=EC=84=AD/On-Device=20Lab=28SR=29/Enginee?= =?utf8?q?r/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Mon, 4 Nov 2019 18:50:36 +0900 Subject: [PATCH] Enable ncnn(srcn) tensor to support NCHW (#8723) This commit enables ncnn(srcn) tensor to support NCHW(row major). Signed-off-by: jiseob.jang --- runtime/neurun/backend/srcn/Convert.cc | 56 +++++++++++++++++++ runtime/neurun/backend/srcn/Convert.h | 42 ++++++++++++++ runtime/neurun/backend/srcn/MemoryManager.cc | 5 +- runtime/neurun/backend/srcn/MemoryManager.h | 3 +- runtime/neurun/backend/srcn/Swizzle.h | 84 ++++++++++++++++++++++++++++ runtime/neurun/backend/srcn/TensorBuilder.cc | 21 ++++--- runtime/neurun/backend/srcn/TensorBuilder.h | 2 +- runtime/neurun/backend/srcn/TensorManager.cc | 7 ++- runtime/neurun/backend/srcn/TensorManager.h | 2 +- runtime/neurun/backend/srcn/operand/Tensor.h | 6 +- 10 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 runtime/neurun/backend/srcn/Convert.cc create mode 100644 runtime/neurun/backend/srcn/Convert.h create mode 100644 runtime/neurun/backend/srcn/Swizzle.h diff --git a/runtime/neurun/backend/srcn/Convert.cc b/runtime/neurun/backend/srcn/Convert.cc new file mode 100644 index 0000000..bc79ddf --- /dev/null +++ b/runtime/neurun/backend/srcn/Convert.cc @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Convert.h" + +#include +#include +#include "Swizzle.h" +#include + +namespace neurun +{ +namespace backend +{ +namespace srcn +{ + +model::Shape asTensorShape(const model::Shape &shape, model::Layout frontend_layout, + model::Layout backend_layout) +{ + const uint32_t rank = shape.rank(); + + model::Shape ret(rank); + for (uint32_t axis = 0; axis < rank; ++axis) + { + const auto ncnn_axis = ToNCNNAxis(rank, axis, frontend_layout, backend_layout); + ret.dim(ncnn_axis) = shape.dim(axis); + } + + return ret; +} + +model::OperandInfo asTensorInfo(const model::Shape &shape, const model::TypeInfo &typeInfo, + model::Layout frontend_layout, model::Layout backend_layout) +{ + model::OperandInfo info(asTensorShape(shape, frontend_layout, backend_layout), typeInfo); + + return info; +} + +} // namespace srcn +} // namespace backend +} // namespace neurun diff --git a/runtime/neurun/backend/srcn/Convert.h b/runtime/neurun/backend/srcn/Convert.h new file mode 100644 index 0000000..caa6447 --- /dev/null +++ b/runtime/neurun/backend/srcn/Convert.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NEURUN_BACKEND_SRCN_CONVERT_H__ +#define __NEURUN_BACKEND_SRCN_CONVERT_H__ + +#include +#include +#include +#include + +namespace neurun +{ +namespace backend +{ +namespace srcn +{ + +model::Shape asTensorShape(const model::Shape &shape, model::Layout frontend_layout, + model::Layout backend_layout); + +model::OperandInfo asTensorInfo(const model::Shape &shape, const model::TypeInfo &typeInfo, + model::Layout frontend_layout, model::Layout backend_layout); + +} // namespace srcn +} // namespace backend +} // namespace neurun + +#endif // __NEURUN_BACKEND_SRCN_CONVERT_H__ diff --git a/runtime/neurun/backend/srcn/MemoryManager.cc b/runtime/neurun/backend/srcn/MemoryManager.cc index ad3f639..10246fe 100644 --- a/runtime/neurun/backend/srcn/MemoryManager.cc +++ b/runtime/neurun/backend/srcn/MemoryManager.cc @@ -41,9 +41,10 @@ IMemoryPlanner *MemoryManager::createMemoryPlanner() return MemoryPlannerFactory::instance().create(planner_id); } -void MemoryManager::buildTensor(const model::OperandIndex &ind, const model::OperandInfo &info) +void MemoryManager::buildTensor(const model::OperandIndex &ind, const model::OperandInfo &info, + model::Layout layout) { - auto tensor = std::make_shared(info); + auto tensor = std::make_shared(info, layout); _tensors[ind] = tensor; } diff --git a/runtime/neurun/backend/srcn/MemoryManager.h b/runtime/neurun/backend/srcn/MemoryManager.h index f0cf8f0..edfcbd7 100644 --- a/runtime/neurun/backend/srcn/MemoryManager.h +++ b/runtime/neurun/backend/srcn/MemoryManager.h @@ -39,7 +39,8 @@ public: void allocate(void) override; void deallocate(void) override { _mem_alloc->release(); } - void buildTensor(const model::OperandIndex &ind, const model::OperandInfo &info); + void buildTensor(const model::OperandIndex &ind, const model::OperandInfo &info, + model::Layout layout); void claimPlan(const model::OperandIndex &ind, uint32_t size); void releasePlan(const model::OperandIndex &ind); diff --git a/runtime/neurun/backend/srcn/Swizzle.h b/runtime/neurun/backend/srcn/Swizzle.h new file mode 100644 index 0000000..d7404f0 --- /dev/null +++ b/runtime/neurun/backend/srcn/Swizzle.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NEURUN_BACKEND_SRCN_SWIZZLE_H__ +#define __NEURUN_BACKEND_SRCN_SWIZZLE_H__ + +#include +#include + +namespace neurun +{ +namespace backend +{ +namespace srcn +{ + +// Convert axis in ncnn order +inline uint32_t ToNCNNAxis(uint32_t rank, uint32_t axis, + const model::Layout org_layout = model::Layout::UNKNOWN, + const model::Layout ncnn_layout = model::Layout::UNKNOWN) +{ + assert(rank > axis); + + if (rank >= 4 && org_layout == model::Layout::NHWC && ncnn_layout == model::Layout::NCHW) + { + // NHWC -> NCHW + // DEPTH + if (axis == 3) + { + return 1; + } + // WIDTH + if (axis == 2) + { + return 3; + } + // HEIGHT + if (axis == 1) + { + return 2; + } + } + + if (rank >= 4 && org_layout == model::Layout::NCHW && ncnn_layout == model::Layout::NHWC) + { + // NCHW -> NHWC + // WIDTH + if (axis == 3) + { + return 2; + } + // HEIGHT + if (axis == 2) + { + return 1; + } + // DEPTH + if (axis == 1) + { + return 3; + } + } + + return axis; +} + +} // namespace srcn +} // namespace backend +} // namespace neurun + +#endif // __NEURUN_BACKEND_SRCN_SWIZZLE_H__ diff --git a/runtime/neurun/backend/srcn/TensorBuilder.cc b/runtime/neurun/backend/srcn/TensorBuilder.cc index 5cf9e4c..cc0a7d0 100644 --- a/runtime/neurun/backend/srcn/TensorBuilder.cc +++ b/runtime/neurun/backend/srcn/TensorBuilder.cc @@ -17,7 +17,7 @@ #include "TensorBuilder.h" #include - +#include "Convert.h" #include "util/logging.h" namespace neurun @@ -33,13 +33,15 @@ TensorBuilder::TensorBuilder() : _tensor_mgr{new TensorManager()} } void TensorBuilder::registerTensorInfo(const model::OperandIndex &ind, - const model::OperandInfo &info, - model::Layout /*frontend_layout*/, - model::Layout /*backend_layout*/, bool as_const) + const model::OperandInfo &info, model::Layout, + model::Layout backend_layout, bool as_const) { + if (backend_layout == model::Layout::NCHW) + { + throw std::runtime_error("Not supported layout yet"); + } _tensor_info_map.emplace(ind, info); - - // TODO set the layout + _tensor_layout_map.emplace(ind, backend_layout); if (as_const) _constants.append(ind); @@ -57,7 +59,12 @@ void TensorBuilder::notifyFirstUse(const model::OperandIndex &ind) assert(_tensor_info_map.find(ind) != _tensor_info_map.end()); const auto &info = _tensor_info_map.at(ind); const auto size = info.total_size(); - _tensor_mgr->buildTensor(ind, info, _constants.contains(ind)); + // TODO Remove frontend_layout + const auto frontend_layout = model::Layout::NHWC; + const auto &backend_layout = _tensor_layout_map.at(ind); + const auto tensor_info = + asTensorInfo(info.shape(), info.typeInfo(), frontend_layout, backend_layout); + _tensor_mgr->buildTensor(ind, tensor_info, backend_layout, _constants.contains(ind)); _tensor_mgr->claimPlan(ind, size); } diff --git a/runtime/neurun/backend/srcn/TensorBuilder.h b/runtime/neurun/backend/srcn/TensorBuilder.h index 3d92098..a718ebf 100644 --- a/runtime/neurun/backend/srcn/TensorBuilder.h +++ b/runtime/neurun/backend/srcn/TensorBuilder.h @@ -80,7 +80,7 @@ public: private: std::unique_ptr _tensor_mgr; model::OperandIndexMap _tensor_info_map; - model::OperandIndexMap> _tensor_layouts_map; + model::OperandIndexMap _tensor_layout_map; model::OperandIndexSequence _constants; }; diff --git a/runtime/neurun/backend/srcn/TensorManager.cc b/runtime/neurun/backend/srcn/TensorManager.cc index d0c8027..1014ce1 100644 --- a/runtime/neurun/backend/srcn/TensorManager.cc +++ b/runtime/neurun/backend/srcn/TensorManager.cc @@ -37,17 +37,18 @@ void TensorManager::deallocateConsts(void) { _const_mgr->deallocate(); } void TensorManager::deallocateNonconsts(void) { _nonconst_mgr->deallocate(); } void TensorManager::buildTensor(const model::OperandIndex &ind, - const model::OperandInfo &tensor_info, bool as_const) + const model::OperandInfo &tensor_info, model::Layout layout, + bool as_const) { assert(_ind_to_mgr.find(ind) == _ind_to_mgr.end()); if (as_const) { - _const_mgr->buildTensor(ind, tensor_info); + _const_mgr->buildTensor(ind, tensor_info, layout); _ind_to_mgr.insert({ind, *_const_mgr}); } else { - _nonconst_mgr->buildTensor(ind, tensor_info); + _nonconst_mgr->buildTensor(ind, tensor_info, layout); _ind_to_mgr.insert({ind, *_nonconst_mgr}); } } diff --git a/runtime/neurun/backend/srcn/TensorManager.h b/runtime/neurun/backend/srcn/TensorManager.h index 61a10d2..37559fe 100644 --- a/runtime/neurun/backend/srcn/TensorManager.h +++ b/runtime/neurun/backend/srcn/TensorManager.h @@ -40,7 +40,7 @@ public: void deallocateNonconsts(void) override; void buildTensor(const model::OperandIndex &ind, const model::OperandInfo &tensor_info, - bool as_const); + model::Layout layout, bool as_const); void claimPlan(const model::OperandIndex &ind, uint32_t size); void releasePlan(const model::OperandIndex &ind); diff --git a/runtime/neurun/backend/srcn/operand/Tensor.h b/runtime/neurun/backend/srcn/operand/Tensor.h index 762f738..3ef3ac3 100644 --- a/runtime/neurun/backend/srcn/operand/Tensor.h +++ b/runtime/neurun/backend/srcn/operand/Tensor.h @@ -18,6 +18,7 @@ #define __NEURUN_BACKEND_SRCN_OPERAND_TENSOR_H__ #include +#include #include "model/OperandInfo.h" namespace neurun @@ -35,7 +36,7 @@ public: Tensor() = delete; public: - Tensor(const model::OperandInfo &info) : _info(info) + Tensor(const model::OperandInfo &info, model::Layout layout) : _info(info), _layout(layout) { // DO NOTHING } @@ -60,12 +61,13 @@ public: size_t num_dimensions() const override { return _info.shape().rank(); } size_t total_size() const override { return _info.total_size(); } size_t calcOffset(const neurun::util::Coordinates &coords) const override; - model::Layout layout() const override { return model::Layout::NHWC; } + model::Layout layout() const override { return _layout; } bool has_padding() const override { return false; } private: model::OperandInfo _info; uint8_t *_buffer = nullptr; + model::Layout _layout; }; } // namespace operand -- 2.7.4