From 89929377397e02967833c9f1bf26bdca32b61df8 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=9C=A4=ED=98=84=EC=8B=9D/On-Device=20Lab=28SR=29/Princip?= =?utf8?q?al=20Engineer/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Thu, 13 Jun 2019 18:26:32 +0900 Subject: [PATCH] [moco/tf] basic Conv2D (padding not yet implemented) (#3731) * [moco/tf] basic Conv2D (padding not yet implemented) This adds basic implementation of Conv2D. Padding and `FixShapeTransform` is not yet implemented. Note: To run the test, `FixShape()` for Conv2D should be implemented. So I added a minimum implementation in this commit. Signed-off-by: Hyun Sik Yoon * renaming vars * split one GraphUpdate into two * assert for padding * supporting NCHW in FeatureDecode --- contrib/moco/lib/frontend/tf/src/Op/Conv2D.cpp | 231 +++++++++++++++++++++ .../moco/lib/frontend/tf/src/Op/Conv2D.test.cpp | 219 +++++++++++++++++++ .../tf/src/Transforms/FixShapeTransform.cpp | 12 ++ .../lib/frontend/tf/src/Transforms/MocoNodes.lst | 2 + 4 files changed, 464 insertions(+) create mode 100644 contrib/moco/lib/frontend/tf/src/Op/Conv2D.cpp create mode 100644 contrib/moco/lib/frontend/tf/src/Op/Conv2D.test.cpp diff --git a/contrib/moco/lib/frontend/tf/src/Op/Conv2D.cpp b/contrib/moco/lib/frontend/tf/src/Op/Conv2D.cpp new file mode 100644 index 0000000..df2b511 --- /dev/null +++ b/contrib/moco/lib/frontend/tf/src/Op/Conv2D.cpp @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Convert.h" +#include "GraphBuilder.h" +#include "GraphBuilderContext.h" + +#include +#include +#include + +#include + +#include +#include + +namespace +{ +using namespace moco::tf; + +class IFMUpdate final : public GraphUpdate +{ +public: + IFMUpdate(loco::FeatureEncode *ifm_enc, const std::string &ifm_name) + : _ifm_enc(ifm_enc), _ifm_name(ifm_name) + { + } + + void input(const SymbolTable *) const override; + +private: + loco::FeatureEncode *_ifm_enc; + const std::string _ifm_name; +}; + +class KernelUpdate final : public GraphUpdate +{ +public: + KernelUpdate(loco::FilterEncode *ker_enc, const std::string &ker_name) + : _ker_enc(ker_enc), _ker_name(ker_name) + { + } + + void input(const SymbolTable *) const override; + +private: + loco::FilterEncode *_ker_enc; + const std::string _ker_name; +}; + +void IFMUpdate::input(const SymbolTable *node_table) const +{ + loco::Node *ifm_node = node_table->node(_ifm_name); + _ifm_enc->input(ifm_node); +} + +void KernelUpdate::input(const SymbolTable *node_table) const +{ + loco::Node *ker_node = node_table->node(_ker_name); + _ker_enc->input(ker_node); +} + +} // namespace + +namespace moco +{ +namespace tf +{ + +/** + * @brief GraphBuilder for Conv2D node + */ +class Conv2DGraphBuilder final : public GraphBuilder +{ +public: + bool validate(const tensorflow::NodeDef &) const override; + void build(const tensorflow::NodeDef &, GraphBuilderContext *) const override; +}; + +bool Conv2DGraphBuilder::validate(const tensorflow::NodeDef &node) const +{ + assert(node.input_size() == 2); + + // note: even though "data_format" is not entered when a model is written, + // TF seems to generate "data_format" field into a pb file + static std::string attrs[] = {"T", "data_format", "dilations", "padding", "strides"}; + + for (auto attr : attrs) + if (!node.attr().count(attr)) + return false; + + return true; +} + +void Conv2DGraphBuilder::build(const tensorflow::NodeDef &node, GraphBuilderContext *context) const +{ + assert(context != nullptr); + + loco::Graph *graph = context->graph(); + SymbolTable *nodes = context->nodes(); + UpdateQueue *updates = context->updates(); + + // name of loco nodes + std::string conv2d_name = node.name(); + + // tensorflow data_format, e.g., NHWC, NCHW, etc. + auto data_layout = get_data_layout(node, "data_format"); + + auto feature_enc = graph->nodes()->create(); + { + auto enc = stdex::make_unique>(); + + if (data_layout == DataLayout::NHWC) + { + enc->perm()->axis(loco::FeatureAxis::Count) = 0; + enc->perm()->axis(loco::FeatureAxis::Height) = 1; + enc->perm()->axis(loco::FeatureAxis::Width) = 2; + enc->perm()->axis(loco::FeatureAxis::Depth) = 3; + } + else if (data_layout == DataLayout::NCHW) + { + enc->perm()->axis(loco::FeatureAxis::Count) = 0; + enc->perm()->axis(loco::FeatureAxis::Depth) = 1; + enc->perm()->axis(loco::FeatureAxis::Height) = 2; + enc->perm()->axis(loco::FeatureAxis::Width) = 3; + } + else + throw std::runtime_error("Not yet supported"); + + feature_enc->encoder(std::move(enc)); + } + + auto filter_enc = graph->nodes()->create(); + { + auto enc = stdex::make_unique>(); + + // In Tensorflow, conv2d filter is a 4-D tensor of following shape: + // [filter_height, filter_width, in_channels, out_channels] -> HWIO (HWCN) + + enc->perm()->axis(loco::FilterAxis::Height) = 0; + enc->perm()->axis(loco::FilterAxis::Width) = 1; + enc->perm()->axis(loco::FilterAxis::Depth) = 2; + enc->perm()->axis(loco::FilterAxis::Count) = 3; + + filter_enc->encoder(std::move(enc)); + } + + auto conv2d = graph->nodes()->create(); + { + // let's convert attrs: + // Tensorflow attr : T, data_format, dilations, padding, strides + // to loco attr: not defined, TBD, TBD, TBD, stride + + // tf strides -> loco stride + auto tf_strides = get_list_attr(node, "strides"); + auto stride = conv2d->stride(); + + if (data_layout == DataLayout::NHWC) + { + stride->vertical(tf_strides.i(1)); + stride->horizontal(tf_strides.i(2)); + } + else if (data_layout == DataLayout::NCHW) + { + stride->vertical(tf_strides.i(2)); + stride->horizontal(tf_strides.i(3)); + } + + // TODO Consider "SAME" padding + assert(get_string_attr(node, "padding") == "VALID"); + } + + auto feature_dec = graph->nodes()->create(); + { + auto dec = stdex::make_unique>(); + + if (data_layout == DataLayout::NHWC) + { + dec->perm()->axis(loco::FeatureAxis::Count) = 0; + dec->perm()->axis(loco::FeatureAxis::Height) = 1; + dec->perm()->axis(loco::FeatureAxis::Width) = 2; + dec->perm()->axis(loco::FeatureAxis::Depth) = 3; + } + else if (data_layout == DataLayout::NCHW) + { + dec->perm()->axis(loco::FeatureAxis::Count) = 0; + dec->perm()->axis(loco::FeatureAxis::Depth) = 1; + dec->perm()->axis(loco::FeatureAxis::Height) = 2; + dec->perm()->axis(loco::FeatureAxis::Width) = 3; + } + else + throw std::runtime_error("Not supported data layout"); + + feature_dec->decoder(std::move(dec)); + } + + // link nodes + conv2d->ifm(feature_enc); + conv2d->ker(filter_enc); + feature_dec->input(conv2d); + + // To set the input node of encode_node with conv2d_name + nodes->enroll(conv2d_name, feature_dec); + + // Record ifm inputs to featureEncode_node + auto ifm_update = stdex::make_unique(feature_enc, node.input(0)); + auto ker_update = stdex::make_unique(filter_enc, node.input(1)); + + updates->enroll(std::move(ifm_update)); + updates->enroll(std::move(ker_update)); +} + +} // namespace tf +} // namespace moco + +#include "GraphBuilderRegistry.h" + +REGISTER_OP_BUILDER(Conv2D, Conv2DGraphBuilder) diff --git a/contrib/moco/lib/frontend/tf/src/Op/Conv2D.test.cpp b/contrib/moco/lib/frontend/tf/src/Op/Conv2D.test.cpp new file mode 100644 index 0000000..6fc2f04 --- /dev/null +++ b/contrib/moco/lib/frontend/tf/src/Op/Conv2D.test.cpp @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestHelper.h" + +#include + +#include +#include +#include + +#include + +#include + +#include +#include + +using namespace moco::tf::test; + +namespace +{ +// clang-format off +const char *conv2d_01_pbtxtdata = STRING_CONTENT( +node { + name: "ifm" + op: "Const" + attr { key: "dtype" value { type: DT_FLOAT } } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { size: 1 } + dim { size: 4 } + dim { size: 4 } + dim { size: 3 } + } + float_val: 1.1 + } + } + } +} +node { + name: "ker" + op: "Const" + attr { key: "dtype" value { type: DT_FLOAT } } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { size: 2 } + dim { size: 2 } + dim { size: 3 } + dim { size: 100 } + } + float_val: 1.1 + } + } + } +} +node { + name: "conv2d" + op: "Conv2D" + input: "ifm" + input: "ker" + attr { key: "T" value { type: DT_FLOAT } } + attr { key: "data_format" value { s: "NHWC" } } + attr { key: "dilations" value { list { i: 1 i: 1 i: 1 i: 1 } } } + attr { key: "padding" value { s: "VALID" } } + attr { key: "strides" value { list { i: 1 i: 2 i: 3 i: 1 } } } + attr { key: "use_cudnn_on_gpu" value { b: false } } +} +); +// clang-format on +} // namespace + +TEST(TensorFlowFrontend, Conv2D_01) +{ + moco::tf::Frontend frontend; + moco::tf::ModelSignature signature; + + imemstream mempb(conv2d_01_pbtxtdata, std::strlen(conv2d_01_pbtxtdata)); + + signature.add_output("conv2d"); + + std::unique_ptr graph = + frontend.load(signature, &mempb, moco::tf::Frontend::FileType::Text); + + // test 1. + // loco node : ConstGen - FeatureEncode -- Conv2D - FeatureDecode - Push + // ConstGen - FilterEncode / + loco::Graph::NodeContext *loco_nodes = graph->nodes(); + + loco::Graph::InputContext *loco_inputs = graph->inputs(); + ASSERT_EQ(loco_inputs->size(), 0); + ASSERT_EQ(loco_nodes->size(), 7); + + int idx = 0; + + loco::ConstGen *ifm = dynamic_cast(loco_nodes->at(idx++)); + loco::ConstGen *ker = dynamic_cast(loco_nodes->at(idx++)); + + loco::FeatureEncode *ifm_enc = dynamic_cast(loco_nodes->at(idx++)); + loco::FilterEncode *ker_enc = dynamic_cast(loco_nodes->at(idx++)); + + loco::Conv2D *conv2d = dynamic_cast(loco_nodes->at(idx++)); + loco::FeatureDecode *dec = dynamic_cast(loco_nodes->at(idx++)); + + loco::Push *push = dynamic_cast(loco_nodes->at(idx++)); + + ASSERT_NE(ifm, nullptr); + ASSERT_NE(ifm_enc, nullptr); + ASSERT_NE(ker, nullptr); + ASSERT_NE(ker_enc, nullptr); + ASSERT_NE(conv2d, nullptr); + ASSERT_NE(dec, nullptr); + ASSERT_NE(push, nullptr); + + // check their connection is all OK + ASSERT_TRUE(ifm_enc->input() == ifm); + ASSERT_TRUE(ker_enc->input() == ker); + ASSERT_TRUE(conv2d->ifm() == ifm_enc); + ASSERT_TRUE(conv2d->ker() == ker_enc); + ASSERT_TRUE(dec->input() == conv2d); + ASSERT_TRUE(push->from() == dec); + + // test 2. + // attrs inside Conv2D + { + // stride + ASSERT_EQ(conv2d->stride()->vertical(), 2); + ASSERT_EQ(conv2d->stride()->horizontal(), 3); + + // TODO add padding test + } + + // test 3. + // attrs inside FeatureEncoder + { + auto ifm_encoder = ifm_enc->encoder(); + ASSERT_TRUE(ifm_encoder != nullptr); + + loco::TensorShape tensor_shape; + tensor_shape.rank(4); + tensor_shape.dim(0) = 1; // COUNT + tensor_shape.dim(1) = 720; // HEIGHT + tensor_shape.dim(2) = 1280; // WIDTH + tensor_shape.dim(3) = 3; // DEPTH + + // Get the feature shape corresponding to a given image + auto feature_shape = ifm_encoder->shape(tensor_shape); + + ASSERT_EQ(feature_shape.count().value(), 1); + ASSERT_EQ(feature_shape.height().value(), 720); + ASSERT_EQ(feature_shape.width().value(), 1280); + ASSERT_EQ(feature_shape.depth().value(), 3); + } + + // test 4. + // attrs inside FilterEncoder + { + auto ker_encoder = ker_enc->encoder(); + ASSERT_TRUE(ker_encoder != nullptr); + + loco::TensorShape tensor_shape; + tensor_shape.rank(4); + tensor_shape.dim(0) = 2; // H + tensor_shape.dim(1) = 4; // W + tensor_shape.dim(2) = 3; // I (C) + tensor_shape.dim(3) = 7; // O (N) + + // Get the feature shape corresponding to a given image + auto ker_shape = ker_encoder->shape(tensor_shape); + + ASSERT_EQ(ker_shape.height().value(), 2); + ASSERT_EQ(ker_shape.width().value(), 4); + ASSERT_EQ(ker_shape.depth().value(), 3); + ASSERT_EQ(ker_shape.count().value(), 7); + } + + // test 5 + // attrs inside FeatureDecoder + { + auto decoder = dec->decoder(); + ASSERT_TRUE(decoder != nullptr); + + loco::FeatureShape feature_shape; + feature_shape.count() = 1; + feature_shape.height() = 720; + feature_shape.width() = 1280; + feature_shape.depth() = 3; + + // Get the tensor shape corresponding to a given image + auto tensor_shape = decoder->shape(feature_shape); + + ASSERT_EQ(tensor_shape.rank(), 4); + ASSERT_EQ(tensor_shape.dim(0).value(), 1); // COUNT + ASSERT_EQ(tensor_shape.dim(1).value(), 720); // HEIGHT + ASSERT_EQ(tensor_shape.dim(2).value(), 1280); // WIDTH + ASSERT_EQ(tensor_shape.dim(3).value(), 3); // DEPTH + } +} diff --git a/contrib/moco/lib/frontend/tf/src/Transforms/FixShapeTransform.cpp b/contrib/moco/lib/frontend/tf/src/Transforms/FixShapeTransform.cpp index d50f82c..903ff3c 100644 --- a/contrib/moco/lib/frontend/tf/src/Transforms/FixShapeTransform.cpp +++ b/contrib/moco/lib/frontend/tf/src/Transforms/FixShapeTransform.cpp @@ -91,6 +91,12 @@ bool fix_shape(loco::ConstGen *node) return true; } +bool fix_shape(loco::Conv2D *node) +{ + // TODO implement this + return false; +} + bool fix_shape(loco::FeatureDecode *node) { // Output shape is same as the input @@ -105,6 +111,12 @@ bool fix_shape(loco::FeatureEncode *node) return copy_shapedata(input, node); } +bool fix_shape(loco::FilterEncode *node) +{ + // TODO implement this + return false; +} + bool fix_shape(loco::Forward *node) { // Output shape is same as the input diff --git a/contrib/moco/lib/frontend/tf/src/Transforms/MocoNodes.lst b/contrib/moco/lib/frontend/tf/src/Transforms/MocoNodes.lst index f707709..1db45a1 100644 --- a/contrib/moco/lib/frontend/tf/src/Transforms/MocoNodes.lst +++ b/contrib/moco/lib/frontend/tf/src/Transforms/MocoNodes.lst @@ -5,8 +5,10 @@ // MOCONODE(Name) : alphabetic order please MOCONODE(ConstGen) +MOCONODE(Conv2D) MOCONODE(FeatureDecode) MOCONODE(FeatureEncode) +MOCONODE(FilterEncode) MOCONODE(Forward) MOCONODE(MaxPool2D) MOCONODE(Pull) -- 2.7.4