From: 박세희/On-Device Lab(SR)/Principal Engineer/삼성전자 Date: Wed, 26 Jun 2019 08:15:06 +0000 (+0900) Subject: [moco-tf] introduce Import (#3981) X-Git-Tag: nncc_backup~261 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=95db4d2e9bfedaf8638c30988aef2d6820d986cb;p=platform%2Fcore%2Fml%2Fnnfw.git [moco-tf] introduce Import (#3981) * [moco-tf] introduce Import This will introduce Import class part from Frontend that converts Tensorflow graphdef to loco graph Signed-off-by: SaeHie Park * remove cleanup * add final --- diff --git a/contrib/moco-tf/src/Import.cpp b/contrib/moco-tf/src/Import.cpp new file mode 100644 index 0000000..dd8b76b --- /dev/null +++ b/contrib/moco-tf/src/Import.cpp @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Import.h" + +#include "GraphBuilder.h" +#include "GraphBuilderContext.h" +#include "GraphBuilderRegistry.h" +#include "Transforms.h" + +#include "Annotations/ShapeInferenceData.h" + +#include + +#include +#include +#include + +#include +#include +#include + +namespace +{ + +void convert_graph(const moco::tf::ModelSignature &signature, tensorflow::GraphDef &tf_graph_def, + loco::Graph *graph) +{ + auto nodedef = stdex::make_unique(); + auto tensor_names = stdex::make_unique(); + auto updates = stdex::make_unique(); + + moco::tf::GraphBuilderContext gb_context(graph, nodedef.get(), tensor_names.get(), updates.get()); + + // Building a loco graph + // 1. Convert all the nodes to loco::Node + // 2. Connect inputs: set all node input(from a string) to actual node object + // 3. Set graph input + // 4. Create loco::Push node and set input and set graph output + + /** + * @brief Prepare tensorflow::NodeDef search table from name + */ + for (const auto &n : tf_graph_def.node()) + { + nodedef->enroll(n.name(), &n); + } + + /** + * @brief 1. Convert all the nodes to loco::Node + * + * @note In each build for a TF node, four things happen + * 1) create corresponding loco::Node(s) + * 2) read and set the attributes to created loco::Node(s) + * 3) register name-loco::Node(last one of Nodes) that will be used as the output + * 4) queue a task to set the input of the loco::Node(first one of the Nodes) + * this is done only for required nodes depending on the operator + * + * @example Placeholder("in") - Identity("out") + * %1 = Pull --> 0x1001 (loco::Node* object address) + * (symboltable: register %1, after the registeration table will contain as below; + * "in" : 0x1001 + * ) + * (queue: this will be empty as Pull does not queue a task to set input; + * ) + * + * %2 = Forward --> 0x1002 + * (symboltable: register %2 and table will look like below; + * "in" : 0x1001 + * "out" : 0x1002 + * ) + * (queue: Forward will queue a task with input "in"; + * 0x1002: {"in"} + * ) + */ + for (const auto &n : tf_graph_def.node()) + { + if (const auto *graph_builder = moco::tf::GraphBuilderRegistry::get().lookup(n.op())) + { + if (!graph_builder->validate(n)) + { + throw std::runtime_error{"Invalid operator: " + n.op()}; + } + + graph_builder->build(n, &gb_context); + } + else + { + throw std::runtime_error{"Not supported: " + n.op()}; + } + } + + /** + * @brief 2. Connect inputs: Iterate updates and call each update input method + * + * @note Continue from above example graph, connecting inputs is done in following steps + * a) iterate queue + * b) call the input method for each update + * c) each update has the loco::Node *node and names of the input to connect + * node = 0x1002 and names = {"in"} + * d) from symbol table, "in" will return 0x1001 + * e) set input of 0x1002 with 0x1001 + */ + for (auto &update : updates->queue()) + { + update->input(tensor_names.get()); + } + + /** + * @brief 3. Set graph input + */ + for (auto input : signature.inputs()) + { + auto node = tensor_names->node(input); + assert(node != nullptr); + + auto graph_input = graph->inputs()->create(); + + loco::Pull *pull_node = dynamic_cast(node); + assert(pull_node != nullptr); + + graph_input->name(input.nodeName()); + graph_input->node(pull_node); + } + + /** + * @brief 4. Create loco::Push node and set graph input and output + */ + for (auto output : signature.outputs()) + { + auto output_node = tensor_names->node(output); + assert(output_node); + + // create loco::Push for output of graph + auto push_node = graph->nodes()->create(); + push_node->from(output_node); // set input of Push to output node + + // set the graph output name and node object + auto graph_output = graph->outputs()->create(); + graph_output->name(output.nodeName()); + graph_output->node(push_node); + } + + // validate graph + assert(loco::valid(graph)); +} + +void dump_shapeinferencedata(loco::Node *node, const std::string &name) +{ + LOGGER(node_shapeinferencedata); + + const moco::tf::ShapeInferenceData *shapedata = node->annot(); + if (shapedata == nullptr) + { + INFO(node_shapeinferencedata) << "ShapeInferenceData is null for " << name << ":" << node + << std::endl; + } + else + { + std::stringstream ss; + + ss << "ShapeInferenceData for " << name << ":" << node; + // clang-format off + switch (shapedata->domain()) + { + case loco::Domain::Tensor: ss << " (Tensor)"; break; + case loco::Domain::Feature: ss << " (Feature)"; break; + case loco::Domain::Filter: ss << " (Filter)"; break; + case loco::Domain::Bias: ss << " (Bias)"; break; + default: assert(false && "Unknown Domain"); break; + } + // clang-format on + ss << " rank(" << shapedata->rank() << ") ["; + for (uint32_t index = 0; index < shapedata->rank(); ++index) + { + if (index) + ss << ","; + if (shapedata->dim(index).known()) + ss << shapedata->dim(index).value(); + else + ss << "?"; + } + ss << "]"; + + INFO(node_shapeinferencedata) << ss.str() << std::endl; + } +} + +void transform_graph(loco::Graph *graph) +{ + LOGGER(transform_graph); + + std::vector> prepare; + std::vector> transforms; + + // Transforms that run only once for preparation and finalization + { + // TODO add one time preparation when needed + } + + // Transforms that run multiple times until there is no transform occured + { + transforms.emplace_back(stdex::make_unique()); + transforms.emplace_back(stdex::make_unique()); + // TODO add more TensorFlow related transformations + } + + // Run preparation + for (auto &tr : prepare) + { + tr->run(graph); + } + + bool changed; + do + { + changed = false; + + for (auto &tr : transforms) + { + INFO(transform_graph) << "Before transform"; + INFO(transform_graph) << locop::fmt(graph); + + if (tr->run(graph)) + changed = true; + + INFO(transform_graph) << "After transform (changed: " << (changed ? 'Y' : 'N') << ")"; + INFO(transform_graph) << locop::fmt(graph); + } + + } while (changed); + + // TODO would be better to run this code only when log is enabled + { + for (uint32_t i = 0; i < graph->outputs()->size(); ++i) + { + loco::Node *node = graph->outputs()->at(i)->node(); + std::string name = "Output(" + std::to_string(i) + ")"; + dump_shapeinferencedata(node, name); + } + } + + // validate graph + assert(loco::valid(graph)); +} + +} // namespace + +namespace moco +{ +namespace tf +{ + +Import::Import() +{ + // DO NOTHING +} + +std::unique_ptr Import::load(const ModelSignature &signature, + tensorflow::GraphDef &tf_graph_def) const +{ + auto graph = loco::make_graph(); + + convert_graph(signature, tf_graph_def, graph.get()); + + transform_graph(graph.get()); + + return std::move(graph); +} + +} // namespace tf +} // namespace moco diff --git a/contrib/moco-tf/src/Import.h b/contrib/moco-tf/src/Import.h new file mode 100644 index 0000000..115f44f --- /dev/null +++ b/contrib/moco-tf/src/Import.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __IMPORT_H__ +#define __IMPORT_H__ + +#include +#include + +#include + +#include + +#include + +namespace moco +{ +namespace tf +{ + +class Import final +{ +public: + Import(); + +public: + std::unique_ptr load(const ModelSignature &, tensorflow::GraphDef &) const; +}; + +} // namespace tf +} // namespace moco + +#endif // __IMPORT_H__ diff --git a/contrib/moco-tf/src/Import.test.cpp b/contrib/moco-tf/src/Import.test.cpp new file mode 100644 index 0000000..412f59d --- /dev/null +++ b/contrib/moco-tf/src/Import.test.cpp @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Import.h" + +#include "TestHelper.h" + +#include + +#include + +#include +#include +#include + +#include + +using namespace moco::tf::test; + +TEST(TensorFlowImport, Dummy) { moco::tf::Import import; } + +namespace +{ + +// clang-format off +const char *basic_pbtxtdata = STRING_CONTENT( +node { + name: "Placeholder" + op: "Placeholder" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "shape" + value { + shape { + dim { + size: 1 + } + dim { + size: 2 + } + dim { + size: 1 + } + dim { + size: 2 + } + } + } + } +} +node { + name: "output/identity" + op: "Identity" + input: "Placeholder" + attr { + key: "T" + value { + type: DT_FLOAT + } + } +} +); +// clang-format on + +} // namespace + +TEST(TensorFlowImport, load_model_withio) +{ + moco::tf::Import import; + moco::tf::ModelSignature signature; + + imemstream mempb(basic_pbtxtdata, std::strlen(basic_pbtxtdata)); + + signature.add_input(moco::tf::TensorName("Placeholder", 0)); + signature.add_output(moco::tf::TensorName("output/identity", 0)); + + tensorflow::GraphDef graph_def; + google::protobuf::io::IstreamInputStream iis(&mempb); + google::protobuf::TextFormat::Parse(&iis, &graph_def); + std::unique_ptr graph = import.load(signature, graph_def); + + loco::Graph::InputContext *inputs = graph->inputs(); + ASSERT_EQ(inputs->size(), 1); + loco::GraphInput *input = inputs->at(0); + loco::Pull *pull = input->node(); + ASSERT_EQ(pull->dtype(), loco::DataType::FLOAT32); + ASSERT_EQ(pull->rank(), 4); + loco::Dimension dim1 = loco::make_dimension(1); + loco::Dimension dim2 = loco::make_dimension(2); + ASSERT_EQ(pull->dim(0).value(), dim1.value()); + ASSERT_EQ(pull->dim(1).value(), dim2.value()); + ASSERT_EQ(pull->dim(2).value(), dim1.value()); + ASSERT_EQ(pull->dim(3).value(), dim2.value()); + + loco::Graph::OutputContext *outputs = graph->outputs(); + ASSERT_EQ(outputs->size(), 1); + loco::GraphOutput *output = outputs->at(0); + loco::Push *push = output->node(); + // Currently we don't know the shape of output node(s) + // ASSERT_EQ(push->rank(), 4); + // ASSERT_EQ(push->dim(0).value(), dim1.value()); + // ASSERT_EQ(push->dim(1).value(), dim2.value()); + // ASSERT_EQ(push->dim(2).value(), dim1.value()); + // ASSERT_EQ(push->dim(3).value(), dim2.value()); + + loco::Graph::NodeContext *nodes = graph->nodes(); + ASSERT_EQ(nodes->size(), 3); + loco::Pull *node0 = dynamic_cast(nodes->at(0)); + ASSERT_EQ(node0, pull); + loco::Push *node2 = dynamic_cast(nodes->at(2)); + ASSERT_EQ(node2, push); + loco::Forward *node1 = dynamic_cast(nodes->at(1)); + ASSERT_NE(node1, nullptr); +}