[moco-tf] introduce Import (#3981)

* [moco-tf] introduce Import

This will introduce Import class part from Frontend that converts Tensorflow graphdef to loco graph

Signed-off-by: SaeHie Park <saehie.park@samsung.com>
* remove cleanup

* add final

diff --git a/contrib/moco-tf/src/Import.cpp b/contrib/moco-tf/src/Import.cpp
new file mode 100644 (file)
index 0000000..dd8b76b
--- /dev/null
+++ b/contrib/moco-tf/src/Import.cpp
@@ -0,0 +1,285 @@
+/*
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Import.h"
+
+#include "GraphBuilder.h"
+#include "GraphBuilderContext.h"
+#include "GraphBuilderRegistry.h"
+#include "Transforms.h"
+
+#include "Annotations/ShapeInferenceData.h"
+
+#include <moco/Log.h>
+
+#include <loco/IR/Verifier.h>
+#include <locop/FormattedGraph.h>
+#include <stdex/Memory.h>
+
+#include <cassert>
+#include <sstream>
+#include <stdexcept>
+
+namespace
+{
+
+void convert_graph(const moco::tf::ModelSignature &signature, tensorflow::GraphDef &tf_graph_def,
+                   loco::Graph *graph)
+{
+  auto nodedef = stdex::make_unique<moco::tf::NodeDefTable>();
+  auto tensor_names = stdex::make_unique<moco::tf::SymbolTable>();
+  auto updates = stdex::make_unique<moco::tf::UpdateQueue>();
+
+  moco::tf::GraphBuilderContext gb_context(graph, nodedef.get(), tensor_names.get(), updates.get());
+
+  // Building a loco graph
+  // 1. Convert all the nodes to loco::Node
+  // 2. Connect inputs: set all node input(from a string) to actual node object
+  // 3. Set graph input
+  // 4. Create loco::Push node and set input and set graph output
+
+  /**
+   * @brief Prepare tensorflow::NodeDef search table from name
+   */
+  for (const auto &n : tf_graph_def.node())
+  {
+    nodedef->enroll(n.name(), &n);
+  }
+
+  /**
+   * @brief 1. Convert all the nodes to loco::Node
+   *
+   * @note In each build for a TF node, four things happen
+   *       1) create corresponding loco::Node(s)
+   *       2) read and set the attributes to created loco::Node(s)
+   *       3) register name-loco::Node(last one of Nodes) that will be used as the output
+   *       4) queue a task to set the input of the loco::Node(first one of the Nodes)
+   *          this is done only for required nodes depending on the operator
+   *
+   * @example Placeholder("in") - Identity("out")
+   *        %1 = Pull --> 0x1001 (loco::Node* object address)
+   *        (symboltable: register %1, after the registeration table will contain as below;
+   *           "in" : 0x1001
+   *        )
+   *        (queue: this will be empty as Pull does not queue a task to set input;
+   *        )
+   *
+   *        %2 = Forward --> 0x1002
+   *        (symboltable: register %2 and table will look like below;
+   *           "in" : 0x1001
+   *           "out" : 0x1002
+   *        )
+   *        (queue: Forward will queue a task with input "in";
+   *           0x1002: {"in"}
+   *        )
+   */
+  for (const auto &n : tf_graph_def.node())
+  {
+    if (const auto *graph_builder = moco::tf::GraphBuilderRegistry::get().lookup(n.op()))
+    {
+      if (!graph_builder->validate(n))
+      {
+        throw std::runtime_error{"Invalid operator: " + n.op()};
+      }
+
+      graph_builder->build(n, &gb_context);
+    }
+    else
+    {
+      throw std::runtime_error{"Not supported: " + n.op()};
+    }
+  }
+
+  /**
+   * @brief 2. Connect inputs: Iterate updates and call each update input method
+   *
+   * @note  Continue from above example graph, connecting inputs is done in following steps
+   *        a) iterate queue
+   *        b) call the input method for each update
+   *        c) each update has the loco::Node *node and names of the input to connect
+   *           node = 0x1002 and names = {"in"}
+   *        d) from symbol table, "in" will return 0x1001
+   *        e) set input of 0x1002 with 0x1001
+   */
+  for (auto &update : updates->queue())
+  {
+    update->input(tensor_names.get());
+  }
+
+  /**
+   * @brief 3. Set graph input
+   */
+  for (auto input : signature.inputs())
+  {
+    auto node = tensor_names->node(input);
+    assert(node != nullptr);
+
+    auto graph_input = graph->inputs()->create();
+
+    loco::Pull *pull_node = dynamic_cast<loco::Pull *>(node);
+    assert(pull_node != nullptr);
+
+    graph_input->name(input.nodeName());
+    graph_input->node(pull_node);
+  }
+
+  /**
+   * @brief 4. Create loco::Push node and set graph input and output
+   */
+  for (auto output : signature.outputs())
+  {
+    auto output_node = tensor_names->node(output);
+    assert(output_node);
+
+    // create loco::Push for output of graph
+    auto push_node = graph->nodes()->create<loco::Push>();
+    push_node->from(output_node); // set input of Push to output node
+
+    // set the graph output name and node object
+    auto graph_output = graph->outputs()->create();
+    graph_output->name(output.nodeName());
+    graph_output->node(push_node);
+  }
+
+  // validate graph
+  assert(loco::valid(graph));
+}
+
+void dump_shapeinferencedata(loco::Node *node, const std::string &name)
+{
+  LOGGER(node_shapeinferencedata);
+
+  const moco::tf::ShapeInferenceData *shapedata = node->annot<moco::tf::ShapeInferenceData>();
+  if (shapedata == nullptr)
+  {
+    INFO(node_shapeinferencedata) << "ShapeInferenceData is null for " << name << ":" << node
+                                  << std::endl;
+  }
+  else
+  {
+    std::stringstream ss;
+
+    ss << "ShapeInferenceData for " << name << ":" << node;
+    // clang-format off
+    switch (shapedata->domain())
+    {
+    case loco::Domain::Tensor:  ss << " (Tensor)"; break;
+    case loco::Domain::Feature: ss << " (Feature)"; break;
+    case loco::Domain::Filter:  ss << " (Filter)"; break;
+    case loco::Domain::Bias:    ss << " (Bias)"; break;
+    default: assert(false && "Unknown Domain"); break;
+    }
+    // clang-format on
+    ss << " rank(" << shapedata->rank() << ") [";
+    for (uint32_t index = 0; index < shapedata->rank(); ++index)
+    {
+      if (index)
+        ss << ",";
+      if (shapedata->dim(index).known())
+        ss << shapedata->dim(index).value();
+      else
+        ss << "?";
+    }
+    ss << "]";
+
+    INFO(node_shapeinferencedata) << ss.str() << std::endl;
+  }
+}
+
+void transform_graph(loco::Graph *graph)
+{
+  LOGGER(transform_graph);
+
+  std::vector<std::unique_ptr<moco::tf::Transform>> prepare;
+  std::vector<std::unique_ptr<moco::tf::Transform>> transforms;
+
+  // Transforms that run only once for preparation and finalization
+  {
+      // TODO add one time preparation when needed
+  }
+
+  // Transforms that run multiple times until there is no transform occured
+  {
+    transforms.emplace_back(stdex::make_unique<moco::tf::FixShapeTransform>());
+    transforms.emplace_back(stdex::make_unique<moco::tf::FixPaddingTransform>());
+    // TODO add more TensorFlow related transformations
+  }
+
+  // Run preparation
+  for (auto &tr : prepare)
+  {
+    tr->run(graph);
+  }
+
+  bool changed;
+  do
+  {
+    changed = false;
+
+    for (auto &tr : transforms)
+    {
+      INFO(transform_graph) << "Before transform";
+      INFO(transform_graph) << locop::fmt<locop::LinearV1>(graph);
+
+      if (tr->run(graph))
+        changed = true;
+
+      INFO(transform_graph) << "After transform (changed: " << (changed ? 'Y' : 'N') << ")";
+      INFO(transform_graph) << locop::fmt<locop::LinearV1>(graph);
+    }
+
+  } while (changed);
+
+  // TODO would be better to run this code only when log is enabled
+  {
+    for (uint32_t i = 0; i < graph->outputs()->size(); ++i)
+    {
+      loco::Node *node = graph->outputs()->at(i)->node();
+      std::string name = "Output(" + std::to_string(i) + ")";
+      dump_shapeinferencedata(node, name);
+    }
+  }
+
+  // validate graph
+  assert(loco::valid(graph));
+}
+
+} // namespace
+
+namespace moco
+{
+namespace tf
+{
+
+Import::Import()
+{
+  // DO NOTHING
+}
+
+std::unique_ptr<loco::Graph> Import::load(const ModelSignature &signature,
+                                          tensorflow::GraphDef &tf_graph_def) const
+{
+  auto graph = loco::make_graph();
+
+  convert_graph(signature, tf_graph_def, graph.get());
+
+  transform_graph(graph.get());
+
+  return std::move(graph);
+}
+
+} // namespace tf
+} // namespace moco

diff --git a/contrib/moco-tf/src/Import.h b/contrib/moco-tf/src/Import.h
new file mode 100644 (file)
index 0000000..115f44f
--- /dev/null
+++ b/contrib/moco-tf/src/Import.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IMPORT_H__
+#define __IMPORT_H__
+
+#include <moco/tf/Frontend.h>
+#include <moco/tf/Names.h>
+
+#include <loco.h>
+
+#include <tensorflow/core/framework/graph.pb.h>
+
+#include <memory>
+
+namespace moco
+{
+namespace tf
+{
+
+class Import final
+{
+public:
+  Import();
+
+public:
+  std::unique_ptr<loco::Graph> load(const ModelSignature &, tensorflow::GraphDef &) const;
+};
+
+} // namespace tf
+} // namespace moco
+
+#endif // __IMPORT_H__

diff --git a/contrib/moco-tf/src/Import.test.cpp b/contrib/moco-tf/src/Import.test.cpp
new file mode 100644 (file)
index 0000000..412f59d
--- /dev/null
+++ b/contrib/moco-tf/src/Import.test.cpp
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Import.h"
+
+#include "TestHelper.h"
+
+#include <loco.h>
+
+#include <gtest/gtest.h>
+
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <google/protobuf/text_format.h>
+
+#include <cstring>
+
+using namespace moco::tf::test;
+
+TEST(TensorFlowImport, Dummy) { moco::tf::Import import; }
+
+namespace
+{
+
+// clang-format off
+const char *basic_pbtxtdata = STRING_CONTENT(
+node {
+  name: "Placeholder"
+  op: "Placeholder"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "shape"
+    value {
+      shape {
+        dim {
+          size: 1
+        }
+        dim {
+          size: 2
+        }
+        dim {
+          size: 1
+        }
+        dim {
+          size: 2
+        }
+      }
+    }
+  }
+}
+node {
+  name: "output/identity"
+  op: "Identity"
+  input: "Placeholder"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+);
+// clang-format on
+
+} // namespace
+
+TEST(TensorFlowImport, load_model_withio)
+{
+  moco::tf::Import import;
+  moco::tf::ModelSignature signature;
+
+  imemstream mempb(basic_pbtxtdata, std::strlen(basic_pbtxtdata));
+
+  signature.add_input(moco::tf::TensorName("Placeholder", 0));
+  signature.add_output(moco::tf::TensorName("output/identity", 0));
+
+  tensorflow::GraphDef graph_def;
+  google::protobuf::io::IstreamInputStream iis(&mempb);
+  google::protobuf::TextFormat::Parse(&iis, &graph_def);
+  std::unique_ptr<loco::Graph> graph = import.load(signature, graph_def);
+
+  loco::Graph::InputContext *inputs = graph->inputs();
+  ASSERT_EQ(inputs->size(), 1);
+  loco::GraphInput *input = inputs->at(0);
+  loco::Pull *pull = input->node();
+  ASSERT_EQ(pull->dtype(), loco::DataType::FLOAT32);
+  ASSERT_EQ(pull->rank(), 4);
+  loco::Dimension dim1 = loco::make_dimension(1);
+  loco::Dimension dim2 = loco::make_dimension(2);
+  ASSERT_EQ(pull->dim(0).value(), dim1.value());
+  ASSERT_EQ(pull->dim(1).value(), dim2.value());
+  ASSERT_EQ(pull->dim(2).value(), dim1.value());
+  ASSERT_EQ(pull->dim(3).value(), dim2.value());
+
+  loco::Graph::OutputContext *outputs = graph->outputs();
+  ASSERT_EQ(outputs->size(), 1);
+  loco::GraphOutput *output = outputs->at(0);
+  loco::Push *push = output->node();
+  // Currently we don't know the shape of output node(s)
+  // ASSERT_EQ(push->rank(), 4);
+  // ASSERT_EQ(push->dim(0).value(), dim1.value());
+  // ASSERT_EQ(push->dim(1).value(), dim2.value());
+  // ASSERT_EQ(push->dim(2).value(), dim1.value());
+  // ASSERT_EQ(push->dim(3).value(), dim2.value());
+
+  loco::Graph::NodeContext *nodes = graph->nodes();
+  ASSERT_EQ(nodes->size(), 3);
+  loco::Pull *node0 = dynamic_cast<loco::Pull *>(nodes->at(0));
+  ASSERT_EQ(node0, pull);
+  loco::Push *node2 = dynamic_cast<loco::Push *>(nodes->at(2));
+  ASSERT_EQ(node2, push);
+  loco::Forward *node1 = dynamic_cast<loco::Forward *>(nodes->at(1));
+  ASSERT_NE(node1, nullptr);
+}