2 * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "luci/Importer.h"
18 #include "CircleImportMetadata.h"
19 #include "PostImport.h"
21 #include "luci/Import/GraphBuilder.h"
22 #include "luci/Import/GraphBuilderContext.h"
23 #include "luci/Import/GraphBuilderRegistry.h"
24 #include "luci/Import/CircleReader.h"
25 #include "luci/Import/Nodes/CircleConst.h"
27 #include <luci/IR/Module.h>
28 #include <luci/IR/CircleNodes.h>
29 #include <luci/Profile/CircleNodeID.h>
30 #include <luci/Profile/CircleNodeOrigin.h>
31 #include <luci/Plan/CircleNodeExecutionPlan.h>
33 #include <luci/LogHelper.h>
35 #include <oops/InternalExn.h>
36 #include <oops/UserExn.h>
43 void convert_graph(const luci::GraphBuilderSource &source, luci::CircleReader &reader,
48 auto nodefinder = std::make_unique<luci::IndexNodeFinder>();
49 auto tensoroutputs = std::make_unique<luci::IndexTensorOutputs>();
51 luci::GraphBuilderContext gb_context(graph, &reader, nodefinder.get(), tensoroutputs.get());
53 const auto &operators = reader.operators();
54 const auto &tensors = reader.tensors();
55 auto tensors_ptr = reader.tensors_ptr();
56 assert(tensors_ptr != nullptr);
57 auto circle_metadata = std::make_unique<luci::CircleImportMetadata>(reader);
59 // build a cache to identify if a tensor is output of an operator
60 // if this is set, we should not create a CircleConst for this tensor
61 for (uint32_t i = 0; i < operators.size(); ++i)
63 const circle::OperatorT &op = *operators[i];
64 const auto &outputs = op.outputs;
66 for (uint32_t j = 0; j < outputs.size(); ++j)
68 auto tidx = outputs[j];
69 tensoroutputs->enroll(tidx);
73 // graph inputs; there are no input nodes in TFlite but just Tensors
74 // creating virtual input nodes will make possible to connect nodes that uses them
75 // all attributes of tensor should be copied to CircleInput node
76 for (const auto input : reader.inputs())
78 auto input_node = graph->nodes()->create<luci::CircleInput>();
79 assert(input_node != nullptr);
80 const circle::TensorT &tensor = *tensors[input];
82 luci::copy_tensor_attributes(tensor, input_node);
83 if (tensors_ptr->Get(input)->shape() == nullptr)
84 input_node->shape_status(luci::ShapeStatus::NOSHAPE);
86 input_node->shape_status(luci::ShapeStatus::VALID);
88 INFO(l) << "[luci] NodeFinder INPUT(" << input << ") = " << input_node << std::endl;
89 nodefinder->enroll(input, input_node);
91 // input_node is also an output to a tensor
92 tensoroutputs->enroll(input);
95 auto graph_input = graph->inputs()->create();
96 graph_input->name(input_node->name());
98 // Set GraphInputOutputIndex for graph
99 input_node->index(graph_input->index());
102 graph_input->dtype(input_node->dtype());
104 assert(tensor.shape_signature.size() == 0 ||
105 tensor.shape_signature.size() == tensor.shape.size());
107 // Shape of GraphInput
108 auto input_shape = std::make_unique<loco::TensorShape>();
109 const std::vector<int32_t> &input_dims = tensor.shape; // in NHWC
110 input_shape->rank(input_dims.size());
111 for (uint32_t r = 0; r < input_dims.size(); ++r)
113 if (tensor.shape_signature.size() > 0 && tensor.shape_signature.at(r) == -1)
114 input_shape->dim(r).unset();
116 input_shape->dim(r).set(input_dims[r]);
118 graph_input->shape(std::move(input_shape));
121 // Create CircleConst nodes for constant tensors.
122 // NOTE Origin is intentionally not provided for constants.
123 for (uint32_t i = 0; i < tensors.size(); ++i)
125 luci::CircleConst *const_node = luci::create_circleconst(&gb_context, i);
126 if (const_node != nullptr)
127 nodefinder->enroll(i, const_node);
130 // Import the operators.
131 // Note that operators in model are stored in execution order. This means that when importing
132 // an operator, its input operators have already been imported. We exploit this fact to set up
133 // node's inputs right after creating the node.
134 auto origin_table = circle_metadata->origin_table();
135 for (uint32_t i = 0; i < operators.size(); ++i)
137 const circle::OperatorT &op = *operators[i];
138 circle::BuiltinOperator builtincode = reader.builtin_code(op);
140 if (const auto *builder = source.lookup(builtincode))
142 luci::GraphBuilder::ValidateArgs args(op, reader);
143 if (!builder->validate(args))
145 throw oops::UserExn("Invalid operator", reader.opcode_name(op));
148 auto built_op = builder->build(op, &gb_context);
149 set_node_id(built_op, i);
150 if (origin_table.find(i) != origin_table.end())
151 add_origin(built_op, origin_table.at(i));
153 add_origin(built_op, luci::single_origin(i, built_op->name()));
157 throw oops::UserExn("Not supported", reader.opcode_name(op));
162 for (auto output : reader.outputs())
164 const circle::TensorT &tensor = *tensors[output];
166 auto output_node = graph->nodes()->create<luci::CircleOutput>();
167 assert(output_node != nullptr);
168 auto output_from = nodefinder->node(output);
169 if (output_from != nullptr)
170 output_node->from(output_from);
173 // NOTE loco::Graph requires all input node(s) to a node should exist.
174 // Here, CircleOutput needs an input node.
175 // We add a dummy node to make it happy.
176 auto output_dummy = graph->nodes()->create<luci::CircleOutputDummy>();
177 assert(output_dummy != nullptr);
178 output_node->from(output_dummy);
180 luci::copy_tensor_attributes(tensor, output_dummy);
181 if (tensors_ptr->Get(output)->shape() == nullptr)
182 output_dummy->shape_status(luci::ShapeStatus::NOSHAPE);
184 output_dummy->shape_status(luci::ShapeStatus::VALID);
187 INFO(l) << "[luci] NodeFinder OUTPUT(" << output << ") = " << output_node << std::endl;
189 // set the graph output name and node object
190 auto graph_output = graph->outputs()->create();
191 std::string tname = luci::tensor_name(tensor);
192 assert(tname.length() > 0);
193 graph_output->name(tname);
195 luci::copy_tensor_attributes(tensor, output_node);
197 // Set GraphInputOutputIndex for graph
198 output_node->index(graph_output->index());
200 assert(tensor.shape_signature.size() == 0 ||
201 tensor.shape_signature.size() == tensor.shape.size());
204 auto output_shape = std::make_unique<loco::TensorShape>();
205 const std::vector<int32_t> &output_dims = tensor.shape; // in NHWC
206 output_shape->rank(output_dims.size());
207 for (uint32_t r = 0; r < output_dims.size(); ++r)
209 if (tensor.shape_signature.size() > 0 && tensor.shape_signature.at(r) == -1)
210 output_shape->dim(r).unset();
212 output_shape->dim(r).set(output_dims[r]);
214 graph_output->shape(std::move(output_shape));
217 auto dtype = luci::luci_datatype(tensor.type);
218 graph_output->dtype(dtype);
222 class ValidateCollector final : public loco::ErrorListener
225 void notify(const loco::ErrorDetail<loco::ErrorCategory::MissingArgument> &d) override
228 INFO(l) << "[luci] GraphValidate error " << d.node() << "(" << d.index() << ")" << std::endl;
242 std::unique_ptr<loco::Graph> Importer::import(const circle::Model *model) const
244 auto graph = loco::make_graph();
246 const GraphBuilderSource *source_ptr = &GraphBuilderRegistry::get();
248 if (_source != nullptr)
250 // Use user-defined GraphBuilderSource
251 source_ptr = _source;
255 if (!reader.parse(model))
258 if (reader.num_subgraph() != 1)
260 INTERNAL_EXN("Use 'importModule()' for multiple subgraphs");
262 if (!reader.select_subgraph(0))
265 // Convert circle::Model to loco::Graph
266 convert_graph(*source_ptr, reader, graph.get());
269 VERBOSE(l, 3) << "--- graph dump begin -------------------------------------------";
270 VERBOSE(l, 3) << "Name: " << graph->name();
271 VERBOSE(l, 3) << fmt(graph.get());
272 VERBOSE(l, 3) << "--- graph dump end ---------------------------------------------";
274 assert(loco::valid(graph.get(), std::make_unique<ValidateCollector>()));
279 std::unique_ptr<Module> Importer::importModule(const circle::Model *model) const
281 auto module = make_module();
283 const GraphBuilderSource *source_ptr = &GraphBuilderRegistry::get();
285 if (_source != nullptr)
287 // Use user-defined GraphBuilderSource
288 source_ptr = _source;
292 if (!reader.parse(model))
295 for (uint32_t g = 0; g < reader.num_subgraph(); ++g)
297 auto graph = loco::make_graph();
299 if (!reader.select_subgraph(g))
302 graph->name(reader.name());
304 // Convert circle::Model to loco::Graph
305 convert_graph(*source_ptr, reader, graph.get());
308 VERBOSE(l, 3) << "--- graph dump begin -------------------------------------------";
309 VERBOSE(l, 3) << "Name: " << graph->name();
310 VERBOSE(l, 3) << fmt(graph.get());
311 VERBOSE(l, 3) << "--- graph dump end ---------------------------------------------";
313 assert(loco::valid(graph.get(), std::make_unique<ValidateCollector>()));
315 module->add(std::move(graph));
318 post_import_graph(module.get(), reader);
320 // Initialize 'source_table'
321 auto circle_metadata = std::make_unique<luci::CircleImportMetadata>(reader);
322 if (circle_metadata->source_table().size() > 0)
324 // If there is 'source_table' metadata in circle model, copy the table.
325 module->source_table(circle_metadata->source_table());
329 // If there is no 'source_table' metadata in circle model,
330 // create new table with circle nodes.
331 std::map<uint32_t, std::string> table;
333 // NOTE Only first subgraph is considered
334 for (auto node : loco::all_nodes(module->graph(0)))
336 auto circle_node = loco::must_cast<luci::CircleNode *>(node);
338 // Virtual nodes may not have id
339 if (!has_node_id(circle_node))
342 assert(table.find(get_node_id(circle_node)) == table.end());
343 table.insert({get_node_id(circle_node), circle_node->name()});
346 module->source_table(table);
349 // Add execution_plan annotations
350 if (circle_metadata->execution_plan_table().size() > 0)
352 auto execution_plan_table = circle_metadata->execution_plan_table();
353 auto node_position = 0;
354 for (auto node : loco::postorder_traversal(loco::output_nodes(module->graph())))
356 if (auto circle_node = dynamic_cast<luci::CircleNode *>(node))
358 auto node_plan = execution_plan_table[node_position];
359 luci::add_execution_plan(
361 luci::CircleNodeExecutionPlan(
362 node_plan[0], std::vector<uint32_t>(node_plan.begin() + 1, node_plan.end())));