2 * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "ExecutorFactory.h"
20 #include "../backend/builtin/BackendContext.h"
21 #include "../backend/builtin/Config.h"
22 #include "../backend/builtin/UserTensor.h"
23 #include "../dumper/text/GraphDumper.h"
24 #include "../exec/DataflowExecutor.h"
25 #include "../exec/ExecTime.h"
26 #include "../exec/ExecutionObservers.h"
27 #include "../exec/LinearExecutor.h"
28 #include "../exec/ParallelExecutor.h"
29 #include "../ir/OperationCloner.h"
31 #include <backend/IPortableTensor.h>
32 #include <compiler/BackendManager.h>
33 #include <compiler/ExecutionBuilder.h>
34 #include <util/TracingCtx.h>
44 class SyncFunction final : public exec::IFunction
47 virtual ~SyncFunction() = default;
48 SyncFunction(std::unique_ptr<exec::IFunction> fn, const std::shared_ptr<backend::IConfig> config)
49 : _fn{std::move(fn)}, _config{config}
61 void prepare() override { _fn->prepare(); }
64 std::unique_ptr<exec::IFunction> _fn;
65 std::shared_ptr<backend::IConfig> _config;
68 using DeallocList = std::vector<backend::ITensor *>;
69 // Deallocation after execution of an operation used by Linear Executor
70 class DeallocFunction final : public exec::IFunction
73 DeallocFunction(const DeallocList &tensors) : _dealloc_list{tensors} {}
77 for (auto tensor : _dealloc_list)
79 if (!tensor->is_dynamic())
81 tensor->deallocBuffer();
86 DeallocList _dealloc_list;
89 void initializeSubgraphIOTensors(compiler::LoweredGraph &lowered_graph,
90 const backend::BackendContexts &backend_contexts,
91 const ir::OperandIndexSequence &indices)
93 // TODO Store builtin backend in BackendContext
94 std::shared_ptr<backend::builtin::TensorRegistry> builtin_tensor_reg;
95 for (const auto &e : backend_contexts)
97 auto backend = e.first;
98 auto &context = e.second;
99 if (backend->config()->id() == backend::builtin::Config::ID)
102 std::dynamic_pointer_cast<backend::builtin::TensorRegistry>(context->tensor_registry);
105 assert(builtin_tensor_reg);
107 for (auto ind : indices)
109 const auto &operand = lowered_graph.graph().operands().at(ind);
110 auto tensor = std::make_unique<backend::builtin::IOTensor>(
112 ir::Layout::NHWC /* FIXME find operation for this operand and use frontend_layout */
115 // Add tensor to builtin TensorRegistry.
116 builtin_tensor_reg->setNativeIOTensor(ind, std::move(tensor));
120 backend::BackendContexts createBackendContexts(compiler::LoweredGraph &lgraph, bool linear_executor)
122 backend::BackendContexts contexts;
123 auto &backend_manager = compiler::BackendManager::get();
125 std::unordered_map<const backend::Backend *, backend::ContextData> context_data_map;
127 // Generate partial graphs for each backend
128 for (auto backend : backend_manager.getAll())
130 auto &data = context_data_map[backend];
131 auto graph = std::make_unique<ir::Graph>();
132 graph->setLayout(lgraph.graph().layout());
133 data.graph = std::move(graph);
136 auto &whole_graph = lgraph.graph();
137 // Separate operands into partial graphs
138 whole_graph.operands().iterate([&](const ir::OperandIndex &operand_ind, ir::Operand &operand) {
139 auto &operand_li = lgraph.lower_info().operand;
140 const auto &def_factors = operand_li.at(operand_ind).def_factors();
141 if (def_factors.size() == 0) // Ignore unused tensor
143 const auto &def_factor = def_factors.getOnlyElement();
144 const auto backend = def_factor.backend();
145 auto &partial_graph = *context_data_map[backend].graph;
146 auto &operand_layouts = context_data_map[backend].operand_layouts;
147 assert(operand_layouts.find(operand_ind) == operand_layouts.end());
148 operand_layouts[operand_ind] = def_factor.layout();
150 // Copy the operand and insert it to the partial graph
151 auto new_operand = std::make_unique<ir::Operand>(operand);
152 new_operand->clearDefUse();
153 operand.releaseData(); // Deref data of LoweredGraph
154 auto new_operand_ind = partial_graph.addOperand(operand_ind, std::move(new_operand));
155 UNUSED_RELEASE(new_operand_ind);
156 assert(new_operand_ind == operand_ind);
158 // Separate operations into partial graphs
159 whole_graph.operations().iterate(
160 [&](const ir::OperationIndex &op_ind, const ir::Operation &operation) {
161 auto &op_li = lgraph.lower_info().operation;
162 auto backend = op_li.at(op_ind).backend();
163 auto &partial_graph = *context_data_map[backend].graph;
164 auto &external_operands = context_data_map[backend].external_operands;
165 auto &operand_layouts = context_data_map[backend].operand_layouts;
168 // Add missing operands (externals)
169 auto io_list = (operation.getInputs() + operation.getOutputs()) | ir::Remove::DUPLICATED |
170 ir::Remove::UNDEFINED;
171 for (auto operand_ind : io_list)
173 if (partial_graph.operands().exist(operand_ind))
176 // Copy the operand and insert it to the partial graph
177 const auto &operand = whole_graph.operands().at(operand_ind);
178 auto new_operand = std::make_unique<ir::Operand>(operand);
179 new_operand->clearDefUse();
180 auto new_operand_ind = partial_graph.addOperand(operand_ind, std::move(new_operand));
181 UNUSED_RELEASE(new_operand_ind);
182 assert(new_operand_ind == operand_ind);
185 lgraph.lower_info().operand.at(operand_ind).def_factors().getOnlyElement().layout();
186 assert(operand_layouts.find(operand_ind) == operand_layouts.end());
187 operand_layouts[operand_ind] = layout;
188 external_operands.add(operand_ind);
191 auto new_op_ind = partial_graph.addOperation(op_ind, clone(operation));
192 UNUSED_RELEASE(new_op_ind);
193 assert(new_op_ind == op_ind);
198 auto whole_op_order = lgraph.graph().topolSortOperations();
199 for (auto &&pair : context_data_map)
201 auto backend = pair.first;
202 auto &data = pair.second;
203 // Handle graph input/outputs or external tensors
204 data.graph->operands().iterate([&](const ir::OperandIndex &ind, const ir::Operand &operand) {
205 if (whole_graph.getInputs().contains(ind) || whole_graph.getOutputs().contains(ind))
206 data.external_operands.add(ind);
207 // Inputs are either "graph input" or "no def op and non-constant"
208 if (whole_graph.getInputs().contains(ind) ||
209 (!operand.getDef().valid() && !operand.isConstant()))
210 // Outputs are either "graph output" or "no uses"
211 data.graph->addInput(ind);
212 if (whole_graph.getOutputs().contains(ind) || operand.getUses().size() == 0)
213 data.graph->addOutput(ind);
215 dumper::text::dumpGraph(*data.graph);
217 std::copy_if(whole_op_order.begin(), whole_op_order.end(), std::back_inserter(data.op_order),
218 [&](const auto &ind) { return data.graph->operations().exist(ind); });
219 data.is_linear_executor = linear_executor;
220 data.custom_kernel_builder = lgraph.graph().getKernelBuilder();
221 contexts.emplace(backend, backend->newContext(std::move(data)));
234 ExecutorFactory &ExecutorFactory::get()
236 static ExecutorFactory singleton;
240 ExecutorFactory::ExecutorFactory()
242 _map["Linear"] = createLinearExecutor;
244 std::bind(createDataflowExecutor, std::placeholders::_1, std::placeholders::_2,
245 std::placeholders::_3, std::placeholders::_4, std::placeholders::_5, false);
247 std::bind(createDataflowExecutor, std::placeholders::_1, std::placeholders::_2,
248 std::placeholders::_3, std::placeholders::_4, std::placeholders::_5, true);
251 exec::IExecutor *ExecutorFactory::create(std::unique_ptr<compiler::LoweredGraph> lowered_graph,
252 const util::TracingCtx *tracing_ctx,
253 const compiler::CompilerOptions &options,
254 const std::shared_ptr<exec::IExecutors> &executors,
255 const ir::ModelIndex &index)
257 return _map.at(options.executor)(std::move(lowered_graph), tracing_ctx, options, executors,
261 void ExecutorFactory::prepareMigrantTensors(compiler::LoweredGraph &lowered_graph,
262 const backend::BackendContexts &backend_contexts)
264 TensorRegistries tensor_regs{backend_contexts, true};
266 lowered_graph.graph().operations().iterate(
267 [&](const ir::OperationIndex &op_ind, const ir::Operation &op) {
268 auto lower_info = lowered_graph.lower_info().operation.getRawPtr(op_ind);
269 auto &backend_ctx = backend_contexts.at(lower_info->backend());
271 (op.getInputs() + op.getOutputs()) | ir::Remove::DUPLICATED | ir::Remove::UNDEFINED)
273 // If an Operation's input/output tensor does not have an own tensor object,
274 // it must be using migrant tensors, so find the tensor from other tensor registries and
275 // register it to the current tensor registry if it is portable
276 if (!backend_ctx->tensor_registry->getITensor(ind))
278 auto tensor = tensor_regs.getITensor(ind);
279 assert(tensor); // The tensor must have been registered
280 auto ptensor = dynamic_cast<backend::IPortableTensor *>(tensor);
282 backend_ctx->tensor_registry->setMigrantTensor(ind, ptensor);
288 void ExecutorFactory::prepareBuiltinBackend(const TensorRegistries &tensor_regs,
289 const std::shared_ptr<exec::IExecutors> &executors,
290 const backend::BackendContexts &backend_contexts,
291 const ir::ModelIndex &index)
293 for (auto &&pair : backend_contexts)
295 auto builtin_context = dynamic_cast<backend::builtin::BackendContext *>(pair.second.get());
296 if (builtin_context != nullptr)
298 auto builtin_kernel_gen = builtin_context->kernel_gen;
299 builtin_kernel_gen->setTensorRegistries(tensor_regs);
300 builtin_kernel_gen->setExecutors(executors);
301 builtin_kernel_gen->setModelIndex(index);
306 std::deque<std::pair<const backend::Backend *, backend::BackendContext *>>
307 ExecutorFactory::orderBackendContext(const backend::BackendContexts &backend_contexts)
309 std::deque<std::pair<const backend::Backend *, backend::BackendContext *>> ordered_contexts;
311 for (auto &&pair : backend_contexts)
313 // NOTE builtin backend must be processed lastly.
314 // This is because of Permute layer's specialty which is the only operation that could have
315 // different ITensor objects for the input and the output. And it requires all other backends'
316 // tensors are ready to use.
317 if (pair.first->config()->id() == "builtin")
318 ordered_contexts.emplace_back(pair.first, pair.second.get());
320 ordered_contexts.emplace_front(pair.first, pair.second.get());
323 return ordered_contexts;
326 exec::IExecutor *ExecutorFactory::createLinearExecutor(
327 std::unique_ptr<compiler::LoweredGraph> lowered_graph, const util::TracingCtx *tracing_ctx,
328 const compiler::CompilerOptions &options, const std::shared_ptr<exec::IExecutors> &executors,
329 const ir::ModelIndex &index)
331 auto &graph = lowered_graph->graph();
333 backend::BackendContexts backend_contexts =
334 createBackendContexts(*lowered_graph, options.executor == "Linear");
336 TensorRegistries tensor_regs{backend_contexts, true};
338 initializeSubgraphIOTensors(
339 *lowered_graph, backend_contexts,
340 (lowered_graph->graph().getInputs() + lowered_graph->graph().getOutputs()) |
341 ir::Remove::DUPLICATED | ir::Remove::UNDEFINED);
344 auto order = Linear::linearize(*lowered_graph);
345 Linear::dump(*lowered_graph, order);
347 for (auto &&pair : backend_contexts)
349 pair.second->genTensors();
352 prepareMigrantTensors(*lowered_graph, backend_contexts);
354 // Give some runtime objects to builtin KernelGenerator
355 prepareBuiltinBackend(tensor_regs, executors, backend_contexts, index);
357 ExecutionBuilder builder;
359 // Adjust the order of backends for the upcoming iteration
360 auto ordered_contexts = orderBackendContext(backend_contexts);
362 // Simulate the execution for deallocation of tensors
363 std::unordered_map<ir::OperationIndex, DeallocList> dealloc_list_map;
365 ir::OperandIndexMap<uint32_t> uses_map;
366 ir::OperandIndexSequence constants;
369 (graph.getInputs() + graph.getOutputs()) | ir::Remove::UNDEFINED | ir::Remove::DUPLICATED;
372 graph.operands().iterate([&](const ir::OperandIndex &ind, const ir::Operand &obj) {
373 uses_map[ind] = obj.getUses().size();
375 if (obj.isConstant())
376 constants.append(ind);
379 // A trick to consider constants as an execption
380 for (const auto &ind : constants)
385 for (const auto op_ind : order)
387 const auto &op = graph.operations().at(op_ind);
388 auto op_inputs = op.getInputs() | ir::Remove::DUPLICATED | ir::Remove::UNDEFINED;
389 auto op_outputs = op.getOutputs() | ir::Remove::DUPLICATED | ir::Remove::UNDEFINED;
391 for (const auto &ind : op_inputs)
393 const auto &operand = graph.operands().at(ind);
394 assert(uses_map.find(ind) != uses_map.end());
395 assert(uses_map[ind] > 0);
397 if (uses_map[ind] == 0 && !operand.info().isVariable() && !model_io.contains(ind))
399 dealloc_list_map[op_ind].emplace_back(tensor_regs.getITensor(ind));
404 // Dispose and validate
405 for (const auto &ind : constants)
411 std::all_of(uses_map.begin(), uses_map.end(),
412 [](std::pair<const ir::OperandIndex, uint32_t> it) { return it.second == 0; }));
416 for (auto &&pair : ordered_contexts)
418 auto codes = pair.second->genKernels();
419 for (auto &&pair : codes)
421 auto &op_ind = pair.first;
422 auto &fn_seq = pair.second;
423 auto &op = lowered_graph->graph().operations().at(op_ind);
424 auto lower_info = lowered_graph->lower_info().operation.getRawPtr(op_ind);
425 if (options.he_profiling_mode)
426 fn_seq->wrap<SyncFunction>(lower_info->backend()->config());
427 if (!dealloc_list_map[op_ind].empty())
428 fn_seq->append(std::make_unique<DeallocFunction>(dealloc_list_map[op_ind]));
429 builder.append(op_ind, {op_ind, &op, lower_info, std::move(fn_seq)});
433 auto code_map = builder.releaseCodeMap();
435 auto exec = new exec::LinearExecutor{std::move(lowered_graph),
436 std::move(backend_contexts),
442 if (!options.trace_filepath.empty())
444 std::unique_ptr<exec::IExecutionObserver> ctp =
445 std::make_unique<exec::TracingObserver>(options.trace_filepath, exec->graph(), tracing_ctx);
446 exec->addObserver(std::move(ctp));
452 exec::IExecutor *ExecutorFactory::createDataflowExecutor(
453 std::unique_ptr<compiler::LoweredGraph> lowered_graph, const util::TracingCtx *tracing_ctx,
454 const compiler::CompilerOptions &options, const std::shared_ptr<exec::IExecutors> &executors,
455 const ir::ModelIndex &index, bool parallel)
457 backend::BackendContexts backend_contexts =
458 createBackendContexts(*lowered_graph, options.executor == "Linear");
460 TensorRegistries tensor_regs{backend_contexts, true};
462 initializeSubgraphIOTensors(
463 *lowered_graph, backend_contexts,
464 (lowered_graph->graph().getInputs() + lowered_graph->graph().getOutputs()) |
465 ir::Remove::DUPLICATED | ir::Remove::UNDEFINED);
467 for (auto &&pair : backend_contexts)
469 pair.second->genTensors();
472 prepareMigrantTensors(*lowered_graph, backend_contexts);
474 // Give some runtime objects to builtin KernelGenerator
475 prepareBuiltinBackend(tensor_regs, executors, backend_contexts, index);
477 ExecutionBuilder builder;
479 // Adjust the order of backends for the upcoming iteration
480 auto ordered_contexts = orderBackendContext(backend_contexts);
483 for (auto &&pair : ordered_contexts)
485 auto codes = pair.second->genKernels();
486 for (auto &&pair : codes)
488 auto &op_ind = pair.first;
489 auto &fn_seq = pair.second;
490 auto &op = lowered_graph->graph().operations().at(op_ind);
491 auto lower_info = lowered_graph->lower_info().operation.getRawPtr(op_ind);
492 if (options.he_profiling_mode)
493 fn_seq->wrap<SyncFunction>(lower_info->backend()->config());
494 builder.append(op_ind, {op_ind, &op, lower_info, std::move(fn_seq)});
498 auto code_map = builder.releaseCodeMap();
500 exec::ExecutorBase *exec = nullptr;
503 exec = new exec::ParallelExecutor{std::move(lowered_graph), std::move(backend_contexts),
504 tensor_regs, std::move(code_map), tracing_ctx};
509 new exec::DataflowExecutor{std::move(lowered_graph), std::move(backend_contexts), tensor_regs,
510 std::move(code_map), tracing_ctx};
511 if (options.he_profiling_mode)
513 std::vector<const backend::Backend *> backends;
514 for (const auto &pair : backend_contexts)
516 backends.push_back(pair.first);
518 auto et = std::make_shared<exec::ExecTime>(backends);
519 std::unique_ptr<exec::IExecutionObserver> obs =
520 std::make_unique<exec::ProfileObserver>(et, dataflow_exec->graph());
521 dataflow_exec->addObserver(std::move(obs));
523 exec = dataflow_exec;
526 if (!options.trace_filepath.empty())
528 std::unique_ptr<exec::IExecutionObserver> ctp =
529 std::make_unique<exec::TracingObserver>(options.trace_filepath, exec->graph(), tracing_ctx);
530 exec->addObserver(std::move(ctp));
536 } // namespace compiler