2 * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "compiler/Compiler.h"
19 #include "ParamChecker.h"
20 #include "ExecutorFactory.h"
21 #include "OperationValidator.h"
22 #include "Fp32ToFp16Converter.h"
24 #include <backend/controlflow/Config.h>
25 #include "compiler/BackendManager.h"
26 #include "compiler/IScheduler.h"
27 #include "compiler/ManualScheduler.h"
28 #include "compiler/HEScheduler.h"
29 #include "compiler/StaticShapeInference.h"
30 #include "exec/ExecTime.h"
31 #include "ir/operation/LowerInfo.h"
32 #include "dumper/dot/DotDumper.h"
33 #include "compiler/Linear.h"
34 #include "interp/InterpExecutor.h"
35 #include "util/ConfigSource.h"
36 #include "util/logging.h"
37 #include "ir/OperationDumper.h"
38 #include "misc/string_helpers.h"
46 CompilerOptions fetchCompilerOptionsFromGlobalConfig(const ir::Subgraphs &subgs)
48 CompilerOptions options;
49 options.backend_list = nnfw::misc::split(util::getConfigString(util::config::BACKENDS), ';');
50 options.is_primary_subgraph = false;
51 options.trace_filepath = util::getConfigString(util::config::TRACE_FILEPATH);
52 options.graph_dump_level = util::getConfigInt(util::config::GRAPH_DOT_DUMP);
53 options.op_seq_max_node = util::getConfigInt(util::config::OP_SEQ_MAX_NODE);
54 options.executor = util::getConfigString(util::config::EXECUTOR);
55 options.he_scheduler = util::getConfigBool(util::config::USE_SCHEDULER);
56 options.he_profiling_mode = util::getConfigBool(util::config::PROFILING_MODE);
57 options.disable_compile = util::getConfigBool(util::config::DISABLE_COMPILE);
58 options.fp16_enable = util::getConfigBool(util::config::FP16_ENABLE);
60 options.op_seq_max_node = 1;
65 auto &ms_options = options.manual_scheduler_options;
67 // Default value for op_backend_all is first element in the backend list
68 ms_options.backend_for_all = util::getConfigString(util::config::OP_BACKEND_ALLOPS);
73 const auto &backend_str = util::getConfigString(util::config::OP_BACKEND_##OpName); \
74 if (!backend_str.empty()) \
76 ms_options.opcode_to_backend[ir::OpCode::OpName] = backend_str; \
79 #include "ir/Operations.lst"
83 // TODO Support multiple subgraphs for manual scheduling
84 auto map_str = util::getConfigString(util::config::OP_BACKEND_MAP);
85 auto key_val_list = nnfw::misc::split(map_str, ';');
86 for (const auto &key_val_str : key_val_list)
88 if (key_val_str.empty())
93 auto key_val = nnfw::misc::split(key_val_str, '=');
94 const auto &key_str = key_val.at(0);
95 const auto &val = key_val.at(1);
96 auto key = static_cast<uint32_t>(std::stoi(key_str));
98 subgs.at(ir::SubgraphIndex{0})
100 .at(ir::OperationIndex{key}); // Check if exist, or this wil throw
101 ms_options.index_to_backend.emplace(ir::OperationIndex{key}, val);
107 Compiler::Compiler(const std::shared_ptr<ir::Subgraphs> &subgs)
108 : _subgraphs{subgs}, _state{State::CREATED}
110 // Set default values for CompilerOptions
111 // All these default values should not be fetched from Env, when we stop supporting Android NN
113 _options = fetchCompilerOptionsFromGlobalConfig(*subgs);
116 void Compiler::enableToFp16() { _options.fp16_enable = true; }
118 void Compiler::checkProfilerConditions()
120 if (!_options.he_scheduler)
121 throw std::runtime_error("Heterogeneous scheduler must be enabled during profiling.");
123 if (_options.executor != "Dataflow")
124 throw std::runtime_error("Profiling mode works only with 'Dataflow' executor");
127 std::shared_ptr<exec::ExecutorMap> Compiler::compile(void)
129 // Set control flow backend for control flow operators
131 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::If] =
132 backend::controlflow::Config::ID;
133 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::While] =
134 backend::controlflow::Config::ID;
137 // FIXME This is a workaround for bcq operations, should remove it
139 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::BCQFullyConnected] = "bcq";
140 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::BCQGather] = "bcq";
144 VERBOSE(Compiler) << std::boolalpha;
145 VERBOSE(Compiler) << "==== Compiler Options ====" << std::endl;
146 VERBOSE(Compiler) << "backend_list : "
147 << nnfw::misc::join(_options.backend_list.begin(),
148 _options.backend_list.end(), "/")
150 VERBOSE(Compiler) << "trace_filepath : " << _options.trace_filepath << std::endl;
151 VERBOSE(Compiler) << "graph_dump_level : " << _options.graph_dump_level << std::endl;
152 VERBOSE(Compiler) << "op_seq_max_node : " << _options.op_seq_max_node << std::endl;
153 VERBOSE(Compiler) << "executor : " << _options.executor << std::endl;
154 VERBOSE(Compiler) << "manual_scheduler_options : (Too many things to print)" << std::endl;
155 VERBOSE(Compiler) << "he_scheduler : " << _options.he_scheduler << std::endl;
156 VERBOSE(Compiler) << "he_profiling_mode : " << _options.he_profiling_mode << std::endl;
157 VERBOSE(Compiler) << "disable_compile : " << _options.disable_compile << std::endl;
158 VERBOSE(Compiler) << "fp16_enable : " << _options.fp16_enable << std::endl;
159 VERBOSE(Compiler) << std::noboolalpha;
162 /***************************************************
163 * Prepare compilation phase
164 ***************************************************/
166 auto executors = std::make_shared<exec::ExecutorMap>();
169 // TODO: Support hybrid execution -
170 // execution between interpreter and compiled executor (including control flow)
171 if (!checkCompilable())
173 _subgraphs->iterate([&](const ir::SubgraphIndex &index, ir::Graph &subg) {
174 executors->emplace(index, std::make_unique<interp::InterpExecutor>(subg));
176 _state = State::COMPILED;
181 if (_options.he_profiling_mode)
182 checkProfilerConditions();
184 /***************************************************
185 * Backend independent analysis & optimization phase
186 ***************************************************/
187 auto dump_level = static_cast<dumper::dot::DotDumper::Level>(_options.graph_dump_level);
189 // Lower: Assign backend
190 std::unordered_map<ir::SubgraphIndex, std::unique_ptr<compiler::LoweredGraph>> lowered_subgs;
191 _subgraphs->iterate([&](const ir::SubgraphIndex &index, ir::Graph &subg) {
192 _options.is_primary_subgraph = (index == ir::SubgraphIndex{0});
193 onert::dumper::dot::DotDumper dot_dumper(subg, dump_level);
194 dot_dumper.dump(nnfw::misc::str("before_lower_subg-", index.value()));
196 // Lower: Assign backend
197 lowered_subgs[index] = std::make_unique<compiler::LoweredGraph>(subg, _options);
199 // Check backend(s) for subgraph support FP16
200 bool backends_support_fp16 = true;
201 auto &contexts = (*lowered_subgs[index]).backend_contexts();
202 for (auto it = contexts.begin(); it != contexts.end(); it++)
204 // Controlflow backend is not for actual computaion of operations so it is an exception
205 if (it->first->config()->id() != backend::controlflow::Config::ID)
206 backends_support_fp16 &= it->first->config()->supportFP16();
209 if (_options.fp16_enable && backends_support_fp16)
211 // NOTE: the only acl_cl backend enables fp16 mode
212 Fp32ToFp16Converter(*lowered_subgs[index]).run();
215 subg.setSubgraphs(nullptr);
222 const auto primary_subg_idx = ir::SubgraphIndex{0};
223 StaticShapeInferer inferer(primary_subg_idx, lowered_subgs);
224 lowered_subgs.at(primary_subg_idx)
225 ->iterateTopolOpSeqs([&](const ir::OpSequenceIndex &, ir::OpSequence &op_seq) {
226 auto has_dynamic_tensor = inferer.infer(op_seq);
227 op_seq.has_dynamic_tensor(has_dynamic_tensor);
232 /*************************************************************
233 * Backend independent analysis & optimization phase finished
234 *************************************************************/
236 // operation validation
237 for (auto &pair : lowered_subgs)
239 auto &lowered_subg = pair.second;
240 compiler::OperationValidator{lowered_subg->graph()}();
243 executors = std::make_shared<exec::ExecutorMap>();
244 for (auto &pair : lowered_subgs)
246 const auto &subg_index = pair.first;
247 auto &lowered_subg = pair.second;
248 auto indexed_ranks = lowered_subg->indexed_ranks();
250 _options.is_primary_subgraph = (subg_index == ir::SubgraphIndex{0});
252 onert::dumper::dot::DotDumper dot_dumper_lowered(lowered_subg.get(), dump_level);
253 dot_dumper_lowered.dump("after_lower_subg-" + std::to_string(subg_index.value()));
255 ir::OperationDumper dumper("START SUBGRAPH " + std::to_string(subg_index.value()));
256 lowered_subg->graph().operations().iterate(
257 [&](const ir::OperationIndex &, const ir::Operation &op) { op.accept(dumper); });
258 auto executor = std::unique_ptr<exec::IExecutor>{
259 ExecutorFactory::get().create(std::move(lowered_subg), _options, executors)};
260 executor->setIndexedRanks(indexed_ranks);
261 executors->insert(std::make_pair(subg_index, std::move(executor)));
264 /********************************
265 * Code generation phase finished
266 ********************************/
267 _state = State::COMPILED;
271 bool Compiler::checkCompilable()
273 // Disable compile phase
274 // When ready to use interpreter backend, remove this config and use backend setting
275 if (_options.disable_compile)
280 // TODO check unspecified operand shape
282 // Check compilable parameter
283 for (uint32_t i = 0; i < _subgraphs->count(); ++i)
285 auto graph = _subgraphs->at(ir::SubgraphIndex{i});
286 ParamChecker paramChecker{graph};
288 if (paramChecker.haveNoneConstParam())
297 } // namespace compiler