2 * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "compiler/Compiler.h"
19 #include "ParamChecker.h"
20 #include "ExecutorFactory.h"
21 #include "ShapeValidator.h"
22 #include "Fp32ToFp16Converter.h"
24 #include <backend/controlflow/Config.h>
25 #include "compiler/BackendManager.h"
26 #include "compiler/IScheduler.h"
27 #include "compiler/ManualScheduler.h"
28 #include "compiler/HEScheduler.h"
29 #include "compiler/StaticShapeInferer.h"
30 #include "compiler/pass/ConstantOutputPass.h"
31 #include "compiler/pass/OddOutputPass.h"
32 #include "compiler/pass/PassRunner.h"
33 #include "exec/ExecTime.h"
34 #include "ir/operation/LowerInfo.h"
35 #include "ir/verifier/Verifier.h"
36 #include "dumper/dot/DotDumper.h"
37 #include "compiler/Linear.h"
38 #include "interp/InterpExecutor.h"
39 #include "util/ConfigSource.h"
40 #include "util/logging.h"
41 #include "ir/OperationDumper.h"
42 #include "misc/string_helpers.h"
47 using namespace onert;
49 std::string getOpBackends(std::unordered_map<ir::OpCode, std::string> &opcode_to_backend)
51 std::unordered_map<ir::OpCode, std::string>::iterator it;
52 std::string opbackends;
54 for (it = opcode_to_backend.begin(); it != opcode_to_backend.end(); ++it)
56 if (!opbackends.empty())
57 opbackends = opbackends + ", ";
59 auto opcode = it->first;
60 const std::string opname = ir::toString(opcode);
61 opbackends += opname + "=" + it->second;
74 CompilerOptions fetchCompilerOptionsFromGlobalConfig(const ir::Subgraphs &subgs)
76 CompilerOptions options;
77 options.backend_list = nnfw::misc::split(util::getConfigString(util::config::BACKENDS), ';');
78 options.trace_filepath = util::getConfigString(util::config::TRACE_FILEPATH);
79 options.graph_dump_level = util::getConfigInt(util::config::GRAPH_DOT_DUMP);
80 options.op_seq_max_node = util::getConfigInt(util::config::OP_SEQ_MAX_NODE);
81 options.executor = util::getConfigString(util::config::EXECUTOR);
82 options.he_scheduler = util::getConfigBool(util::config::USE_SCHEDULER);
83 options.he_profiling_mode = util::getConfigBool(util::config::PROFILING_MODE);
84 options.disable_compile = util::getConfigBool(util::config::DISABLE_COMPILE);
85 options.fp16_enable = util::getConfigBool(util::config::FP16_ENABLE);
87 options.op_seq_max_node = 1;
92 auto &ms_options = options.manual_scheduler_options;
94 // Default value for op_backend_all is first element in the backend list
95 ms_options.backend_for_all = util::getConfigString(util::config::OP_BACKEND_ALLOPS);
100 const auto &backend_str = util::getConfigString(util::config::OP_BACKEND_##OpName); \
101 if (!backend_str.empty()) \
103 ms_options.opcode_to_backend[ir::OpCode::OpName] = backend_str; \
106 #include "ir/Operations.lst"
110 // TODO Support multiple subgraphs for manual scheduling
111 auto map_str = util::getConfigString(util::config::OP_BACKEND_MAP);
112 auto key_val_list = nnfw::misc::split(map_str, ';');
113 for (const auto &key_val_str : key_val_list)
115 if (key_val_str.empty())
120 auto key_val = nnfw::misc::split(key_val_str, '=');
121 const auto &key_str = key_val.at(0);
122 const auto &val = key_val.at(1);
123 auto key = static_cast<uint32_t>(std::stoi(key_str));
125 subgs.at(ir::SubgraphIndex{0})
127 .at(ir::OperationIndex{key}); // Check if exist, or this wil throw
128 ms_options.index_to_backend.emplace(ir::OperationIndex{key}, val);
134 Compiler::Compiler(const std::shared_ptr<ir::Subgraphs> &subgs, util::TracingCtx *tracing_ctx)
135 : _subgraphs{subgs}, _state{State::CREATED}
137 // Set default values for CompilerOptions
138 // All these default values should not be fetched from Env, when we stop supporting Android NN
140 _options = fetchCompilerOptionsFromGlobalConfig(*subgs);
142 _options.tracing_ctx = tracing_ctx;
145 void Compiler::enableToFp16() { _options.fp16_enable = true; }
147 void Compiler::checkProfilerConditions()
149 if (!_options.he_scheduler)
150 throw std::runtime_error("Heterogeneous scheduler must be enabled during profiling.");
152 if (_options.executor != "Dataflow")
153 throw std::runtime_error("Profiling mode works only with 'Dataflow' executor");
156 std::shared_ptr<exec::ExecutorMap> Compiler::compile(void)
158 // Set control flow backend for control flow operators
160 auto &cfid = backend::controlflow::Config::ID;
161 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::If] = cfid;
162 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::While] = cfid;
163 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::Permute] = cfid;
166 // FIXME This is a workaround for bcq operations, should remove it
168 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::BCQFullyConnected] = "bcq";
169 _options.manual_scheduler_options.opcode_to_backend[ir::OpCode::BCQGather] = "bcq";
173 VERBOSE(Compiler) << std::boolalpha;
174 VERBOSE(Compiler) << "==== Compiler Options ====" << std::endl;
175 VERBOSE(Compiler) << "backend_list : "
176 << nnfw::misc::join(_options.backend_list.begin(),
177 _options.backend_list.end(), "/")
179 VERBOSE(Compiler) << "trace_filepath : " << _options.trace_filepath << std::endl;
180 VERBOSE(Compiler) << "graph_dump_level : " << _options.graph_dump_level << std::endl;
181 VERBOSE(Compiler) << "op_seq_max_node : " << _options.op_seq_max_node << std::endl;
182 VERBOSE(Compiler) << "executor : " << _options.executor << std::endl;
183 VERBOSE(Compiler) << "manual backend_for_all : "
184 << _options.manual_scheduler_options.backend_for_all << std::endl;
185 VERBOSE(Compiler) << "manual_scheduler_options : "
186 << getOpBackends(_options.manual_scheduler_options.opcode_to_backend)
188 VERBOSE(Compiler) << "he_scheduler : " << _options.he_scheduler << std::endl;
189 VERBOSE(Compiler) << "he_profiling_mode : " << _options.he_profiling_mode << std::endl;
190 VERBOSE(Compiler) << "disable_compile : " << _options.disable_compile << std::endl;
191 VERBOSE(Compiler) << "fp16_enable : " << _options.fp16_enable << std::endl;
192 VERBOSE(Compiler) << std::noboolalpha;
195 _subgraphs->iterate([&](const ir::SubgraphIndex &, ir::Graph &subg) {
198 .append(std::make_unique<pass::ConstantOutputPass>(subg))
199 .append(std::make_unique<pass::OddOutputPass>(subg))
203 /***************************************************
204 * Prepare compilation phase
205 ***************************************************/
206 auto executors = std::make_shared<exec::ExecutorMap>();
209 // TODO: Support hybrid execution -
210 // execution between interpreter and compiled executor (including control flow)
211 if (!checkCompilable())
213 _subgraphs->iterate([&](const ir::SubgraphIndex &index, ir::Graph &subg) {
214 executors->emplace(index, std::make_unique<interp::InterpExecutor>(subg));
216 _state = State::COMPILED;
221 if (_options.he_profiling_mode)
222 checkProfilerConditions();
224 /***************************************************
225 * Backend independent analysis & optimization phase
226 ***************************************************/
227 auto dump_level = static_cast<dumper::dot::DotDumper::Level>(_options.graph_dump_level);
229 // Lower: Assign backend
230 std::unordered_map<ir::SubgraphIndex, std::unique_ptr<compiler::LoweredGraph>> lowered_subgs;
231 _subgraphs->iterate([&](const ir::SubgraphIndex &index, ir::Graph &subg) {
232 onert::dumper::dot::DotDumper dot_dumper(subg, dump_level);
233 dot_dumper.dump(nnfw::misc::str("before_lower_subg-", index.value()));
235 // Lower: Assign backend
236 lowered_subgs[index] = std::make_unique<compiler::LoweredGraph>(subg, _options);
238 // Check backend(s) for subgraph support FP16
239 bool backends_support_fp16 = true;
240 auto &contexts = (*lowered_subgs[index]).backend_contexts();
241 for (auto it = contexts.begin(); it != contexts.end(); it++)
243 // Controlflow backend is not for actual computaion of operations so it is an exception
244 if (it->first->config()->id() != backend::controlflow::Config::ID)
245 backends_support_fp16 &= it->first->config()->supportFP16();
248 if (_options.fp16_enable && backends_support_fp16)
250 // NOTE: the only acl_cl backend enables fp16 mode
251 Fp32ToFp16Converter(*lowered_subgs[index]).run();
254 subg.setSubgraphs(nullptr);
259 for (auto &pair : lowered_subgs)
261 const auto &subg_index = pair.first;
262 auto &lowered_subg = pair.second;
263 onert::dumper::dot::DotDumper dot_dumper_lowered(lowered_subg.get(), dump_level);
264 dot_dumper_lowered.dump("after_lower_subg-" + std::to_string(subg_index.value()));
269 const auto primary_subg_idx = ir::SubgraphIndex{0};
270 StaticShapeInferer inferer(primary_subg_idx, lowered_subgs);
271 lowered_subgs.at(primary_subg_idx)
272 ->iterateTopolOpSeqs([&](const ir::OpSequenceIndex &, ir::OpSequence &op_seq) {
273 auto has_dynamic_tensor = inferer.infer(op_seq);
274 op_seq.has_dynamic_tensor(has_dynamic_tensor);
280 // TODO Move shape independent feature check from ShapeValidator to OperationValidator
281 // TODO Move ShapeValidator into shape inference
282 // - Check input tensor shape validation
283 // - Check parameter value validation which valid value is depend on input tensor shape
284 // - Output tensor shape validation check is needless because
285 // static/dynamic shape inferer will make valid output shape
286 for (auto &pair : lowered_subgs)
288 auto &lowered_subg = pair.second;
289 compiler::ShapeValidator{lowered_subg->graph()}();
292 /*************************************************************
293 * Backend independent analysis & optimization phase finished
294 *************************************************************/
296 executors = std::make_shared<exec::ExecutorMap>();
297 for (auto &pair : lowered_subgs)
299 const auto &subg_index = pair.first;
300 auto &lowered_subg = pair.second;
301 auto indexed_ranks = lowered_subg->indexed_ranks();
303 ir::OperationDumper dumper("Executor generation of Subgraph " +
304 std::to_string(subg_index.value()));
305 lowered_subg->graph().operations().iterate(
306 [&](const ir::OperationIndex &, const ir::Operation &op) { op.accept(dumper); });
307 auto executor = std::unique_ptr<exec::IExecutor>{
308 ExecutorFactory::get().create(std::move(lowered_subg), _options, executors)};
309 executor->setIndexedRanks(indexed_ranks);
310 executors->insert(std::make_pair(subg_index, std::move(executor)));
313 /********************************
314 * Code generation phase finished
315 ********************************/
316 _state = State::COMPILED;
320 bool Compiler::checkCompilable()
322 // Disable compile phase
323 // When ready to use interpreter backend, remove this config and use backend setting
324 if (_options.disable_compile)
329 // TODO check unspecified operand shape
331 // Check compilable parameter
332 for (uint32_t i = 0; i < _subgraphs->count(); ++i)
334 auto graph = _subgraphs->at(ir::SubgraphIndex{i});
335 ParamChecker paramChecker{graph};
337 if (paramChecker.haveNoneConstParam())
346 } // namespace compiler