1 // Copyright (C) 2018-2020 Intel Corporation
2 // SPDX-License-Identifier: Apache-2.0
6 * @brief a header for advanced hardware related properties for IE plugins
8 * To use in SetConfig() method of plugins
9 * LoadNetwork() method overloads that accept config as parameter
11 * @file ie_plugin_config.hpp
19 namespace InferenceEngine {
27 * @def METRIC_KEY(name)
28 * @brief shortcut for defining common Inference Engine metrics
30 #define METRIC_KEY(name) InferenceEngine::Metrics::METRIC_##name
33 * @def EXEC_NETWORK_METRIC_KEY(name)
34 * @brief shortcut for defining common Inference Engine ExecutableNetwork metrics
36 #define EXEC_NETWORK_METRIC_KEY(name) METRIC_KEY(name)
38 #ifndef DECLARE_METRIC_KEY_IMPL
39 #define DECLARE_METRIC_KEY(name, ...) \
40 static constexpr auto METRIC_##name = #name
42 #define DECLARE_METRIC_KEY(name, ...) \
43 static constexpr auto METRIC_##name = #name; \
44 DECLARE_METRIC_KEY_IMPL(name, __VA_ARGS__)
47 #define DECLARE_EXEC_NETWORK_METRIC_KEY(name, ...) DECLARE_METRIC_KEY(name, __VA_ARGS__)
50 * @def METRIC_VALUE(name)
51 * @brief shortcut for defining metric values
53 #define METRIC_VALUE(name) InferenceEngine::Metrics::name
54 #define DECLARE_METRIC_VALUE(name) static constexpr auto name = #name
57 * @brief Metric to get a std::vector<std::string> of available device IDs. String value is "AVAILABLE_DEVICES"
59 DECLARE_METRIC_KEY(AVAILABLE_DEVICES, std::vector<std::string>);
62 * @brief Metric to get a std::vector<std::string> of supported metrics. String value is "SUPPORTED_METRICS"
64 * This can be used as an executable network metric as well.
66 * Each of the returned device metrics can be passed to Core::GetMetric, executable network metrics
67 * can be passed to ExecutableNetwork::GetMetric.
70 DECLARE_METRIC_KEY(SUPPORTED_METRICS, std::vector<std::string>);
73 * @brief Metric to get a std::vector<std::string> of supported config keys. String value is "SUPPORTED_CONFIG_KEYS"
75 * This can be used as an executable network metric as well.
77 * Each of the returned device configuration keys can be passed to Core::SetConfig, Core::GetConfig, and
78 * Core::LoadNetwork, configuration keys for executable networks can be passed to ExecutableNetwork::SetConfig and
79 * ExecutableNetwork::GetConfig.
82 DECLARE_METRIC_KEY(SUPPORTED_CONFIG_KEYS, std::vector<std::string>);
85 * @brief Metric to get a std::string value representing a full device name. String value is "FULL_DEVICE_NAME"
87 DECLARE_METRIC_KEY(FULL_DEVICE_NAME, std::string);
90 * @brief Metric to get a std::vector<std::string> of optimization options per device. String value is
91 * "OPTIMIZATION_CAPABILITIES"
93 * The possible values:
94 * - "FP32" - device can support FP32 models
95 * - "BF16" - device can support BF16 computations for models
96 * - "FP16" - device can support FP16 models
97 * - "INT8" - device can support models with INT8 layers
98 * - "BIN" - device can support models with BIN layers
99 * - "WINOGRAD" - device can support models where convolution implemented via Winograd transformations
101 DECLARE_METRIC_KEY(OPTIMIZATION_CAPABILITIES, std::vector<std::string>);
103 DECLARE_METRIC_VALUE(FP32);
104 DECLARE_METRIC_VALUE(BF16);
105 DECLARE_METRIC_VALUE(FP16);
106 DECLARE_METRIC_VALUE(INT8);
107 DECLARE_METRIC_VALUE(BIN);
108 DECLARE_METRIC_VALUE(WINOGRAD);
111 * @brief Metric to provide information about a range for streams on platforms where streams are supported.
113 * Metric returns a value of std::tuple<unsigned int, unsigned int> type, where:
114 * - First value is bottom bound.
115 * - Second value is upper bound.
116 * String value for metric name is "RANGE_FOR_STREAMS".
118 DECLARE_METRIC_KEY(RANGE_FOR_STREAMS, std::tuple<unsigned int, unsigned int>);
121 * @brief Metric to provide a hint for a range for number of async infer requests. If device supports streams,
122 * the metric provides range for number of IRs per stream.
124 * Metric returns a value of std::tuple<unsigned int, unsigned int, unsigned int> type, where:
125 * - First value is bottom bound.
126 * - Second value is upper bound.
127 * - Third value is step inside this range.
128 * String value for metric name is "RANGE_FOR_ASYNC_INFER_REQUESTS".
130 DECLARE_METRIC_KEY(RANGE_FOR_ASYNC_INFER_REQUESTS, std::tuple<unsigned int, unsigned int, unsigned int>);
133 * @brief Metric to get an unsigned int value of number of waiting infer request.
135 * String value is "NUMBER_OF_WAITNING_INFER_REQUESTS". This can be used as an executable network metric as well
137 DECLARE_METRIC_KEY(NUMBER_OF_WAITING_INFER_REQUESTS, unsigned int);
140 * @brief Metric to get an unsigned int value of number of infer request in execution stage.
142 * String value is "NUMBER_OF_EXEC_INFER_REQUESTS". This can be used as an executable network metric as well
144 DECLARE_METRIC_KEY(NUMBER_OF_EXEC_INFER_REQUESTS, unsigned int);
147 * @brief Metric to get a name of network. String value is "NETWORK_NAME".
149 DECLARE_EXEC_NETWORK_METRIC_KEY(NETWORK_NAME, std::string);
152 * @brief Metric to get a float of device thermal. String value is "DEVICE_THERMAL"
154 DECLARE_METRIC_KEY(DEVICE_THERMAL, float);
157 * @brief Metric to get an unsigned integer value of optimal number of executable network infer requests.
159 DECLARE_EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS, unsigned int);
161 } // namespace Metrics
164 * @brief Generic plugin configuration
166 namespace PluginConfigParams {
169 * @def CONFIG_KEY(name)
170 * @brief shortcut for defining configuration keys
172 #define CONFIG_KEY(name) InferenceEngine::PluginConfigParams::_CONFIG_KEY(name)
173 #define _CONFIG_KEY(name) KEY_##name
174 #define DECLARE_CONFIG_KEY(name) static constexpr auto _CONFIG_KEY(name) = #name
177 * @def CONFIG_VALUE(name)
178 * @brief shortcut for defining configuration values
180 #define CONFIG_VALUE(name) InferenceEngine::PluginConfigParams::name
181 #define DECLARE_CONFIG_VALUE(name) static constexpr auto name = #name
184 * @brief generic boolean values
186 DECLARE_CONFIG_VALUE(YES);
187 DECLARE_CONFIG_VALUE(NO);
190 * @brief Limit `#threads` that are used by Inference Engine for inference on the CPU.
192 DECLARE_CONFIG_KEY(CPU_THREADS_NUM);
195 * @brief The name for setting CPU affinity per thread option.
197 * It is passed to Core::SetConfig(), this option should be used with values:
198 * PluginConfigParams::YES (pinning threads to cores, best for static benchmarks),
199 * PluginConfigParams::NUMA (pinning therads to NUMA nodes, best for real-life, contented cases)
200 * this is TBB-specific knob, and the only pinning option (beyond 'NO', below) on the Windows*
201 * PluginConfigParams::NO (no pinning for CPU inference threads)
202 * All settings are ignored, if the OpenVINO compiled with OpenMP threading and any affinity-related OpenMP's
203 * environment variable is set (as affinity is configured explicitly)
205 DECLARE_CONFIG_KEY(CPU_BIND_THREAD);
206 DECLARE_CONFIG_VALUE(NUMA);
209 * @brief Optimize CPU execution to maximize throughput.
211 * It is passed to Core::SetConfig(), this option should be used with values:
212 * - KEY_CPU_THROUGHPUT_NUMA creates as many streams as needed to accomodate NUMA and avoid associated penalties
213 * - KEY_CPU_THROUGHPUT_AUTO creates bare minimum of streams to improve the performance,
214 * this is the most portable option if you have no insights into how many cores you target machine will have
215 * (and what is the optimal number of streams)
216 * - finally, specifying the positive integer value creates the requested number of streams
218 DECLARE_CONFIG_VALUE(CPU_THROUGHPUT_NUMA);
219 DECLARE_CONFIG_VALUE(CPU_THROUGHPUT_AUTO);
220 DECLARE_CONFIG_KEY(CPU_THROUGHPUT_STREAMS);
223 * @brief Optimize GPU plugin execution to maximize throughput.
225 * It is passed to Core::SetConfig(), this option should be used with values:
226 * - KEY_GPU_THROUGHPUT_AUTO creates bare minimum of streams that might improve performance in some cases,
227 * this option allows to enable throttle hint for opencl queue thus reduce CPU load without significant performance
229 * - a positive integer value creates the requested number of streams
231 DECLARE_CONFIG_VALUE(GPU_THROUGHPUT_AUTO);
232 DECLARE_CONFIG_KEY(GPU_THROUGHPUT_STREAMS);
235 * @brief The name for setting performance counters option.
237 * It is passed to Core::SetConfig(), this option should be used with values:
238 * PluginConfigParams::YES or PluginConfigParams::NO
240 DECLARE_CONFIG_KEY(PERF_COUNT);
243 * @brief The key defines dynamic limit of batch processing.
245 * Specified value is applied to all following Infer() calls. Inference Engine processes
246 * min(batch_limit, original_batch_size) first pictures from input blob. For example, if input
247 * blob has sizes 32x3x224x224 after applying plugin.SetConfig({KEY_DYN_BATCH_LIMIT, 10})
248 * Inference Engine primitives processes only beginner subblobs with size 10x3x224x224.
249 * This value can be changed before any Infer() call to specify a new batch limit.
251 * The paired parameter value should be convertible to integer number. Acceptable values:
252 * -1 - Do not limit batch processing
253 * >0 - Direct value of limit. Batch size to process is min(new batch_limit, original_batch)
255 DECLARE_CONFIG_KEY(DYN_BATCH_LIMIT);
257 DECLARE_CONFIG_KEY(DYN_BATCH_ENABLED);
259 DECLARE_CONFIG_KEY(DUMP_QUANTIZED_GRAPH_AS_DOT);
260 DECLARE_CONFIG_KEY(DUMP_QUANTIZED_GRAPH_AS_IR);
263 * @brief The key controls threading inside Inference Engine.
265 * It is passed to Core::SetConfig(), this option should be used with values:
266 * PluginConfigParams::YES or PluginConfigParams::NO
268 DECLARE_CONFIG_KEY(SINGLE_THREAD);
271 * @brief This key directs the plugin to load a configuration file.
273 * The value should be a file name with the plugin specific configuration
275 DECLARE_CONFIG_KEY(CONFIG_FILE);
278 * @brief This key enables dumping of the kernels used by the plugin for custom layers.
280 * This option should be used with values: PluginConfigParams::YES or PluginConfigParams::NO (default)
282 DECLARE_CONFIG_KEY(DUMP_KERNELS);
285 * @brief This key controls performance tuning done or used by the plugin.
287 * This option should be used with values:
288 * PluginConfigParams::TUNING_DISABLED (default)
289 * PluginConfigParams::TUNING_USE_EXISTING - use existing data from tuning file
290 * PluginConfigParams::TUNING_CREATE - create tuning data for parameters not present in tuning file
291 * PluginConfigParams::TUNING_UPDATE - perform non-tuning updates like removal of invalid/deprecated data
292 * PluginConfigParams::TUNING_RETUNE - create tuning data for all parameters, even if already present
294 * For values TUNING_CREATE and TUNING_RETUNE the file will be created if it does not exist.
296 DECLARE_CONFIG_KEY(TUNING_MODE);
298 DECLARE_CONFIG_VALUE(TUNING_CREATE);
299 DECLARE_CONFIG_VALUE(TUNING_USE_EXISTING);
300 DECLARE_CONFIG_VALUE(TUNING_DISABLED);
301 DECLARE_CONFIG_VALUE(TUNING_UPDATE);
302 DECLARE_CONFIG_VALUE(TUNING_RETUNE);
305 * @brief This key defines the tuning data filename to be created/used
307 DECLARE_CONFIG_KEY(TUNING_FILE);
310 * @brief the key for setting desirable log level.
312 * This option should be used with values: PluginConfigParams::LOG_NONE (default),
313 * PluginConfigParams::LOG_ERROR, PluginConfigParams::LOG_WARNING,
314 * PluginConfigParams::LOG_INFO, PluginConfigParams::LOG_DEBUG, PluginConfigParams::LOG_TRACE
316 DECLARE_CONFIG_KEY(LOG_LEVEL);
318 DECLARE_CONFIG_VALUE(LOG_NONE); // turn off logging
319 DECLARE_CONFIG_VALUE(LOG_ERROR); // error events that might still allow the application to continue running
320 DECLARE_CONFIG_VALUE(LOG_WARNING); // potentially harmful situations which may further lead to ERROR
321 DECLARE_CONFIG_VALUE(
322 LOG_INFO); // informational messages that display the progress of the application at coarse-grained level
323 DECLARE_CONFIG_VALUE(LOG_DEBUG); // fine-grained events that are most useful to debug an application.
324 DECLARE_CONFIG_VALUE(LOG_TRACE); // finer-grained informational events than the DEBUG
327 * @brief the key for setting of required device to execute on
328 * values: device id starts from "0" - first device, "1" - second device, etc
330 DECLARE_CONFIG_KEY(DEVICE_ID);
333 * @brief the key for enabling exclusive mode for async requests of different executable networks and the same plugin.
335 * Sometimes it is necessary to avoid oversubscription requests that are sharing the same device in parallel.
336 * E.g. There 2 task executors for CPU device: one - in the Hetero plugin, another - in pure CPU plugin.
337 * Parallel execution both of them might lead to oversubscription and not optimal CPU usage. More efficient
338 * to run the corresponding tasks one by one via single executor.
339 * By default, the option is set to YES for hetero cases, and to NO for conventional (single-plugin) cases
340 * Notice that setting YES disables the CPU streams feature (see another config key in this file)
342 DECLARE_CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS);
345 * @brief This key enables dumping of the internal primitive graph.
347 * Should be passed into LoadNetwork method to enable dumping of internal graph of primitives and
348 * corresponding configuration information. Value is a name of output dot file without extension.
349 * Files `<dot_file_name>_init.dot` and `<dot_file_name>_perf.dot` will be produced.
351 DECLARE_CONFIG_KEY(DUMP_EXEC_GRAPH_AS_DOT);
355 * @brief The name for setting to execute in bfloat16 precision whenever it is possible
357 * This option let plugin know to downscale the precision where it see performance benefits from
359 * Such option do not guarantee accuracy of the network, the accuracy in this mode should be
360 * verified separately by the user and basing on performance and accuracy results it should be
361 * user's decision to use this option or not to use
363 DECLARE_CONFIG_KEY(ENFORCE_BF16);
365 } // namespace PluginConfigParams
366 } // namespace InferenceEngine