ie_plugin_config.hpp
Go to the documentation of this file.
1 // Copyright (C) 2018-2019 Intel Corporation
2 // SPDX-License-Identifier: Apache-2.0
3 //
4 
5 /**
6  * @brief a header for advanced hardware related properties for IE plugins
7  * To use in SetConfig() method of plugins
8  * LoadNetwork() method overloads that accept config as parameter
9  *
10  * @file ie_plugin_config.hpp
11  */
12 #pragma once
13 
14 #include <string>
15 #include <tuple>
16 #include <vector>
17 
18 namespace InferenceEngine {
19 
20 namespace Metrics {
21 
22 #ifndef DECLARE_METRIC_KEY_IMPL
23 # define DECLARE_METRIC_KEY_IMPL(...)
24 #endif
25 
26 /**
27 * @brief shortcut for defining common Inference Engine metrics
28 */
29 
30 #define METRIC_KEY(name) InferenceEngine::Metrics::METRIC_##name
31 #define EXEC_NETWORK_METRIC_KEY(name) METRIC_KEY(name)
32 
33 #define DECLARE_METRIC_KEY(name, ...) \
34  static constexpr auto METRIC_##name = #name; \
35  DECLARE_METRIC_KEY_IMPL(name, __VA_ARGS__)
36 
37 #define DECLARE_EXEC_NETWORK_METRIC_KEY(name, ...) DECLARE_METRIC_KEY(name, __VA_ARGS__)
38 
39 /**
40 * @brief shortcut for defining metric values
41 */
42 #define METRIC_VALUE(name) InferenceEngine::Metrics::name
43 #define DECLARE_METRIC_VALUE(name) static constexpr auto name = #name
44 
45 /**
46 * @brief Metric to get a std::vector<std::string> of available device IDs. String value is "AVAILABLE_DEVICES"
47 */
48 DECLARE_METRIC_KEY(AVAILABLE_DEVICES, std::vector<std::string>);
49 
50 /**
51 * @brief Metric to get a std::vector<std::string> of supported metrics. String value is "SUPPORTED_METRICS"
52 * This can be used as an executable network metric as well.
53 *
54 * Each of the returned device metrics can be passed to Core::GetMetric, executable network metrics
55 * can be passed to ExecutableNetwork::GetMetric.
56 *
57 */
58 DECLARE_METRIC_KEY(SUPPORTED_METRICS, std::vector<std::string>);
59 
60 /**
61 * @brief Metric to get a std::vector<std::string> of supported config keys. String value is "SUPPORTED_CONFIG_KEYS"
62 * This can be used as an executable network metric as well.
63 *
64 * Each of the returned device configuration keys can be passed to Core::SetConfig, Core::GetConfig, and Core::LoadNetwork,
65 * configuration keys for executable networks can be passed to ExecutableNetwork::SetConfig and ExecutableNetwork::GetConfig.
66 *
67 */
68 DECLARE_METRIC_KEY(SUPPORTED_CONFIG_KEYS, std::vector<std::string>);
69 
70 /**
71 * @brief Metric to get a std::string value representing a full device name. String value is "FULL_DEVICE_NAME"
72 */
73 DECLARE_METRIC_KEY(FULL_DEVICE_NAME, std::string);
74 
75 /**
76 * @brief Metric to get a std::vector<std::string> of optimization options per device. String value is "OPTIMIZATION_CAPABILITIES"
77 * The possible values:
78 * - "FP32" - device can support FP32 models
79 * - "FP16" - device can support FP16 models
80 * - "INT8" - device can support models with INT8 layers
81 * - "BIN" - device can support models with BIN layers
82 * - "WINOGRAD" - device can support models where convolution implemented via Winograd transformations
83 */
84 DECLARE_METRIC_KEY(OPTIMIZATION_CAPABILITIES, std::vector<std::string>);
85 
86 DECLARE_METRIC_VALUE(FP32);
87 DECLARE_METRIC_VALUE(FP16);
88 DECLARE_METRIC_VALUE(INT8);
89 DECLARE_METRIC_VALUE(BIN);
90 DECLARE_METRIC_VALUE(WINOGRAD);
91 
92 /**
93 * @brief Metric to provide information about a range for streams on platforms where streams are supported.
94 * Metric returns a value of std::tuple<unsigned int, unsigned int> type, where:
95 * - First value is bottom bound.
96 * - Second value is upper bound.
97 * String value for metric name is "RANGE_FOR_STREAMS".
98 */
99 DECLARE_METRIC_KEY(RANGE_FOR_STREAMS, std::tuple<unsigned int, unsigned int>);
100 
101 /**
102 * @brief Metric to provide a hint for a range for number of async infer requests. If device supports streams,
103 * the metric provides range for number of IRs per stream.
104 * Metric returns a value of std::tuple<unsigned int, unsigned int, unsigned int> type, where:
105 * - First value is bottom bound.
106 * - Second value is upper bound.
107 * - Third value is step inside this range.
108 * String value for metric name is "RANGE_FOR_ASYNC_INFER_REQUESTS".
109 */
110 DECLARE_METRIC_KEY(RANGE_FOR_ASYNC_INFER_REQUESTS, std::tuple<unsigned int, unsigned int, unsigned int>);
111 
112 /**
113 * @brief Metric to get an unsigned int value of number of waiting infer request.
114 * String value is "NUMBER_OF_WAITNING_INFER_REQUESTS". This can be used as an executable network metric as well
115 */
116 DECLARE_METRIC_KEY(NUMBER_OF_WAITING_INFER_REQUESTS, unsigned int);
117 
118 /**
119 * @brief Metric to get an unsigned int value of number of infer request in execution stage.
120 * String value is "NUMBER_OF_EXEC_INFER_REQUESTS". This can be used as an executable network metric as well
121 */
122 DECLARE_METRIC_KEY(NUMBER_OF_EXEC_INFER_REQUESTS, unsigned int);
123 
124 /**
125 * @brief Metric to get a name of network. String value is "NETWORK_NAME".
126 */
127 DECLARE_EXEC_NETWORK_METRIC_KEY(NETWORK_NAME, std::string);
128 
129 /**
130  * @brief Metric to get a float of device thermal. String value is "DEVICE_THERMAL"
131  */
132 DECLARE_METRIC_KEY(DEVICE_THERMAL, float);
133 
134 /**
135 * @brief Metric to get an unsigned integer value of optimal number of executable network infer requests.
136 */
137 DECLARE_EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS, unsigned int);
138 
139 } // namespace Metrics
140 
141 namespace PluginConfigParams {
142 
143 /**
144 * @brief shortcut for defining configuration keys
145 */
146 #define CONFIG_KEY(name) InferenceEngine::PluginConfigParams::_CONFIG_KEY(name)
147 #define _CONFIG_KEY(name) KEY_##name
148 #define DECLARE_CONFIG_KEY(name) static constexpr auto _CONFIG_KEY(name) = #name
149 
150 /**
151 * @brief shortcut for defining configuration values
152 */
153 #define CONFIG_VALUE(name) InferenceEngine::PluginConfigParams::name
154 #define DECLARE_CONFIG_VALUE(name) static constexpr auto name = #name
155 
156 /**
157 * @brief generic boolean values
158 */
159 DECLARE_CONFIG_VALUE(YES);
160 DECLARE_CONFIG_VALUE(NO);
161 
162 /**
163 * @brief Limit #threads that are used by Inference Engine for inference on the CPU.
164 */
165 DECLARE_CONFIG_KEY(CPU_THREADS_NUM);
166 
167 /**
168 * @brief The name for setting CPU affinity per thread option.
169 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
170 * PluginConfigParams::YES or PluginConfigParams::NO
171 * Ignored, if the OpenVINO compiled with OpenMP threading and any affinity-related OpenMP's
172 * environment variable is set
173 */
174 DECLARE_CONFIG_KEY(CPU_BIND_THREAD);
175 
176 /**
177 * @brief Optimize CPU execution to maximize throughput.
178 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
179 * - KEY_CPU_THROUGHPUT_NUMA creates as many streams as needed to accomodate NUMA and avoid associated penalties
180 * - KEY_CPU_THROUGHPUT_AUTO creates bare minimum of streams to improve the performance,
181 * this is the most portable option if you have no insights into how many cores you target machine will have
182 * (and what is the optimal number of streams)
183 * - finally, specifying the positive integer value creates the requested number of streams
184 */
185 DECLARE_CONFIG_VALUE(CPU_THROUGHPUT_NUMA);
186 DECLARE_CONFIG_VALUE(CPU_THROUGHPUT_AUTO);
187 DECLARE_CONFIG_KEY(CPU_THROUGHPUT_STREAMS);
188 
189 /**
190 * @brief Optimize GPU plugin execution to maximize throughput.
191 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
192 * - KEY_GPU_THROUGHPUT_AUTO creates bare minimum of streams that might improve performance in some cases,
193 * this option allows to enable throttle hint for opencl queue thus reduce CPU load without significant performance drop
194 * - a positive integer value creates the requested number of streams
195 */
196 DECLARE_CONFIG_VALUE(GPU_THROUGHPUT_AUTO);
197 DECLARE_CONFIG_KEY(GPU_THROUGHPUT_STREAMS);
198 
199 
200 /**
201 * @brief The name for setting performance counters option.
202 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
203 * PluginConfigParams::YES or PluginConfigParams::NO
204 */
205 DECLARE_CONFIG_KEY(PERF_COUNT);
206 
207 /**
208 * @brief The key defines dynamic limit of batch processing.
209 * Specified value is applied to all following Infer() calls. Inference Engine processes
210 * min(batch_limit, original_batch_size) first pictures from input blob. For example, if input
211 * blob has sizes 32x3x224x224 after applying plugin.SetConfig({KEY_DYN_BATCH_LIMIT, 10})
212 * Inference Engine primitives processes only beginner subblobs with size 10x3x224x224.
213 * This value can be changed before any Infer() call to specify a new batch limit.
214 *
215 * The paired parameter value should be convertible to integer number. Acceptable values:
216 * -1 - Do not limit batch processing
217 * >0 - Direct value of limit. Batch size to process is min(new batch_limit, original_batch)
218 */
219 DECLARE_CONFIG_KEY(DYN_BATCH_LIMIT);
220 
221 DECLARE_CONFIG_KEY(DYN_BATCH_ENABLED);
222 
223 /**
224 * @brief The key controls threading inside Inference Engine.
225 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
226 * PluginConfigParams::YES or PluginConfigParams::NO
227 */
228 DECLARE_CONFIG_KEY(SINGLE_THREAD);
229 
230 /**
231 * @brief This key directs the plugin to load a configuration file.
232 * The value should be a file name with the plugin specific configuration
233 */
234 DECLARE_CONFIG_KEY(CONFIG_FILE);
235 
236 /**
237 * @brief This key enables dumping of the kernels used by the plugin for custom layers.
238 * This option should be used with values: PluginConfigParams::YES or PluginConfigParams::NO (default)
239 */
240 DECLARE_CONFIG_KEY(DUMP_KERNELS);
241 
242 /**
243 * @brief This key controls performance tuning done or used by the plugin.
244 * This option should be used with values: PluginConfigParams::TUNING_CREATE,
245 * PluginConfigParams::TUNING_USE_EXISTING or PluginConfigParams::TUNING_DISABLED (default)
246 */
247 DECLARE_CONFIG_KEY(TUNING_MODE);
248 
249 
250 DECLARE_CONFIG_VALUE(TUNING_CREATE);
251 DECLARE_CONFIG_VALUE(TUNING_USE_EXISTING);
252 DECLARE_CONFIG_VALUE(TUNING_DISABLED);
253 
254 /**
255 * @brief This key defines the tuning data filename to be created/used
256 */
257 DECLARE_CONFIG_KEY(TUNING_FILE);
258 
259 /**
260 * @brief the key for setting desirable log level.
261 * This option should be used with values: PluginConfigParams::LOG_NONE (default),
262 * PluginConfigParams::LOG_WARNING, PluginConfigParams::LOG_INFO, PluginConfigParams::LOG_DEBUG
263 */
264 DECLARE_CONFIG_KEY(LOG_LEVEL);
265 
266 DECLARE_CONFIG_VALUE(LOG_NONE);
267 DECLARE_CONFIG_VALUE(LOG_WARNING);
268 DECLARE_CONFIG_VALUE(LOG_INFO);
269 DECLARE_CONFIG_VALUE(LOG_DEBUG);
270 
271 /**
272 * @brief the key for setting of required device to execute on
273 * values: device id starts from "0" - first device, "1" - second device, etc
274 */
275 DECLARE_CONFIG_KEY(DEVICE_ID);
276 
277 /**
278 * @brief the key for enabling exclusive mode for async requests of different executable networks and the same plugin.
279 * Sometimes it's necessary to avoid oversubscription requests that are sharing the same device in parallel.
280 * E.g. There 2 task executors for CPU device: one - in the Hetero plugin, another - in pure CPU plugin.
281 * Parallel execution both of them might lead to oversubscription and not optimal CPU usage. More efficient
282 * to run the corresponding tasks one by one via single executor.
283 * By default, the option is set to YES for hetero cases, and to NO for conventional (single-plugin) cases
284 * Notice that setting YES disables the CPU streams feature (see another config key in this file)
285 */
286 DECLARE_CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS);
287 
288 /**
289  * @brief This key enables dumping of the internal primitive graph.
290  * Should be passed into LoadNetwork method to enable dumping of internal graph of primitives and
291  * corresponding configuration information. Value is a name of output dot file without extension.
292  * Files <dot_file_name>_init.dot and <dot_file_name>_perf.dot will be produced.
293  */
294 DECLARE_CONFIG_KEY(DUMP_EXEC_GRAPH_AS_DOT);
295 
296 } // namespace PluginConfigParams
297 } // namespace InferenceEngine
Definition: ie_argmax_layer.hpp:11