2 // Copyright © 2017 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
7 #include "CpuTensorHandleFwd.hpp"
8 #include "WorkloadDataFwd.hpp"
10 #include <InternalTypes.hpp>
12 #include <armnn/Deprecated.hpp>
13 #include <armnn/Descriptors.hpp>
14 #include <armnn/Exceptions.hpp>
15 #include <armnn/Types.hpp>
16 #include <armnn/Tensor.hpp>
18 #include <backendsCommon/OutputHandler.hpp>
19 #include <backendsCommon/WorkloadInfo.hpp>
24 //A helper function that returns the bias data type required for given input data type.
25 DataType GetBiasDataType(DataType inputDataType);
29 struct QueueDescriptor
31 std::vector<ITensorHandle*> m_Inputs;
32 std::vector<ITensorHandle*> m_Outputs;
34 void ValidateInputsOutputs(const std::string& descName,
35 unsigned int numExpectedIn, unsigned int numExpectedOut) const;
39 ~QueueDescriptor() = default;
40 QueueDescriptor() = default;
41 QueueDescriptor(QueueDescriptor const&) = default;
42 QueueDescriptor& operator=(QueueDescriptor const&) = default;
45 // Base class for queue descriptors which contain parameters.
46 template <typename LayerDescriptor>
47 struct QueueDescriptorWithParameters : public QueueDescriptor
49 LayerDescriptor m_Parameters;
52 ~QueueDescriptorWithParameters() = default;
53 QueueDescriptorWithParameters() = default;
54 QueueDescriptorWithParameters(QueueDescriptorWithParameters const&) = default;
55 QueueDescriptorWithParameters& operator=(QueueDescriptorWithParameters const&) = default;
58 struct MemCopyQueueDescriptor : QueueDescriptor
60 void Validate(const WorkloadInfo& workloadInfo) const;
63 using InputQueueDescriptor = MemCopyQueueDescriptor;
64 using OutputQueueDescriptor = MemCopyQueueDescriptor;
66 struct MemImportQueueDescriptor : QueueDescriptor
68 void Validate(const WorkloadInfo& workloadInfo) const;
71 struct MemSyncQueueDescriptor : QueueDescriptor
73 void Validate(const WorkloadInfo& workloadInfo) const;
76 // Softmax layer workload data.
77 struct SoftmaxQueueDescriptor : QueueDescriptorWithParameters<SoftmaxDescriptor>
79 void Validate(const WorkloadInfo& workloadInfo) const;
82 // Splitter layer workload data.
83 struct SplitterQueueDescriptor : QueueDescriptorWithParameters<ViewsDescriptor>
88 ViewOrigin(std::vector<unsigned int> const& origin) : m_Origin(origin) {}
90 //View origin (size of the vector is the same as number of dimensions of the view).
91 std::vector<unsigned int> m_Origin;
94 //View defines a tensor that will be carved from the input tensor.
95 //View origins are stored here, the extents are defined by sizes of the output tensors.
96 std::vector<ViewOrigin> m_ViewOrigins;
98 void Validate(const WorkloadInfo& workloadInfo) const;
101 // Concat layer workload data.
102 struct ConcatQueueDescriptor : QueueDescriptorWithParameters<OriginsDescriptor>
107 ViewOrigin(const std::vector<unsigned int>& origin) : m_Origin(origin) {}
109 //View origin (size of the vector is the same as number of dimensions of the view).
110 std::vector<unsigned int> m_Origin;
113 //View defines a sub-area of the output tensor that will be filled with the corresponding input tensor.
114 //View origins are stored here, the extents are defined by sizes of the input tensors.
115 std::vector<ViewOrigin> m_ViewOrigins;
117 void Validate(const WorkloadInfo& workloadInfo) const;
120 // Deprecated. Use ConcatQueueDescriptor instead
121 using MergerQueueDescriptor = ConcatQueueDescriptor;
123 // Stack layer workload data.
124 struct StackQueueDescriptor : QueueDescriptorWithParameters<StackDescriptor>
126 void Validate(const WorkloadInfo& workloadInfo) const;
129 // Activation layer workload data.
130 struct ActivationQueueDescriptor : QueueDescriptorWithParameters<ActivationDescriptor>
132 void Validate(const WorkloadInfo& workloadInfo) const;
135 struct ArgMinMaxQueueDescriptor : QueueDescriptorWithParameters<ArgMinMaxDescriptor>
137 void Validate(const WorkloadInfo& workloadInfo) const;
140 // Fully connected layer workload data.
141 struct FullyConnectedQueueDescriptor : QueueDescriptorWithParameters<FullyConnectedDescriptor>
143 FullyConnectedQueueDescriptor()
149 const ConstCpuTensorHandle* m_Weight;
150 const ConstCpuTensorHandle* m_Bias;
152 void Validate(const WorkloadInfo& workloadInfo) const;
155 // Permute layer workload data.
156 struct PermuteQueueDescriptor : QueueDescriptorWithParameters<PermuteDescriptor>
158 void Validate(const WorkloadInfo& workloadInfo) const;
161 // Pooling 2D layer workload data.
162 struct Pooling2dQueueDescriptor : QueueDescriptorWithParameters<Pooling2dDescriptor>
164 void Validate(const WorkloadInfo& workloadInfo) const;
167 // Convolution 2D layer workload data.
168 struct Convolution2dQueueDescriptor : QueueDescriptorWithParameters<Convolution2dDescriptor>
170 Convolution2dQueueDescriptor()
176 const ConstCpuTensorHandle* m_Weight;
177 const ConstCpuTensorHandle* m_Bias;
179 void Validate(const WorkloadInfo& workloadInfo) const;
182 // Depthwise Convolution 2D layer workload data.
183 struct DepthwiseConvolution2dQueueDescriptor : QueueDescriptorWithParameters<DepthwiseConvolution2dDescriptor>
185 DepthwiseConvolution2dQueueDescriptor()
191 const ConstCpuTensorHandle* m_Weight;
192 const ConstCpuTensorHandle* m_Bias;
194 void Validate(const WorkloadInfo& workloadInfo) const;
197 struct DetectionPostProcessQueueDescriptor : QueueDescriptorWithParameters<DetectionPostProcessDescriptor>
199 DetectionPostProcessQueueDescriptor()
204 const ConstCpuTensorHandle* m_Anchors;
206 void Validate(const WorkloadInfo& workloadInfo) const;
209 // Normalization layer workload data.
210 struct NormalizationQueueDescriptor : QueueDescriptorWithParameters<NormalizationDescriptor>
212 void Validate(const WorkloadInfo& workloadInfo) const;
215 // Add layer workload data.
216 struct AdditionQueueDescriptor : QueueDescriptor
218 void Validate(const WorkloadInfo& workloadInfo) const;
221 // Multiplication layer workload data.
222 struct MultiplicationQueueDescriptor : QueueDescriptor
224 void Validate(const WorkloadInfo& workloadInfo) const;
227 // Division layer workload data.
228 struct DivisionQueueDescriptor : QueueDescriptor
230 void Validate(const WorkloadInfo& workloadInfo) const;
233 // Subtraction layer workload data.
234 struct SubtractionQueueDescriptor : QueueDescriptor
236 void Validate(const WorkloadInfo& workloadInfo) const;
239 // Maximum layer workload data.
240 struct MaximumQueueDescriptor : QueueDescriptor
242 void Validate(const WorkloadInfo& workloadInfo) const;
245 // Mean layer workload data.
246 struct MeanQueueDescriptor : QueueDescriptorWithParameters<MeanDescriptor>
248 void Validate(const WorkloadInfo& workloadInfo) const;
251 // Pad layer workload data
252 struct PadQueueDescriptor : QueueDescriptorWithParameters<PadDescriptor>
254 void Validate(const WorkloadInfo& workloadInfo) const;
257 struct QuantizeQueueDescriptor : QueueDescriptor
259 void Validate(const WorkloadInfo& workloadInfo) const;
262 // Equal layer workload data
263 struct EqualQueueDescriptor : QueueDescriptor
265 void Validate(const WorkloadInfo& workloadInfo) const;
268 // Batch norm layer workload data.
269 struct BatchNormalizationQueueDescriptor : QueueDescriptorWithParameters<BatchNormalizationDescriptor>
271 BatchNormalizationQueueDescriptor()
273 , m_Variance(nullptr)
279 const ConstCpuTensorHandle* m_Mean;
280 const ConstCpuTensorHandle* m_Variance;
281 const ConstCpuTensorHandle* m_Beta;
282 const ConstCpuTensorHandle* m_Gamma;
284 void Validate(const WorkloadInfo& workloadInfo) const;
287 struct ResizeBilinearQueueDescriptor : QueueDescriptorWithParameters<ResizeBilinearDescriptor>
289 void Validate(const WorkloadInfo& workloadInfo) const;
292 struct ResizeQueueDescriptor : QueueDescriptorWithParameters<ResizeDescriptor>
294 void Validate(const WorkloadInfo& workloadInfo) const;
297 struct FakeQuantizationQueueDescriptor : QueueDescriptorWithParameters<FakeQuantizationDescriptor>
299 FakeQuantizationQueueDescriptor()
305 const ConstCpuTensorHandle* m_Min;
306 const ConstCpuTensorHandle* m_Max;
308 void Validate(const WorkloadInfo& workloadInfo) const;
311 struct L2NormalizationQueueDescriptor : QueueDescriptorWithParameters<L2NormalizationDescriptor>
313 void Validate(const WorkloadInfo& workloadInfo) const;
316 struct ConstantQueueDescriptor : QueueDescriptor
318 ConstantQueueDescriptor()
319 : m_LayerOutput(nullptr)
323 const ConstCpuTensorHandle* m_LayerOutput;
325 void Validate(const WorkloadInfo& workloadInfo) const;
328 struct ReshapeQueueDescriptor : QueueDescriptorWithParameters<ReshapeDescriptor>
330 void Validate(const WorkloadInfo& workloadInfo) const;
333 struct SpaceToBatchNdQueueDescriptor : QueueDescriptorWithParameters<SpaceToBatchNdDescriptor>
335 void Validate(const WorkloadInfo& workloadInfo) const;
338 struct SpaceToDepthQueueDescriptor : QueueDescriptorWithParameters<SpaceToDepthDescriptor>
340 void Validate(const WorkloadInfo& workloadInfo) const;
343 struct FloorQueueDescriptor : QueueDescriptor
345 void Validate(const WorkloadInfo& workloadInfo) const;
348 struct LstmQueueDescriptor : QueueDescriptorWithParameters<LstmDescriptor>
350 LstmQueueDescriptor()
351 : m_InputToInputWeights(nullptr)
352 , m_InputToForgetWeights(nullptr)
353 , m_InputToCellWeights(nullptr)
354 , m_InputToOutputWeights(nullptr)
355 , m_RecurrentToInputWeights(nullptr)
356 , m_RecurrentToForgetWeights(nullptr)
357 , m_RecurrentToCellWeights(nullptr)
358 , m_RecurrentToOutputWeights(nullptr)
359 , m_CellToInputWeights(nullptr)
360 , m_CellToForgetWeights(nullptr)
361 , m_CellToOutputWeights(nullptr)
362 , m_InputGateBias(nullptr)
363 , m_ForgetGateBias(nullptr)
364 , m_CellBias(nullptr)
365 , m_OutputGateBias(nullptr)
366 , m_ProjectionWeights(nullptr)
367 , m_ProjectionBias(nullptr)
368 , m_InputLayerNormWeights(nullptr)
369 , m_ForgetLayerNormWeights(nullptr)
370 , m_CellLayerNormWeights(nullptr)
371 , m_OutputLayerNormWeights(nullptr)
375 const ConstCpuTensorHandle* m_InputToInputWeights;
376 const ConstCpuTensorHandle* m_InputToForgetWeights;
377 const ConstCpuTensorHandle* m_InputToCellWeights;
378 const ConstCpuTensorHandle* m_InputToOutputWeights;
379 const ConstCpuTensorHandle* m_RecurrentToInputWeights;
380 const ConstCpuTensorHandle* m_RecurrentToForgetWeights;
381 const ConstCpuTensorHandle* m_RecurrentToCellWeights;
382 const ConstCpuTensorHandle* m_RecurrentToOutputWeights;
383 const ConstCpuTensorHandle* m_CellToInputWeights;
384 const ConstCpuTensorHandle* m_CellToForgetWeights;
385 const ConstCpuTensorHandle* m_CellToOutputWeights;
386 const ConstCpuTensorHandle* m_InputGateBias;
387 const ConstCpuTensorHandle* m_ForgetGateBias;
388 const ConstCpuTensorHandle* m_CellBias;
389 const ConstCpuTensorHandle* m_OutputGateBias;
390 const ConstCpuTensorHandle* m_ProjectionWeights;
391 const ConstCpuTensorHandle* m_ProjectionBias;
392 const ConstCpuTensorHandle* m_InputLayerNormWeights;
393 const ConstCpuTensorHandle* m_ForgetLayerNormWeights;
394 const ConstCpuTensorHandle* m_CellLayerNormWeights;
395 const ConstCpuTensorHandle* m_OutputLayerNormWeights;
397 void Validate(const WorkloadInfo& workloadInfo) const;
400 struct ConvertFp16ToFp32QueueDescriptor : QueueDescriptor
402 void Validate(const WorkloadInfo& workloadInfo) const;
405 struct ConvertFp32ToFp16QueueDescriptor : QueueDescriptor
407 void Validate(const WorkloadInfo& workloadInfo) const;
410 struct BatchToSpaceNdQueueDescriptor : QueueDescriptorWithParameters<BatchToSpaceNdDescriptor>
412 void Validate(const WorkloadInfo& workloadInfo) const;
415 struct StridedSliceQueueDescriptor : QueueDescriptorWithParameters<StridedSliceDescriptor>
417 void Validate(const WorkloadInfo& workloadInfo) const;
420 // Minimum layer workload data.
421 struct MinimumQueueDescriptor : QueueDescriptor
423 void Validate(const WorkloadInfo& workloadInfo) const;
426 struct GreaterQueueDescriptor : QueueDescriptor
428 void Validate(const WorkloadInfo& workloadInfo) const;
431 struct DebugQueueDescriptor : QueueDescriptor
433 void Validate(const WorkloadInfo& workloadInfo) const;
436 std::string m_LayerName;
437 unsigned int m_SlotIndex;
440 struct RsqrtQueueDescriptor : QueueDescriptor
442 void Validate(const WorkloadInfo& workloadInfo) const;
445 struct GatherQueueDescriptor : QueueDescriptor
447 void Validate(const WorkloadInfo& workloadInfo) const;
450 struct PreCompiledQueueDescriptor : QueueDescriptorWithParameters<PreCompiledDescriptor>
452 PreCompiledQueueDescriptor()
453 : m_PreCompiledObject(nullptr)
457 void* m_PreCompiledObject;
459 void Validate(const WorkloadInfo& workloadInfo) const;
462 struct DequantizeQueueDescriptor : QueueDescriptor
464 void Validate(const WorkloadInfo& workloadInfo) const;
467 struct MergeQueueDescriptor : QueueDescriptor
469 void Validate(const WorkloadInfo& workloadInfo) const;
472 struct SwitchQueueDescriptor : QueueDescriptor
474 void Validate(const WorkloadInfo& workloadInfo) const;
477 struct PreluQueueDescriptor : QueueDescriptor
479 void Validate(const WorkloadInfo& workloadInfo) const;
482 struct TransposeConvolution2dQueueDescriptor : QueueDescriptorWithParameters<TransposeConvolution2dDescriptor>
484 TransposeConvolution2dQueueDescriptor() :
489 const ConstCpuTensorHandle* m_Weight;
490 const ConstCpuTensorHandle* m_Bias;
492 void Validate(const WorkloadInfo& workloadInfo) const;
495 struct QuantizedLstmQueueDescriptor : QueueDescriptor
497 QuantizedLstmQueueDescriptor()
498 : m_InputToInputWeights(nullptr)
499 , m_InputToForgetWeights(nullptr)
500 , m_InputToCellWeights(nullptr)
501 , m_InputToOutputWeights(nullptr)
503 , m_RecurrentToInputWeights(nullptr)
504 , m_RecurrentToForgetWeights(nullptr)
505 , m_RecurrentToCellWeights(nullptr)
506 , m_RecurrentToOutputWeights(nullptr)
508 , m_InputGateBias(nullptr)
509 , m_ForgetGateBias(nullptr)
510 , m_CellBias(nullptr)
511 , m_OutputGateBias(nullptr)
514 const ConstCpuTensorHandle* m_InputToInputWeights;
515 const ConstCpuTensorHandle* m_InputToForgetWeights;
516 const ConstCpuTensorHandle* m_InputToCellWeights;
517 const ConstCpuTensorHandle* m_InputToOutputWeights;
519 const ConstCpuTensorHandle* m_RecurrentToInputWeights;
520 const ConstCpuTensorHandle* m_RecurrentToForgetWeights;
521 const ConstCpuTensorHandle* m_RecurrentToCellWeights;
522 const ConstCpuTensorHandle* m_RecurrentToOutputWeights;
524 const ConstCpuTensorHandle* m_InputGateBias;
525 const ConstCpuTensorHandle* m_ForgetGateBias;
526 const ConstCpuTensorHandle* m_CellBias;
527 const ConstCpuTensorHandle* m_OutputGateBias;
529 void Validate(const WorkloadInfo& workloadInfo) const;
532 struct AbsQueueDescriptor : QueueDescriptor
534 void Validate(const WorkloadInfo& workloadInfo) const;