2 // Copyright © 2017 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
6 #include "CpuTensorHandle.hpp"
9 #include <LayersFwd.hpp>
11 #include <armnn/Types.hpp>
12 #include <armnn/LayerSupport.hpp>
14 #include <backendsCommon/LayerSupportRegistry.hpp>
15 #include <backendsCommon/WorkloadFactory.hpp>
17 #include <boost/cast.hpp>
18 #include <boost/iterator/transform_iterator.hpp>
28 const TensorInfo OverrideDataType(const TensorInfo& info, Optional<DataType> type)
35 return TensorInfo(info.GetShape(), type.value(), info.GetQuantizationScale(), info.GetQuantizationOffset());
38 Optional<DataType> GetBiasTypeFromWeightsType(Optional<DataType> weightsType)
45 switch(weightsType.value())
47 case DataType::Float16:
48 case DataType::Float32:
50 case DataType::QuantisedAsymm8:
51 return DataType::Signed32;
53 BOOST_ASSERT_MSG(false, "GetBiasTypeFromWeightsType(): Unsupported data type.");
55 return EmptyOptional();
58 } // anonymous namespace
60 bool IWorkloadFactory::IsLayerSupported(const BackendId& backendId,
61 const IConnectableLayer& connectableLayer,
62 Optional<DataType> dataType,
63 std::string& outReasonIfUnsupported)
65 Optional<std::string&> reason = outReasonIfUnsupported;
67 const Layer& layer = *(boost::polymorphic_downcast<const Layer*>(&connectableLayer));
69 auto const& layerSupportRegistry = LayerSupportRegistryInstance();
70 auto layerSupportFactory = layerSupportRegistry.GetFactory(backendId);
71 auto layerSupportObject = layerSupportFactory(EmptyInitializer());
73 switch(layer.GetType())
75 case LayerType::Activation:
77 auto cLayer = boost::polymorphic_downcast<const ActivationLayer*>(&layer);
78 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
79 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
80 result = layerSupportObject->IsActivationSupported(
81 OverrideDataType(input, dataType),
82 OverrideDataType(output, dataType),
83 cLayer->GetParameters(),
87 case LayerType::Addition:
89 const TensorInfo& input0 = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
90 const TensorInfo& input1 = layer.GetInputSlot(1).GetConnection()->GetTensorInfo();
91 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
92 result = layerSupportObject->IsAdditionSupported(
93 OverrideDataType(input0, dataType),
94 OverrideDataType(input1, dataType),
95 OverrideDataType(output, dataType),
99 case LayerType::BatchNormalization:
101 auto cLayer = boost::polymorphic_downcast<const BatchNormalizationLayer*>(&layer);
102 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
103 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
104 const TensorInfo& mean = cLayer->m_Mean->GetTensorInfo();
105 const TensorInfo& var = cLayer->m_Variance->GetTensorInfo();
106 const TensorInfo& beta = cLayer->m_Beta->GetTensorInfo();
107 const TensorInfo& gamma = cLayer->m_Gamma->GetTensorInfo();
108 result = layerSupportObject->IsBatchNormalizationSupported(
109 OverrideDataType(input, dataType),
110 OverrideDataType(output, dataType),
111 OverrideDataType(mean, dataType),
112 OverrideDataType(var, dataType),
113 OverrideDataType(beta, dataType),
114 OverrideDataType(gamma, dataType),
115 cLayer->GetParameters(),
119 case LayerType::Constant:
121 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
122 result = layerSupportObject->IsConstantSupported(OverrideDataType(output, dataType), reason);
125 case LayerType::ConvertFp16ToFp32:
127 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
128 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
129 result = layerSupportObject->IsConvertFp16ToFp32Supported(input, output, reason);
132 case LayerType::ConvertFp32ToFp16:
134 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
135 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
136 result = layerSupportObject->IsConvertFp32ToFp16Supported(input, output, reason);
139 case LayerType::Convolution2d:
141 auto cLayer = boost::polymorphic_downcast<const Convolution2dLayer*>(&layer);
143 const TensorInfo input = OverrideDataType(layer.GetInputSlot(0).GetConnection()->GetTensorInfo(),
145 const TensorInfo output = OverrideDataType(layer.GetOutputSlot(0).GetTensorInfo(), dataType);
146 BOOST_ASSERT(cLayer->m_Weight.get() != nullptr);
148 const Convolution2dDescriptor& descriptor = cLayer->GetParameters();
150 // Construct optional biases object based on the value of m_BiasEnabled
151 Optional<TensorInfo> biases;
152 if (descriptor.m_BiasEnabled)
155 OverrideDataType(cLayer->m_Bias->GetTensorInfo(), GetBiasTypeFromWeightsType(dataType));
158 result = layerSupportObject->IsConvolution2dSupported(
162 OverrideDataType(cLayer->m_Weight->GetTensorInfo(), dataType),
167 case LayerType::MemCopy:
169 // MemCopy supported for CpuRef, CpuAcc and GpuAcc backends,
170 // (also treat Undefined as CpuRef to avoid breaking lots of Unit tests).
171 result = backendId == Compute::CpuRef || backendId == Compute::Undefined
172 || backendId == Compute::CpuAcc || backendId == Compute::GpuAcc;
173 reason.value() = "Unsupported backend type";
176 case LayerType::DepthwiseConvolution2d:
178 auto cLayer = boost::polymorphic_downcast<const DepthwiseConvolution2dLayer*>(&layer);
179 const TensorInfo& input = OverrideDataType(layer.GetInputSlot(0).GetConnection()->GetTensorInfo(),
181 const TensorInfo& output = OverrideDataType(layer.GetOutputSlot(0).GetTensorInfo(), dataType);
182 BOOST_ASSERT(cLayer->m_Weight.get() != nullptr);
184 const DepthwiseConvolution2dDescriptor& descriptor = cLayer->GetParameters();
186 // Construct optional biases object based on the value of m_BiasEnabled
187 Optional<TensorInfo> biases;
188 if (descriptor.m_BiasEnabled)
191 OverrideDataType(cLayer->m_Bias->GetTensorInfo(), GetBiasTypeFromWeightsType(dataType));
194 result = layerSupportObject->IsDepthwiseConvolutionSupported(
198 OverrideDataType(cLayer->m_Weight->GetTensorInfo(), dataType),
203 case LayerType::FakeQuantization:
205 auto cLayer = boost::polymorphic_downcast<const FakeQuantizationLayer*>(&layer);
206 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
207 result = layerSupportObject->IsFakeQuantizationSupported(OverrideDataType(input, dataType),
208 cLayer->GetParameters(),
212 case LayerType::Floor:
214 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
215 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
216 result = layerSupportObject->IsFloorSupported(OverrideDataType(input, dataType),
217 OverrideDataType(output, dataType),
221 case LayerType::FullyConnected:
223 auto cLayer = boost::polymorphic_downcast<const FullyConnectedLayer*>(&layer);
224 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
225 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
226 BOOST_ASSERT(cLayer->m_Weight.get() != nullptr);
229 const TensorInfo * biasInfoPtr = nullptr;
230 static const TensorInfo dummyFloat16Bias(TensorShape({1,1,1,1}), DataType::Float16);
231 static const TensorInfo dummyFloat32Bias(TensorShape({1,1,1,1}), DataType::Float32);
232 static const TensorInfo dummyQA8Bias(TensorShape({1,1,1,1}), DataType::Signed32);
234 const FullyConnectedDescriptor& descriptor = cLayer->GetParameters();
235 if (descriptor.m_BiasEnabled)
237 BOOST_ASSERT(cLayer->m_Bias.get() != nullptr);
238 biasInfo = OverrideDataType(cLayer->m_Bias->GetTensorInfo(), GetBiasTypeFromWeightsType(dataType));
239 biasInfoPtr = &biasInfo;
243 // If biases are not enabled pass a dummy tensorinfo for the validation
244 switch(input.GetDataType())
246 case DataType::Float16:
248 biasInfoPtr = &dummyFloat16Bias;
251 case DataType::Float32:
253 biasInfoPtr = &dummyFloat32Bias;
256 case DataType::QuantisedAsymm8:
258 biasInfoPtr = &dummyQA8Bias;
263 BOOST_ASSERT_MSG(false, "Unexpected bias type");
268 result = layerSupportObject->IsFullyConnectedSupported(
269 OverrideDataType(input, dataType),
270 OverrideDataType(output, dataType),
271 OverrideDataType(cLayer->m_Weight->GetTensorInfo(), dataType),
277 case LayerType::Input:
279 const TensorInfo& input = layer.GetOutputSlot(0).GetTensorInfo();
280 result = layerSupportObject->IsInputSupported(OverrideDataType(input, dataType), reason);
283 case LayerType::L2Normalization:
285 auto cLayer = boost::polymorphic_downcast<const L2NormalizationLayer*>(&layer);
286 const L2NormalizationDescriptor& descriptor = cLayer->GetParameters();
288 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
289 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
291 result = layerSupportObject->IsL2NormalizationSupported(
292 OverrideDataType(input, dataType),
293 OverrideDataType(output, dataType),
298 case LayerType::Lstm:
300 auto cLayer = boost::polymorphic_downcast<const LstmLayer*>(&layer);
301 const LstmDescriptor& descriptor = cLayer->GetParameters();
304 const TensorInfo& input = OverrideDataType(layer.GetInputSlot(0).GetConnection()->GetTensorInfo(),
306 const TensorInfo& outputStateIn = OverrideDataType(layer.GetInputSlot(1).GetConnection()->GetTensorInfo(),
308 const TensorInfo& cellStateIn = OverrideDataType(layer.GetInputSlot(2).GetConnection()->GetTensorInfo(),
311 const TensorInfo& scratchBuffer = OverrideDataType(layer.GetOutputSlot(0).GetTensorInfo(), dataType);
312 const TensorInfo& outputStateOut = OverrideDataType(layer.GetOutputSlot(1).GetTensorInfo(), dataType);
313 const TensorInfo& cellStateOut = OverrideDataType(layer.GetOutputSlot(2).GetTensorInfo(), dataType);
314 const TensorInfo& output = OverrideDataType(layer.GetOutputSlot(3).GetTensorInfo(), dataType);
317 const TensorInfo& inputToForgetWeights
318 = OverrideDataType(cLayer->m_BasicParameters.m_InputToForgetWeights->GetTensorInfo(), dataType);
319 const TensorInfo& inputToCellWeights
320 = OverrideDataType(cLayer->m_BasicParameters.m_InputToCellWeights->GetTensorInfo(), dataType);
321 const TensorInfo& inputToOutputWeights
322 = OverrideDataType(cLayer->m_BasicParameters.m_InputToOutputWeights->GetTensorInfo(), dataType);
323 const TensorInfo& recurrentToForgetWeights
324 = OverrideDataType(cLayer->m_BasicParameters.m_RecurrentToForgetWeights->GetTensorInfo(), dataType);
325 const TensorInfo& recurrentToCellWeights
326 = OverrideDataType(cLayer->m_BasicParameters.m_RecurrentToCellWeights->GetTensorInfo(), dataType);
327 const TensorInfo& recurrentToOutputWeights
328 = OverrideDataType(cLayer->m_BasicParameters.m_RecurrentToOutputWeights->GetTensorInfo(), dataType);
329 const TensorInfo& forgetGateBias
330 = OverrideDataType(cLayer->m_BasicParameters.m_ForgetGateBias->GetTensorInfo(), dataType);
331 const TensorInfo& cellBias
332 = OverrideDataType(cLayer->m_BasicParameters.m_CellBias->GetTensorInfo(), dataType);
333 const TensorInfo& outputGateBias
334 = OverrideDataType(cLayer->m_BasicParameters.m_OutputGateBias->GetTensorInfo(), dataType);
336 // Optional parameters
337 const TensorInfo* inputToInputWeights = nullptr;
338 const TensorInfo* recurrentToInputWeights = nullptr;
339 const TensorInfo* cellToInputWeights = nullptr;
340 const TensorInfo* inputGateBias = nullptr;
341 const TensorInfo* projectionWeights = nullptr;
342 const TensorInfo* projectionBias = nullptr;
343 const TensorInfo* cellToForgetWeights = nullptr;
344 const TensorInfo* cellToOutputWeights = nullptr;
346 TensorInfo optInputToInputWeights;
347 TensorInfo optRecurrentToInputWeights;
348 TensorInfo optCellToInputWeights;
349 TensorInfo optInputGateBias;
350 TensorInfo optProjectionWeights;
351 TensorInfo optProjectionBias;
352 TensorInfo optCellToForgetWeights;
353 TensorInfo optCellToOutputWeights;
355 if(!descriptor.m_CifgEnabled)
357 optInputToInputWeights =
358 OverrideDataType(cLayer->m_CifgParameters.m_InputToInputWeights->GetTensorInfo(), dataType);
359 inputToInputWeights = &optInputToInputWeights;
361 optRecurrentToInputWeights =
362 OverrideDataType(cLayer->m_CifgParameters.m_RecurrentToInputWeights->GetTensorInfo(), dataType);
363 recurrentToInputWeights = &optRecurrentToInputWeights;
364 if (cLayer->m_CifgParameters.m_CellToInputWeights != nullptr)
366 optCellToInputWeights =
367 OverrideDataType(cLayer->m_CifgParameters.m_CellToInputWeights->GetTensorInfo(), dataType);
368 cellToInputWeights = &optCellToInputWeights;
371 OverrideDataType(cLayer->m_CifgParameters.m_InputGateBias->GetTensorInfo(), dataType);
372 inputGateBias = &optInputGateBias;
375 if(descriptor.m_ProjectionEnabled)
377 optProjectionWeights =
378 OverrideDataType(cLayer->m_ProjectionParameters.m_ProjectionWeights->GetTensorInfo(), dataType);
379 projectionWeights = &optProjectionWeights;
380 if (cLayer->m_ProjectionParameters.m_ProjectionBias != nullptr)
383 OverrideDataType(cLayer->m_ProjectionParameters.m_ProjectionBias->GetTensorInfo(), dataType);
384 projectionBias = &optProjectionBias;
388 if(descriptor.m_PeepholeEnabled)
390 optCellToForgetWeights =
391 OverrideDataType(cLayer->m_PeepholeParameters.m_CellToForgetWeights->GetTensorInfo(), dataType);
392 cellToForgetWeights = &optCellToForgetWeights;
393 optCellToOutputWeights =
394 OverrideDataType(cLayer->m_PeepholeParameters.m_CellToOutputWeights->GetTensorInfo(), dataType);
395 cellToOutputWeights = &optCellToOutputWeights;
398 result = layerSupportObject->IsLstmSupported(
407 inputToForgetWeights,
409 inputToOutputWeights,
410 recurrentToForgetWeights,
411 recurrentToCellWeights,
412 recurrentToOutputWeights,
417 recurrentToInputWeights,
427 case LayerType::Merger:
429 auto cLayer = boost::polymorphic_downcast<const MergerLayer*>(&layer);
431 // Get vector of all inputs.
432 auto getTensorInfo = [&dataType](const InputSlot& slot)
434 return OverrideDataType(slot.GetConnectedOutputSlot()->GetTensorInfo(), dataType);
436 auto beginI = boost::make_transform_iterator(layer.GetInputSlots().begin(), getTensorInfo);
437 auto endI = boost::make_transform_iterator(layer.GetInputSlots().end(), getTensorInfo);
438 std::vector<TensorInfo> inputs(beginI, endI);
440 auto getTensorInfoPtr = [](const TensorInfo& info)
444 auto beginPtr = boost::make_transform_iterator(inputs.begin(), getTensorInfoPtr);
445 auto endPtr = boost::make_transform_iterator(inputs.end(), getTensorInfoPtr);
446 std::vector<const TensorInfo*> inputPtrs(beginPtr, endPtr);
448 result = layerSupportObject->IsMergerSupported(inputPtrs, cLayer->GetParameters(), reason);
451 case LayerType::Multiplication:
453 const TensorInfo& input0 = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
454 const TensorInfo& input1 = layer.GetInputSlot(1).GetConnection()->GetTensorInfo();
455 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
456 result = layerSupportObject->IsMultiplicationSupported(
457 OverrideDataType(input0, dataType),
458 OverrideDataType(input1, dataType),
459 OverrideDataType(output, dataType),
463 case LayerType::Normalization:
465 auto cLayer = boost::polymorphic_downcast<const NormalizationLayer*>(&layer);
466 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
467 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
468 result = layerSupportObject->IsNormalizationSupported(OverrideDataType(input, dataType),
469 OverrideDataType(output, dataType),
470 cLayer->GetParameters(),
474 case LayerType::Output:
476 const TensorInfo& output = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
477 result = layerSupportObject->IsOutputSupported(OverrideDataType(output, dataType), reason);
480 case LayerType::Permute:
482 auto cLayer = boost::polymorphic_downcast<const PermuteLayer*>(&layer);
483 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
484 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
485 result = layerSupportObject->IsPermuteSupported(OverrideDataType(input, dataType),
486 OverrideDataType(output, dataType),
487 cLayer->GetParameters(),
493 auto cLayer = boost::polymorphic_downcast<const PadLayer*>(&layer);
494 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
495 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
496 result = layerSupportObject->IsPadSupported(
497 OverrideDataType(input, dataType),
498 OverrideDataType(output, dataType),
499 cLayer->GetParameters(),
503 case LayerType::Pooling2d:
505 auto cLayer = boost::polymorphic_downcast<const Pooling2dLayer*>(&layer);
506 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
507 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
508 result = layerSupportObject->IsPooling2dSupported(OverrideDataType(input, dataType),
509 OverrideDataType(output, dataType),
510 cLayer->GetParameters(),
514 case LayerType::Division:
516 const TensorInfo& input0 = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
517 const TensorInfo& input1 = layer.GetInputSlot(1).GetConnection()->GetTensorInfo();
518 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
519 result = layerSupportObject->IsDivisionSupported(
520 OverrideDataType(input0, dataType),
521 OverrideDataType(input1, dataType),
522 OverrideDataType(output, dataType),
526 case LayerType::Reshape:
528 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
529 result = layerSupportObject->IsReshapeSupported(OverrideDataType(input, dataType), reason);
532 case LayerType::ResizeBilinear:
534 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
535 result = layerSupportObject->IsResizeBilinearSupported(OverrideDataType(input, dataType), reason);
538 case LayerType::Softmax:
540 auto cLayer = boost::polymorphic_downcast<const SoftmaxLayer*>(&layer);
541 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
542 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
543 result = layerSupportObject->IsSoftmaxSupported(OverrideDataType(input, dataType),
544 OverrideDataType(output, dataType),
545 cLayer->GetParameters(),
549 case LayerType::SpaceToBatchNd:
551 auto cLayer = boost::polymorphic_downcast<const SpaceToBatchNdLayer*>(&layer);
552 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
553 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
554 result = layerSupportObject->IsSpaceToBatchNdSupported(OverrideDataType(input, dataType),
555 OverrideDataType(output, dataType),
556 cLayer->GetParameters(),
560 case LayerType::Splitter:
562 auto cLayer = boost::polymorphic_downcast<const SplitterLayer*>(&layer);
563 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
564 result = layerSupportObject->IsSplitterSupported(OverrideDataType(input, dataType),
565 cLayer->GetParameters(),
569 case LayerType::Subtraction:
571 const TensorInfo& input0 = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
572 const TensorInfo& input1 = layer.GetInputSlot(1).GetConnection()->GetTensorInfo();
573 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
574 result = layerSupportObject->IsSubtractionSupported(
575 OverrideDataType(input0, dataType),
576 OverrideDataType(input1, dataType),
577 OverrideDataType(output, dataType),
581 case LayerType::Mean:
583 auto cLayer = boost::polymorphic_downcast<const MeanLayer*>(&layer);
584 const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
585 const TensorInfo& output = layer.GetOutputSlot(0).GetTensorInfo();
586 result = layerSupportObject->IsMeanSupported(
587 OverrideDataType(input, dataType),
588 OverrideDataType(output, dataType),
589 cLayer->GetParameters(),
595 BOOST_ASSERT_MSG(false, "WorkloadFactory did not recognise type of layer.");
596 reason.value() = "Unrecognised layer type";
604 bool IWorkloadFactory::IsLayerSupported(const IConnectableLayer& connectableLayer,
605 Optional<DataType> dataType,
606 std::string& outReasonIfUnsupported)
608 auto layer = boost::polymorphic_downcast<const Layer*>(&connectableLayer);
609 return IsLayerSupported(layer->GetBackendId(), connectableLayer, dataType, outReasonIfUnsupported);