36 #include <boost/assert.hpp> 37 #include <boost/format.hpp> 38 #include <boost/numeric/conversion/converter_policies.hpp> 39 #include <boost/cast.hpp> 56 delete boost::polymorphic_downcast<Network*>(network);
61 delete boost::polymorphic_downcast<OptimizedNetwork*>(network);
72 return m_Graph->SerializeToDot(stream);
76 Optional<std::vector<std::string>&> errorMessages)
78 std::stringstream fullErrorMessage;
79 fullErrorMessage <<
"ERROR: " << errorMessage;
83 errorMessages.value().push_back(fullErrorMessage.str());
88 Optional<std::vector<std::string>&> warningMessages)
90 std::stringstream fullWarningMessage;
91 fullWarningMessage <<
"WARNING: " << warningMessage;
95 warningMessages.value().push_back(fullWarningMessage.str());
102 Optional<std::vector<std::string>&> errMessages)
104 std::stringstream failureMsg;
116 bool noErrors =
true;
118 for (
unsigned int i = 0; i < numOutputs; i++) {
124 std::stringstream ss;
126 <<
" (" << layer->
GetNameStr() <<
") is of type" 127 <<
" Quantized 8 bit but its scale parameter has not been set";
135 std::stringstream ss;
136 ss <<
"Quantization parameters for Softmax layer (Scale: " <<
138 ") are incorrect and have been updated to Scale: 0.00390625 and Offset: 0";
155 const std::vector<BackendId>& availablePreferredBackends,
156 std::string& reasonIfUnsupported,
157 Optional<std::vector<std::string>&> errMessages)
162 auto ReturnError = [&](
const Layer* layer)
179 std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
182 convertFp16ToFp32Layers =
187 std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
190 convertFp32ToFp16Layers =
195 auto AssignFirstSupportedBackend = [&](
Layer* layer,
BackendId preferredBackend)
197 bool supportedBackendFound =
false;
198 std::string reasonIfUnsupported;
204 reasonIfUnsupported))
206 supportedBackendFound =
true;
210 for (
const auto& backend : availablePreferredBackends)
213 if (backend == preferredBackend)
221 reasonIfUnsupported))
223 supportedBackendFound =
true;
229 return supportedBackendFound;
234 if (!AssignFirstSupportedBackend(convertLayer, backend))
236 return ReturnError(convertLayer);
242 if (!AssignFirstSupportedBackend(convertLayer, backend))
244 return ReturnError(convertLayer);
251 std::stringstream warningMsg;
253 <<
" is not supported on requested backend " << layer->
GetBackendId().
Get()
256 <<
" (reason: " << reasonIfUnsupported
257 <<
"), falling back to the next backend.";
273 Optional<std::vector<std::string>&> errMessages)
278 auto ReturnError = [&](
const Layer* layer)
285 if (availablePreferredBackends.empty())
287 std::stringstream failureMsg;
288 failureMsg <<
"No preferred backends are available";
295 for (
auto it = firstLayer; it != lastLayer; ++it)
300 layer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo().GetDataType();
302 layer->GetOutputSlot(0).GetTensorInfo().GetDataType();
304 std::string reasonIfUnsupported;
314 if (layer->GetBackendHint().has_value() &&
319 layer->GetBackendHint().value(),
322 availablePreferredBackends,
332 for (
const auto& backend : availablePreferredBackends)
334 if (layer->GetBackendHint().has_value() &&
335 layer->GetBackendHint().value() == backend)
346 availablePreferredBackends,
364 BOOST_ASSERT_MSG(res.
IsWarningOnly(),
"OptimizationResult in unexpected state.");
382 layer->SetBackendId(cpuBackendId);
387 return ReturnError(layer);
398 Optional<std::vector<std::string>&> errMessages)
416 auto backendFactory = backendRegistry.GetFactory(selectedBackend);
417 auto backendObjPtr = backendFactory();
418 BOOST_ASSERT(backendObjPtr);
420 backendObjPtr->RegisterTensorHandleFactories(handleFactoryRegistry);
422 backends[backendObjPtr->GetId()] = std::move(backendObjPtr);
431 Optional<std::vector<std::string>&> errMessages)
433 BOOST_ASSERT(optNetObjPtr);
443 auto backendObjPtr = backends.find(selectedBackend)->second.get();
444 BOOST_ASSERT(backendObjPtr);
450 [&backendObjPtr](
const Layer& layer)
456 if (subgraphs.empty())
463 for (
auto& subgraph : subgraphs)
466 OptimizationViews optimizationViews = backendObjPtr->OptimizeSubgraphView(*subgraph);
467 BOOST_ASSERT(optimizationViews.
Validate(*subgraph));
473 SubgraphView& replacementSubgraph = substitution.m_ReplacementSubgraph;
474 SubgraphView& substitutableSubgraph = substitution.m_SubstitutableSubgraph;
478 std::for_each(replacementSubgraph.
begin(), replacementSubgraph.
end(), [&selectedBackend](
Layer* l)
481 l->SetBackendId(selectedBackend);
487 std::stringstream warningMsg;
488 warningMsg <<
"Some sub-graph(s) failed to optimized on " << backendObjPtr->GetId() <<
" backend.";
493 if (!backendObjPtr->GetId().IsCpuRef())
496 settingsCopy.m_IgnoredBackends.insert(backendObjPtr->GetId());
503 std::stringstream subgraphMsg;
504 subgraphMsg <<
"Re-assigning backends to " << failedSubgraph.GetLayers().size()
505 <<
" layers inside sub-graph " << count++;
512 if (reassignmentResult.m_Error)
535 if (srcFactory && dstFactory &&
560 if (frmBackend == backends.end() ||
561 !frmBackend->second->SupportsTensorAllocatorAPI())
568 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
574 const Layer& connectedLayer = connection->GetOwningLayer();
576 auto toBackend = backends.find(connectedLayer.
GetBackendId());
577 BOOST_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
579 if (!toBackend->second.get()->SupportsTensorAllocatorAPI())
585 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
586 for (
auto&& dst : dstPrefs)
599 auto it = factoryScores.find(dst);
600 if (it == factoryScores.end())
603 factoryScores[dst] = 0;
612 factoryScores[dst]++;
615 if (factoryScores[dst] > topScore)
617 topScore = factoryScores[dst];
645 if (frmBackend == backends.end() ||
646 !frmBackend->second->SupportsTensorAllocatorAPI())
652 bool requiresMapUnmap =
false;
655 const Layer& connectedLayer = connection->GetOwningLayer();
658 requiresMapUnmap =
true;
666 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
667 for (
auto&& pref : srcPrefs)
669 if (requiresMapUnmap)
679 auto it = factoryScores.find(pref);
680 if (it == factoryScores.end())
683 factoryScores[pref] = 0;
690 const Layer& connectedLayer = connection->GetOwningLayer();
692 auto toBackend = backends.find(connectedLayer.
GetBackendId());
693 BOOST_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
695 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
696 for (
auto&& src : srcPrefs)
698 if (factoryScores.find(src) == factoryScores.end())
703 for (
auto&& dst : dstPrefs)
708 factoryScores[src]++;
716 int minScore = std::numeric_limits<int>::max();
717 for (
auto it : factoryScores)
719 minScore = std::min(minScore, it.second);
723 std::vector<ITensorHandleFactory::FactoryId> optimalFactories;
724 for (
auto it : factoryScores)
726 if (it.second == minScore)
728 optimalFactories.push_back(it.first);
733 for (
auto&& srcPref : srcPrefs)
735 for (
auto&& comp : optimalFactories)
750 const Layer& connectedLayer,
753 auto toBackend = backends.find(connectedLayer.
GetBackendId());
754 BOOST_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
756 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
779 for (
auto&& pref : dstPrefs)
781 if (pref == srcFactoryId)
791 for (
auto&& pref : dstPrefs)
810 for (
auto&& pref : dstPrefs)
827 Optional<std::vector<std::string>&> errMessages)
831 optGraph.
ForEachLayer([&backends, ®istry, &result, &errMessages](
Layer* layer)
837 BOOST_ASSERT(backends.find(layer->
GetBackendId()) != backends.end());
862 unsigned int connectionIdx = 0;
865 const Layer& connectedLayer = connection->GetOwningLayer();
874 errMessages.value().emplace_back(
"Could not find valid strategy required for compatibility" 875 " between backends.");
891 const std::vector<BackendId>& backendPreferences,
894 Optional<std::vector<std::string>&> messages)
896 if (backendPreferences.empty())
901 const Network& network = *boost::polymorphic_downcast<const Network*>(&inNetwork);
902 std::unique_ptr<Graph> graph = std::make_unique<Graph>(network.
GetGraph());
906 OptimizedNetwork* optNetObjPtr = boost::polymorphic_downcast<OptimizedNetwork*>(optNet.get());
912 using namespace optimizations;
928 optGraph.InferTensorInfos();
941 std::stringstream failureMsg;
942 failureMsg <<
"None of the preferred backends " << backendPreferences
960 if (assignBackendsResult.
m_Error)
974 if (backendOptimizationResult.
m_Error)
990 tensorHandleFactoryRegistry,
999 optGraph.AddCompatibilityLayers(backends, tensorHandleFactoryRegistry);
1009 auto backendPtr = factoryFun();
1010 BOOST_ASSERT(backendPtr.get() !=
nullptr);
1013 auto backendSpecificOptimizations = backendPtr->GetOptimizations();
1016 if (!backendSpecificOptimizations.empty())
1026 : m_Graph(
std::make_unique<
Graph>()),
1043 return m_Graph->AddLayer<
InputLayer>(id, name);
1055 return m_Graph->AddLayer<
ComparisonLayer>(comparisonDescriptor, name);
1074 const auto layer = m_Graph->AddLayer<
FullyConnectedLayer>(fullyConnectedDescriptor, name);
1076 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1078 if (fullyConnectedDescriptor.m_BiasEnabled)
1080 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1091 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, biases, name);
1099 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, biases, name);
1108 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, optionalBiases, name);
1114 return m_Graph->AddLayer<
ConcatLayer>(concatDescriptor, name);
1127 const auto layer = m_Graph->AddLayer<
Convolution2dLayer>(convolution2dDescriptor, name);
1129 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1131 if (convolution2dDescriptor.m_BiasEnabled)
1133 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1144 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1152 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1161 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
1177 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1179 if (convolution2dDescriptor.m_BiasEnabled)
1181 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1199 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1208 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1218 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
1234 return m_Graph->AddLayer<
PermuteLayer>(permuteDescriptor, name);
1240 return m_Graph->AddLayer<
Pooling2dLayer>(pooling2dDescriptor, name);
1246 return m_Graph->AddLayer<
ActivationLayer>(activationDescriptor, name);
1252 return m_Graph->AddLayer<
ArgMinMaxLayer>(argMinMaxDescriptor, name);
1256 normalizationDescriptor,
1264 return m_Graph->AddLayer<
SliceLayer>(sliceDescriptor, name);
1270 return m_Graph->AddLayer<
SoftmaxLayer>(softmaxDescriptor, name);
1276 return m_Graph->AddLayer<
SplitterLayer>(splitterDescriptor, name);
1324 layer->
m_Mean = std::make_unique<ScopedCpuTensorHandle>(mean);
1325 layer->m_Variance = std::make_unique<ScopedCpuTensorHandle>(variance);
1326 layer->m_Beta = std::make_unique<ScopedCpuTensorHandle>(beta);
1327 layer->m_Gamma = std::make_unique<ScopedCpuTensorHandle>(gamma);
1341 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
1345 resizeDescriptor,
const char* name)
1347 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
1372 layer->
m_LayerOutput = std::make_unique<ScopedCpuTensorHandle>(input);
1380 return m_Graph->AddLayer<
ReshapeLayer>(reshapeDescriptor, name);
1404 const auto layer = m_Graph->AddLayer<
LstmLayer>(descriptor, name);
1409 layer->m_BasicParameters.m_InputToCellWeights =
1411 layer->m_BasicParameters.m_InputToOutputWeights =
1413 layer->m_BasicParameters.m_RecurrentToForgetWeights =
1415 layer->m_BasicParameters.m_RecurrentToCellWeights =
1417 layer->m_BasicParameters.m_RecurrentToOutputWeights =
1419 layer->m_BasicParameters.m_ForgetGateBias =
1421 layer->m_BasicParameters.m_CellBias =
1422 std::make_unique<ScopedCpuTensorHandle>(*(params.
m_CellBias));
1423 layer->m_BasicParameters.m_OutputGateBias =
1427 if(!descriptor.m_CifgEnabled)
1436 "AddLstmLayer: Recurrent To Input Weights cannot be NULL");
1442 layer->m_CifgParameters.m_InputToInputWeights =
1444 layer->m_CifgParameters.m_RecurrentToInputWeights =
1449 layer->m_CifgParameters.m_CellToInputWeights =
1452 layer->m_CifgParameters.m_InputGateBias =
1457 if(descriptor.m_ProjectionEnabled)
1463 layer->m_ProjectionParameters.m_ProjectionWeights =
1467 layer->m_ProjectionParameters.m_ProjectionBias =
1473 if(descriptor.m_PeepholeEnabled)
1483 layer->m_PeepholeParameters.m_CellToForgetWeights =
1485 layer->m_PeepholeParameters.m_CellToOutputWeights =
1490 if(descriptor.m_LayerNormEnabled)
1492 if(!descriptor.m_CifgEnabled)
1498 layer->m_LayerNormParameters.m_InputLayerNormWeights =
1514 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
1516 layer->m_LayerNormParameters.m_CellLayerNormWeights =
1518 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
1536 return m_Graph->AddLayer<
MeanLayer>(meanDescriptor,name);
1541 return m_Graph->AddLayer<
PadLayer>(padDescriptor,name);
1607 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1609 if (descriptor.m_BiasEnabled)
1611 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1620 return m_Graph->AddLayer<
TransposeLayer>(transposeDescriptor, name);
1626 return m_Graph->AddLayer<
StackLayer>(stackDescriptor, name);
1644 layer->m_QuantizedLstmParameters.m_InputToForgetWeights =
1646 layer->m_QuantizedLstmParameters.m_InputToCellWeights =
1648 layer->m_QuantizedLstmParameters.m_InputToOutputWeights =
1652 layer->m_QuantizedLstmParameters.m_RecurrentToInputWeights =
1654 layer->m_QuantizedLstmParameters.m_RecurrentToForgetWeights =
1656 layer->m_QuantizedLstmParameters.m_RecurrentToCellWeights =
1658 layer->m_QuantizedLstmParameters.m_RecurrentToOutputWeights =
1662 layer->m_QuantizedLstmParameters.m_InputGateBias =
1664 layer->m_QuantizedLstmParameters.m_ForgetGateBias =
1666 layer->m_QuantizedLstmParameters.m_CellBias =
1667 std::make_unique<ScopedCpuTensorHandle>(params.
GetCellBias());
1668 layer->m_QuantizedLstmParameters.m_OutputGateBias =
1678 layer->Accept(visitor);
1683 : m_Graph(
std::move(graph)),
A layer that the constant data can be bound to.
OptimizeForConnection< Layer, PermuteLayer, SquashEqualSiblingsImpl< PermuteLayer > > SquashEqualPermuteSiblings
void ReportError(const std::string &errorMessage, Optional< std::vector< std::string > &> errorMessages)
bool m_BiasEnabled
Enable/disable bias.
IConnectableLayer * AddPooling2dLayer(const Pooling2dDescriptor &pooling2dDescriptor, const char *name=nullptr) override
Adds a pooling layer to the network.
This layer represents a minimum operation.
static const FactoryId DeferredFactoryId
Use the workload factory to create the tensor handle.
This layer represents a split operation.
LstmBasicParameters m_BasicParameters
const std::vector< InputSlot * > & GetConnections() const
FactoryFunction GetFactory(const BackendId &id) const
This layer represents a batch normalization operation.
static void Destroy(INetwork *network)
A ViewsDescriptor for the SplitterLayer.
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
OptimizeForConnection< PermuteLayer, PermuteLayer, OptimizeInversePermutesImpl< PermuteLayer > > OptimizeInversePermutes
virtual bool SupportsMapUnmap() const final
std::vector< ConvertFp32ToFp16Layer * > InsertConvertFp32ToFp16LayersAfter(Graph &graph, Layer &layer)
bool m_BiasEnabled
Enable/disable bias.
IConnectableLayer * AddOutputLayer(LayerBindingId id, const char *name=nullptr) override
Adds an output layer to the network.
void SetEdgeStrategy(unsigned int connectionIndex, EdgeStrategy strategy)
QuantizedLstmParameters m_QuantizedLstmParameters
This layer represents a 2D transpose convolution operation.
No strategy has been defined. Used internally to verify integrity of optimizations.
std::vector< ConvertFp16ToFp32Layer * > InsertConvertFp16ToFp32LayersBefore(Graph &graph, Layer &layer, bool expectCorrectInputType)
IConnectableLayer * AddL2NormalizationLayer(const L2NormalizationDescriptor &desc, const char *name=nullptr) override
Adds an L2 normalization layer to the network.
A TransposeConvolution2dDescriptor for the TransposeConvolution2dLayer.
CPU Execution: Reference C++ kernels.
Optimizer::Optimizations MakeOptimizations(Args &&... args)
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
IConnectableLayer * AddResizeBilinearLayer(const ResizeBilinearDescriptor &resizeDesc, const char *name=nullptr) override
Adds a resize bilinear layer to the network.
IConnectableLayer * AddMeanLayer(const MeanDescriptor &meanDescriptor, const char *name=nullptr) override
Add a Mean layer to the network.
A ReshapeDescriptor for the ReshapeLayer.
ITensorHandleFactory::FactoryId CalculateSlotOption(BackendsMap &backends, OutputSlot &outputSlot, TensorHandleFactoryRegistry ®istry)
OptimizeForConnection< TransposeLayer, TransposeLayer, OptimizeInversePermutesImpl< TransposeLayer > > OptimizeInverseTransposes
OptimizeForConnection< TransposeLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< TransposeLayer > > TransposeAndBatchToSpaceAsDepthToSpace
#define ARMNN_NO_DEPRECATE_WARN_BEGIN
ITensorHandleFactory::FactoryId CalculateSlotOptionForOutput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry)
A ComparisonDescriptor for the ComparisonLayer.
This layer represents a depthwise convolution 2d operation.
bool RequiresCopy(ITensorHandleFactory::FactoryId src, ITensorHandleFactory::FactoryId dst, TensorHandleFactoryRegistry ®istry)
uint32_t m_TargetWidth
Target width value.
DataLayout m_DataLayout
The data layout to be used (NCHW, NHWC).
IConnectableLayer * AddLogSoftmaxLayer(const LogSoftmaxDescriptor &logSoftmaxDescriptor, const char *name=nullptr) override
Adds a log softmax layer to the network.
A Convolution2dDescriptor for the Convolution2dLayer.
Layer & GetOwningLayer() const
Source backends tensor data can be exported to destination backend tensor without copy...
IConnectableLayer * AddDepthwiseConvolution2dLayer(const DepthwiseConvolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D depthwise convolution layer to the network.
This layer converts data type Float 16 to Float 32.
bool m_BiasEnabled
Enable/disable bias.
const Graph & GetGraph() const
IConnectableLayer * AddSwitchLayer(const char *name=nullptr) override
Adds a switch layer to the network.
ResizeMethod m_Method
The Interpolation method to use (Bilinear, NearestNeighbor).
IConnectableLayer * AddFloorLayer(const char *name=nullptr) override
Adds a floor layer to the network.
static void Pass(Graph &graph, const Optimizations &optimizations)
This layer represents a SpaceToDepth operation.
IConnectableLayer * AddInputLayer(LayerBindingId id, const char *name=nullptr) override
Adds an input layer to the network.
This layer represents a reshape operation.
std::unique_ptr< ScopedCpuTensorHandle > m_InputToInputWeights
A unique pointer to represent 2D weights tensor with dimensions [outputSize, inputSize] (QAsymm8)...
#define ARMNN_LOG(severity)
Main network class which provides the interface for building up a neural network. ...
This layer represents an activation operation with the specified activation function.
BackendRegistry & BackendRegistryInstance()
void Accept(ILayerVisitor &visitor) const override
IConnectableLayer * AddMinimumLayer(const char *name=nullptr) override
Add a Minimum layer to the network.
This layer represents an unknown operation in the input graph.
OptimizeForConnection< Layer, ReshapeLayer, SquashEqualSiblingsImpl< ReshapeLayer > > SquashEqualReshapeSiblings
This layer represents a detection postprocess operator.
BackendIdSet m_SupportedBackends
OptimizeForConnection< Layer, TransposeLayer, MoveTransposeUpImpl > MoveTransposeUp
OptimizationResult ReturnWithError(OptimizationResult res, const Layer *layer, const BackendSettings &backendSettings, Optional< std::vector< std::string > &> errMessages)
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store weight values.
Copyright (c) 2020 ARM Limited.
This layer represents a pad operation.
This layer represents a LSTM operation.
void IgnoreUnused(Ts &&...)
OptimizeForConnection< PadLayer, Convolution2dLayer, FoldPadIntoConvolution2dImpl > FoldPadIntoConvolution2d
void SetBackendId(const BackendId &id)
bool IsBackendSupported(const BackendId &backend) const
LayerList::const_iterator Iterator
IConnectableLayer * AddFullyConnectedLayer(const FullyConnectedDescriptor &fullyConnectedDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a fully connected layer to the network.
A SpaceToDepthDescriptor for the SpaceToDepthLayer.
This layer represents a permutation operation.
OptimizationResult SelectTensorHandleStrategy(Graph &optGraph, BackendsMap &backends, TensorHandleFactoryRegistry ®istry, Optional< std::vector< std::string > &> errMessages)
unsigned int GetNumOutputSlots() const override
Returns the number of connectable output slots.
This layer represents a SpaceToBatchNd operation.
IConnectableLayer * AddRsqrtLayer(const char *name=nullptr) override
Add Reciprocal of square root layer to the network.
A BatchToSpaceNdDescriptor for the BatchToSpaceNdLayer.
OptimizeForType< Layer, AddDebugImpl > InsertDebugLayer
OptimizeForConnection< ReshapeLayer, ReshapeLayer, OptimizeConsecutiveReshapesImpl > OptimizeConsecutiveReshapes
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
This layer represents a elementwiseUnary operation.
constexpr const char * GetDataTypeName(DataType dataType)
A ResizeDescriptor for the ResizeLayer.
IConnectableLayer * AddEqualLayer(const char *name=nullptr) override
Add a Equal layer to the network.
IConnectableLayer * AddConvolution2dLayer(const Convolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D convolution layer to the network.
A StackDescriptor for the StackLayer.
IConnectableLayer * AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor &descriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D transpose convolution layer to the network.
Destination backend can work directly with tensors on source backend.
virtual std::vector< ITensorHandleFactory::FactoryId > GetHandleFactoryPreferences() const
(Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
OptimizeForConnection< ConvertFp16ToFp32Layer, ConvertFp32ToFp16Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp16
The SubgraphView class represents a subgraph of a Graph.
OptimizationResult ApplyBackendOptimizations(OptimizedNetwork *optNetObjPtr, BackendSettings &backendSettings, BackendsMap &backends, Optional< std::vector< std::string > &> errMessages)
A PadDescriptor for the PadLayer.
This layer represents an instance normalization operation.
EdgeStrategy CalculateEdgeStrategy(BackendsMap &backends, ITensorHandleFactory::FactoryId srcFactoryId, const Layer &layer, const Layer &connectedLayer, TensorHandleFactoryRegistry ®istry)
OptimizeForConnection< PermuteLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< PermuteLayer > > PermuteAndBatchToSpaceAsDepthToSpace
OptimizeForConnection< Layer, PermuteLayer, MovePermuteUpImpl > MovePermuteUp
std::unique_ptr< ScopedCpuTensorHandle > m_LayerOutput
IConnectableLayer * AddReshapeLayer(const ReshapeDescriptor &reshapeDescriptor, const char *name=nullptr) override
Adds a reshape layer to the network.
IConnectableLayer * AddTransposeLayer(const TransposeDescriptor &transposeDescriptor, const char *name=nullptr) override
Adds a transpose layer to the network.
A layer user-provided data can be bound to (e.g. inputs, outputs).
void ForEachLayer(Func func) const
Status PrintGraph() override
IConnectableLayer * AddMergeLayer(const char *name=nullptr) override
Adds a merge layer to the network.
This layer dequantizes the input tensor.
ConvertConstants< Float32ToFloat16, IsFloat16Layer > ConvertConstantsFloatToHalf
OptimizeForType< TransposeLayer, TransposeAsReshapeImpl > TransposeAsReshape
IConnectableLayer * AddMaximumLayer(const char *name=nullptr) override
Add a Maximum layer to the network.
This layer represents a Gather operator.
std::unique_ptr< ScopedCpuTensorHandle > m_Anchors
A unique pointer to store Anchor values.
This layer represents a fully connected operation.
An LstmDescriptor for the LstmLayer.
#define ARMNN_NO_DEPRECATE_WARN_END
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
This layer represents a QuantizedLstm operation.
char const * GetLayerTypeAsCString(LayerType type)
This layer represents a log softmax operation.
std::unique_ptr< ScopedCpuTensorHandle > m_Mean
A unique pointer to store Mean values.
A L2NormalizationDescriptor for the L2NormalizationLayer.
int32_t GetQuantizationOffset() const
An ArgMinMaxDescriptor for ArgMinMaxLayer.
float GetQuantizationScale() const
DataType GetDataType() const
An OriginsDescriptor for the ConcatLayer.
bool has_value() const noexcept
A FullyConnectedDescriptor for the FullyConnectedLayer.
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
bool m_BiasEnabled
Enable/disable bias.
This layer represents a stack operation.
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
IConnectableLayer * AddConcatLayer(const ConcatDescriptor &concatDescriptor, const char *name=nullptr) override
Adds a concatenation layer to the network.
const Subgraphs & GetFailedSubgraphs() const
This layer represents a merge operation.
This layer represents a softmax operation.
const std::string & GetNameStr() const
uint32_t m_TargetWidth
Target width value.
This layer represents a BatchToSpaceNd operation.
std::vector< SubgraphViewPtr > Subgraphs
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
IConnectableLayer * AddPadLayer(const PadDescriptor &padDescriptor, const char *name=nullptr) override
Adds a fully pad layer to the network.
IConnectableLayer * AddStandInLayer(const StandInDescriptor &descriptor, const char *name=nullptr) override
Add a stand-in layer for a type unknown to the Arm NN framework.
void SetQuantizationScale(float scale)
This layer represents a ArgMinMax operation.
A StandInDescriptor for the StandIn layer.
IConnectableLayer * AddActivationLayer(const ActivationDescriptor &activationDescriptor, const char *name=nullptr) override
Adds an activation layer to the network.
BackendIdVector GetAvailablePreferredBackends() const
IConnectableLayer * AddSubtractionLayer(const char *name=nullptr) override
Adds a subtraction layer to the network.
IConnectableLayer * AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor &desc, const char *name=nullptr) override
Adds an instance normalization layer to the network.
Device specific knowledge to be passed to the optimizer.
static bool IsLayerSupported(const BackendId &backendId, const IConnectableLayer &layer, Optional< DataType > dataType, std::string &outReasonIfUnsupported)
IConnectableLayer * AddDepthToSpaceLayer(const DepthToSpaceDescriptor &depthToSpaceDescriptor, const char *name=nullptr) override
Adds a depth to space layer to the network.
bool Validate(const SubgraphView &originalSubgraph) const
An ActivationDescriptor for the ActivationLayer.
const BackendId & GetBackendId() const
uint32_t m_TargetHeight
Target height value.
IConnectableLayer * AddStackLayer(const StackDescriptor &stackDescriptor, const char *name=nullptr) override
Adds a stack layer to the network.
This layer represents a floor operation.
uint32_t m_TargetHeight
Target height value.
A SliceDescriptor for the SliceLayer.
This layer represents a normalization operation.
virtual MemorySourceFlags GetExportFlags() const
This layer represents a pooling 2d operation.
This layer converts data type Float 32 to Float 16.
OptimizedNetwork(std::unique_ptr< Graph > graph)
This layer represents a transpose operation.
IConnectableLayer * AddSplitterLayer(const ViewsDescriptor &splitterDescriptor, const char *name=nullptr) override
Adds a splitter layer to the network.
This layer represents an addition operation.
void SubstituteSubgraph(SubgraphView &subgraph, IConnectableLayer *substituteLayer)
Substitutes the given sub-graph with either a new layer or a new sub-graph.
Status PrintGraph() override
IConnectableLayer * AddSpaceToDepthLayer(const SpaceToDepthDescriptor &spaceToDepthDescriptor, const char *name=nullptr) override
Adds a space to depth layer to the network.
bool CheckScaleSetOnQuantizedType(Layer *layer, Optional< std::vector< std::string > &> errMessages)
Private implementation of INetwork.
void SetTensorHandleFactory(const ITensorHandleFactory::FactoryId &id)
A SpaceToBatchNdDescriptor for the SpaceToBatchNdLayer.
IConnectableLayer * AddPreluLayer(const char *name=nullptr) override
Adds a PReLU layer to the network.
OptimizeForType< PermuteLayer, PermuteAsReshapeImpl > PermuteAsReshape
bool IsWarningOnly() const
OptimizeForConnection< Layer, TransposeLayer, SquashEqualSiblingsImpl< TransposeLayer > > SquashEqualTransposeSiblings
static INetwork * CreateRaw()
IConnectableLayer * AddQuantizeLayer(const char *name=nullptr) override
Add a quantize layer to the network.
const Substitutions & GetSubstitutions() const
IConnectableLayer * AddArgMinMaxLayer(const ArgMinMaxDescriptor &desc, const char *name=nullptr) override
Adds an ArgMinMax layer to the network.
BackendIdVector m_PreferredBackends
OptimizationResult AssignBackends(OptimizedNetwork *optNetObjPtr, BackendSettings &backendSettings, Graph::Iterator &firstLayer, Graph::Iterator &lastLayer, Optional< std::vector< std::string > &> errMessages)
This layer represents a subtraction operation.
This layer calculates both true and false outputs for input.
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
ConvertConstants< Float16ToFloat32, IsFloat32Layer > ConvertConstantsHalfToFloat
A ElementwiseUnaryDescriptor for the ElementwiseUnaryLayer.
IConnectableLayer * AddBatchNormalizationLayer(const BatchNormalizationDescriptor &desc, const ConstTensor &mean, const ConstTensor &variance, const ConstTensor &beta, const ConstTensor &gamma, const char *name=nullptr) override
Adds a batch normalization layer to the network.
static Subgraphs SelectSubgraphs(Graph &graph, const LayerSelectorFunction &selector)
Selects subgraphs from a graph based on the selector function and the algorithm.
IConnectableLayer * AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor &batchToSpaceNdDescriptor, const char *name=nullptr) override
Adds a batch to space ND layer to the network.
This layer represents a L2 normalization operation.
BackendsMap CreateSupportedBackends(TensorHandleFactoryRegistry &handleFactoryRegistry, BackendSettings &backendSettings)
OptimizeForConnection< ConvertFp32ToFp16Layer, ConvertFp16ToFp32Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp32
ITensorHandleFactory::FactoryId CalculateSlotOptionForInput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry)
const std::string & Get() const
Status SerializeToDot(std::ostream &stream) const override
IConnectableLayer * AddAdditionLayer(const char *name=nullptr) override
Adds an addition layer to the network.
BackendIdSet m_SelectedBackends
IConnectableLayer * AddDequantizeLayer(const char *name=nullptr) override
Adds a Dequantize layer to the network.
IConnectableLayer * AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor &spaceToBatchNdDescriptor, const char *name=nullptr) override
Adds a space to batch layer to the network.
OptimizationResult AttemptBackendAssignment(BackendSettings &backendSettings, Graph &graph, Layer *layer, BackendId backend, DataType dataTypeIn, DataType dataTypeOut, const std::vector< BackendId > &availablePreferredBackends, std::string &reasonIfUnsupported, Optional< std::vector< std::string > &> errMessages)
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
void SetTensorInfo(const TensorInfo &tensorInfo) override
IConnectableLayer * AddAbsLayer(const char *name=nullptr) override
Add absolute layer to the network.
A MeanDescriptor for the MeanLayer.
This layer represents a division operation.
IConnectableLayer * AddSliceLayer(const SliceDescriptor &sliceDescriptor, const char *name=nullptr) override
Adds a slice layer to the network.
This layer represents a strided slice operation.
LayerType GetType() const
IConnectableLayer * AddGreaterLayer(const char *name=nullptr) override
Add a Greater layer to the network.
This layer represents a maximum operation.
const OutputSlot & GetOutputSlot(unsigned int index=0) const override
Get the const output slot handle by slot index.
IConnectableLayer * AddMergerLayer(const MergerDescriptor &mergerDescriptor, const char *name=nullptr) override
Adds a concat layer to the network.
OptimizeForType< Layer, ConvertFp32NetworkToFp16Impl > Fp32NetworkToFp16Converter
IConnectableLayer * AddResizeLayer(const ResizeDescriptor &resizeDescriptor, const char *name=nullptr) override
Adds a resize layer to the network.
IConnectableLayer * AddConstantLayer(const ConstTensor &input, const char *name=nullptr) override
Adds a layer with no inputs and a single output, which always corresponds to the passed in constant t...
A TransposeDescriptor for the TransposeLayer.
A StridedSliceDescriptor for the StridedSliceLayer.
IConnectableLayer * AddMultiplicationLayer(const char *name=nullptr) override
Adds a multiplication layer to the network.
IConnectableLayer * AddComparisonLayer(const ComparisonDescriptor &comparisonDescriptor, const char *name=nullptr) override
Add a Comparison layer to the network.
void ReportWarning(const std::string &warningMessage, Optional< std::vector< std::string > &> warningMessages)
This layer represents a convolution 2d operation.
bool CheckFlag(MemorySourceFlags flags, MemorySource source)
void SetQuantizationOffset(int32_t offset)
This layer represents a mean operation.
This layer represents a comparison operation.
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
IConnectableLayer * AddNormalizationLayer(const NormalizationDescriptor &normalizationDescriptor, const char *name=nullptr) override
Adds a normalization layer to the network.
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
A Pooling2dDescriptor for the Pooling2dLayer.
This layer dequantizes the input tensor.
armnn::Runtime::CreationOptions::ExternalProfilingOptions options
A NormalizationDescriptor for the NormalizationLayer.
IConnectableLayer * AddPermuteLayer(const PermuteDescriptor &permuteDescriptor, const char *name=nullptr) override
Adds a permute layer to the network.
DataLayout m_DataLayout
The data layout to be used (NCHW, NHWC).
An InstanceNormalizationDescriptor for InstanceNormalizationLayer.
This layer represents a multiplication operation.
IConnectableLayer * AddGatherLayer(const char *name=nullptr) override
Add Gather layer to the network.
IConnectableLayer * AddSoftmaxLayer(const SoftmaxDescriptor &softmaxDescriptor, const char *name=nullptr) override
Adds a softmax layer to the network.
IConnectableLayer * AddLstmLayer(const LstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr) override
Add a Lstm layer to the network.
A ResizeBilinearDescriptor for the ResizeBilinearLayer.
IConnectableLayer * AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor &elementwiseUnaryDescriptor, const char *name=nullptr) override
Add an ElementwiseUnary layer to the network.
const TensorInfo & GetTensorInfo() const override
IConnectableLayer * AddDivisionLayer(const char *name=nullptr) override
Adds a division layer to the network.
static void Destroy(IOptimizedNetwork *network)
virtual MemorySourceFlags GetImportFlags() const
std::unique_ptr< ScopedCpuTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [input_size, num_units].
IConnectableLayer * AddStridedSliceLayer(const StridedSliceDescriptor &stridedSliceDescriptor, const char *name=nullptr) override
Adds a strided slice layer to the network.
A SoftmaxDescriptor for the SoftmaxLayer.
bool IsCpuRefUsed() const
IConnectableLayer * AddQuantizedLstmLayer(const QuantizedLstmInputParams ¶ms, const char *name=nullptr) override
Add a QuantizedLstm layer to the network.
static INetworkPtr Create()
static const FactoryId LegacyFactoryId
IConnectableLayer * AddDetectionPostProcessLayer(const DetectionPostProcessDescriptor &descriptor, const ConstTensor &anchors, const char *name=nullptr) override
Adds a Detection PostProcess layer to the network.
A DepthwiseConvolution2dDescriptor for the DepthwiseConvolution2dLayer.
This layer represents a DepthToSpace operation.
A BatchNormalizationDescriptor for the BatchNormalizationLayer.
std::map< BackendId, std::unique_ptr< class IBackendInternal > > BackendsMap
This layer represents a resize operation.
A PermuteDescriptor for the PermuteLayer.
std::vector< float > anchors({ 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 10.5f, 1.0f, 1.0f, 0.5f, 10.5f, 1.0f, 1.0f, 0.5f, 100.5f, 1.0f, 1.0f })