#include "HeapProfiling.hpp"
#include <backends/CpuTensorHandle.hpp>
+#include <backends/BackendRegistry.hpp>
#include <boost/polymorphic_cast.hpp>
#include <boost/assert.hpp>
}
LoadedNetwork::LoadedNetwork(std::unique_ptr<OptimizedNetwork> net)
- : m_CpuRef()
- , m_OptimizedNetwork(std::move(net))
+ : m_OptimizedNetwork(std::move(net))
, m_WorkingMemLock(m_WorkingMemMutex, std::defer_lock)
{
// Create a profiler and register it for the current thread.
ProfilerManager::GetInstance().RegisterProfiler(m_Profiler.get());
Graph& order = m_OptimizedNetwork->GetGraph().TopologicalSort();
- //First create tensor handlers.
+ //First create tensor handlers, backends and workload factories.
//Handlers are created before workloads are.
//Because workload creation can modify some of the handlers,
//(for example the splitter and merger layers).
for (auto&& layer : order)
{
+ auto const& backend = layer->GetBackendId();
+ if (m_Backends.count(backend) == 0)
+ {
+ auto createBackend = BackendRegistryInstance().GetFactory(backend);
+ auto it = m_Backends.emplace(std::make_pair(backend, createBackend()));
+ m_WorkloadFactories.emplace(std::make_pair(backend,
+ it.first->second->CreateWorkloadFactory()));
+ }
layer->CreateTensorHandles(m_OptimizedNetwork->GetGraph(), GetWorkloadFactory(*layer));
}
m_OptimizedNetwork->GetGraph().AllocateDynamicBuffers();
// Finalize the workload factories before execution.
- m_CpuRef.Finalize();
- m_CpuAcc.Finalize();
- m_GpuAcc.Finalize();
+ for (auto&& workloadFactory : m_WorkloadFactories)
+ {
+ workloadFactory.second->Finalize();
+ }
}
TensorInfo LoadedNetwork::GetInputTensorInfo(LayerBindingId layerId) const
{
const IWorkloadFactory* workloadFactory = nullptr;
- if (layer.GetBackendId() == Compute::CpuAcc)
- {
- workloadFactory = &m_CpuAcc;
- }
- else if (layer.GetBackendId() == Compute::GpuAcc)
- {
- workloadFactory = &m_GpuAcc;
- }
- else if (layer.GetBackendId() == Compute::CpuRef)
+ auto it = m_WorkloadFactories.find(layer.GetBackendId());
+ if (it == m_WorkloadFactories.end())
{
- workloadFactory = &m_CpuRef;
+ throw RuntimeException(
+ boost::str(
+ boost::format("No workload factory for %1% to be used for layer: %2%")
+ % layer.GetBackendId().Get()
+ % layer.GetNameStr()),
+ CHECK_LOCATION());
}
+ workloadFactory = it->second.get();
+
BOOST_ASSERT_MSG(workloadFactory, "No workload factory");
std::string reasonIfUnsupported;
BOOST_ASSERT_MSG(IWorkloadFactory::IsLayerSupported(layer, {}, reasonIfUnsupported),
- "Factory does not support layer");
+ "Factory does not support layer");
boost::ignore_unused(reasonIfUnsupported);
-
return *workloadFactory;
}
{
return;
}
- m_CpuRef.Acquire();
- m_CpuAcc.Acquire();
- m_GpuAcc.Acquire();
+ for (auto&& workloadFactory : m_WorkloadFactories)
+ {
+ workloadFactory.second->Acquire();
+ }
m_IsWorkingMemAllocated = true;
}
return;
}
// Informs the memory managers to release memory in it's respective memory group
- m_CpuRef.Release();
- m_CpuAcc.Release();
- m_GpuAcc.Release();
+ for (auto&& workloadFactory : m_WorkloadFactories)
+ {
+ workloadFactory.second->Release();
+ }
m_IsWorkingMemAllocated = false;
}
#include "LayerFwd.hpp"
#include "Profiling.hpp"
-#include <backends/reference/RefWorkloadFactory.hpp>
-#include <backends/neon/NeonWorkloadFactory.hpp>
-#include <backends/cl/ClWorkloadFactory.hpp>
+#include <backends/IBackendInternal.hpp>
#include <backends/Workload.hpp>
#include <backends/WorkloadFactory.hpp>
#include <mutex>
+#include <unordered_map>
namespace cl
{
const IWorkloadFactory& GetWorkloadFactory(const Layer& layer) const;
- RefWorkloadFactory m_CpuRef;
- NeonWorkloadFactory m_CpuAcc;
- ClWorkloadFactory m_GpuAcc;
+ using BackendPtrMap = std::unordered_map<BackendId, IBackendInternalUniquePtr>;
+ using WorkloadFactoryMap = std::unordered_map<BackendId, IBackendInternal::IWorkloadFactoryPtr>;
+
+ BackendPtrMap m_Backends;
+ WorkloadFactoryMap m_WorkloadFactories;
std::unique_ptr<OptimizedNetwork> m_OptimizedNetwork;
WorkloadQueue m_InputQueue;
// Try preferred backend first
layer->SetBackendId(preferredBackend);
- if (IWorkloadFactory::IsLayerSupported(*layer, boost::none, reasonIfUnsupported))
+ if (IWorkloadFactory::IsLayerSupported(*layer,
+ EmptyOptional(),
+ reasonIfUnsupported))
{
supportedBackendFound = true;
}
}
layer->SetBackendId(backend);
- if (IWorkloadFactory::IsLayerSupported(*layer, boost::none, reasonIfUnsupported))
+ if (IWorkloadFactory::IsLayerSupported(*layer,
+ EmptyOptional(),
+ reasonIfUnsupported))
{
supportedBackendFound = true;
break;
//
#pragma once
+#include <armnn/Optional.hpp>
#include <boost/assert.hpp>
-#include <boost/optional.hpp>
#include <functional>
#include <map>
template <typename TNodeId>
-boost::optional<TNodeId> GetNextChild(TNodeId node,
+armnn::Optional<TNodeId> GetNextChild(TNodeId node,
std::function<std::vector<TNodeId>(TNodeId)> getIncomingEdges,
std::map<TNodeId, NodeState>& nodeStates)
{
nodeStates[current] = NodeState::Visiting;
- boost::optional<TNodeId> nextChildOfCurrent = GetNextChild(current, getIncomingEdges, nodeStates);
+ auto nextChildOfCurrent = GetNextChild(current, getIncomingEdges, nodeStates);
if (nextChildOfCurrent)
{
- TNodeId nextChild = nextChildOfCurrent.get();
+ TNodeId nextChild = nextChildOfCurrent.value();
// If the child has not been searched, add to the stack and iterate over this node
if (nodeStates.find(nextChild) == nodeStates.end())
#include <armnn/Types.hpp>
#include "RegistryCommon.hpp"
+#include "IBackendInternal.hpp"
namespace armnn
{
-using BackendRegistry = RegistryCommon<IBackend, IBackendUniquePtr>;
+using BackendRegistry = RegistryCommon<IBackendInternal, IBackendInternalUniquePtr>;
BackendRegistry& BackendRegistryInstance();
#pragma once
#include <armnn/Types.hpp>
-#include <backends/WorkloadFactory.hpp>
namespace armnn
{
+class IWorkloadFactory;
class IBackendInternal : public IBackend
{
protected:
+ // Creation must be done through a specific
+ // backend interface.
IBackendInternal() = default;
- ~IBackendInternal() override = default;
public:
- virtual std::unique_ptr<IWorkloadFactory> CreateWorkloadFactory() const = 0;
+ // Allow backends created by the factory function
+ // to be destroyed through IBackendInternal.
+ ~IBackendInternal() override = default;
+
+ using IWorkloadFactoryPtr = std::unique_ptr<IWorkloadFactory>;
+ virtual IWorkloadFactoryPtr CreateWorkloadFactory() const = 0;
};
+using IBackendInternalUniquePtr = std::unique_ptr<IBackendInternal>;
+
} // namespace armnn
#include <backends/WorkloadFactory.hpp>
#include <backends/LayerSupportRegistry.hpp>
-#include <backends/reference/RefWorkloadFactory.hpp>
-#include <backends/neon/NeonWorkloadFactory.hpp>
-#include <backends/cl/ClWorkloadFactory.hpp>
-
#include <armnn/Types.hpp>
#include <armnn/LayerSupport.hpp>
#include <Layer.hpp>
namespace
{
- const TensorInfo OverrideDataType(const TensorInfo& info, boost::optional<DataType> type)
+
+const TensorInfo OverrideDataType(const TensorInfo& info, Optional<DataType> type)
+{
+ if (!type)
{
- if (type == boost::none)
- {
- return info;
- }
+ return info;
+ }
+
+ return TensorInfo(info.GetShape(), type.value(), info.GetQuantizationScale(), info.GetQuantizationOffset());
+}
- return TensorInfo(info.GetShape(), type.get(), info.GetQuantizationScale(), info.GetQuantizationOffset());
+Optional<DataType> GetBiasTypeFromWeightsType(Optional<DataType> weightsType)
+{
+ if (!weightsType)
+ {
+ return weightsType;
}
- boost::optional<DataType> GetBiasTypeFromWeightsType(boost::optional<DataType> weightsType)
+ switch(weightsType.value())
{
- if (weightsType == boost::none)
- {
+ case DataType::Float16:
+ case DataType::Float32:
return weightsType;
- }
-
- switch(weightsType.get())
- {
- case DataType::Float16:
- case DataType::Float32:
- return weightsType;
- case DataType::QuantisedAsymm8:
- return DataType::Signed32;
- default:
- BOOST_ASSERT_MSG(false, "GetBiasTypeFromWeightsType(): Unsupported data type.");
- }
- return boost::none;
+ case DataType::QuantisedAsymm8:
+ return DataType::Signed32;
+ default:
+ BOOST_ASSERT_MSG(false, "GetBiasTypeFromWeightsType(): Unsupported data type.");
}
+ return EmptyOptional();
}
+} // anonymous namespace
+
bool IWorkloadFactory::IsLayerSupported(const BackendId& backendId,
const IConnectableLayer& connectableLayer,
- boost::optional<DataType> dataType,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported)
{
Optional<std::string&> reason = outReasonIfUnsupported;
}
bool IWorkloadFactory::IsLayerSupported(const IConnectableLayer& connectableLayer,
- boost::optional<DataType> dataType,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported)
{
auto layer = boost::polymorphic_downcast<const Layer*>(&connectableLayer);
#include <memory>
#include <armnn/TensorFwd.hpp>
+#include <armnn/Optional.hpp>
#include <backends/OutputHandler.hpp>
#include <backends/Workload.hpp>
-#include <boost/optional.hpp>
namespace armnn
{
static bool IsLayerSupported(const BackendId& backendId,
const IConnectableLayer& layer,
- boost::optional<DataType> dataType,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported);
static bool IsLayerSupported(const IConnectableLayer& layer,
- boost::optional<DataType> dataType,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported);
virtual bool SupportsSubTensors() const = 0;
ClBackend::GetIdStatic(),
[]()
{
- return IBackendUniquePtr(new ClBackend, &ClBackend::Destroy);
+ return IBackendInternalUniquePtr(new ClBackend);
}
};
return s_Id;
}
-std::unique_ptr<IWorkloadFactory> ClBackend::CreateWorkloadFactory() const
+IBackendInternal::IWorkloadFactoryPtr ClBackend::CreateWorkloadFactory() const
{
return std::make_unique<ClWorkloadFactory>();
}
-void ClBackend::Destroy(IBackend* backend)
-{
- delete boost::polymorphic_downcast<ClBackend*>(backend);
-}
-
} // namespace armnn
\ No newline at end of file
static const BackendId& GetIdStatic();
const BackendId& GetId() const override { return GetIdStatic(); }
- std::unique_ptr<IWorkloadFactory> CreateWorkloadFactory() const override;
-
- static void Destroy(IBackend* backend);
+ IWorkloadFactoryPtr CreateWorkloadFactory() const override;
};
} // namespace armnn
\ No newline at end of file
{
bool ClWorkloadFactory::IsLayerSupported(const Layer& layer,
- boost::optional<DataType> dataType,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported)
{
return IWorkloadFactory::IsLayerSupported(Compute::GpuAcc, layer, dataType, outReasonIfUnsupported);
#pragma once
#include <armnn/IRuntime.hpp>
+#include <armnn/Optional.hpp>
#include <backends/OutputHandler.hpp>
#include <backends/aclCommon/memory/BaseMemoryManager.hpp>
-#include <boost/optional.hpp>
-
namespace armnn
{
virtual Compute GetCompute() const override { return Compute::GpuAcc; }
- static bool IsLayerSupported(const Layer& layer, boost::optional<DataType> dataType,
+ static bool IsLayerSupported(const Layer& layer,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported);
virtual bool SupportsSubTensors() const override { return true; }
NeonBackend::GetIdStatic(),
[]()
{
- return IBackendUniquePtr(new NeonBackend, &NeonBackend::Destroy);
+ return IBackendInternalUniquePtr(new NeonBackend);
}
};
return s_Id;
}
-std::unique_ptr<IWorkloadFactory> NeonBackend::CreateWorkloadFactory() const
+IBackendInternal::IWorkloadFactoryPtr NeonBackend::CreateWorkloadFactory() const
{
return std::make_unique<NeonWorkloadFactory>();
}
-void NeonBackend::Destroy(IBackend* backend)
-{
- delete boost::polymorphic_downcast<NeonBackend*>(backend);
-}
-
} // namespace armnn
\ No newline at end of file
static const BackendId& GetIdStatic();
const BackendId& GetId() const override { return GetIdStatic(); }
- std::unique_ptr<IWorkloadFactory> CreateWorkloadFactory() const override;
-
- static void Destroy(IBackend* backend);
+ IWorkloadFactoryPtr CreateWorkloadFactory() const override;
};
} // namespace armnn
\ No newline at end of file
namespace armnn
{
-bool NeonWorkloadFactory::IsLayerSupported(const Layer& layer, boost::optional<DataType> dataType,
+bool NeonWorkloadFactory::IsLayerSupported(const Layer& layer,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported)
{
return IWorkloadFactory::IsLayerSupported(Compute::CpuAcc, layer, dataType, outReasonIfUnsupported);
//
#pragma once
+#include <armnn/Optional.hpp>
#include <backends/OutputHandler.hpp>
#include <backends/aclCommon/memory/BaseMemoryManager.hpp>
#include <boost/core/ignore_unused.hpp>
-#include <boost/optional.hpp>
+
namespace armnn
{
virtual Compute GetCompute() const override { return Compute::CpuAcc; }
- static bool IsLayerSupported(const Layer& layer, boost::optional<DataType> dataType,
+ static bool IsLayerSupported(const Layer& layer,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported);
virtual bool SupportsSubTensors() const override { return true; }
RefBackend::GetIdStatic(),
[]()
{
- return IBackendUniquePtr(new RefBackend, &RefBackend::Destroy);
+ return IBackendInternalUniquePtr(new RefBackend);
}
};
return s_Id;
}
-std::unique_ptr<IWorkloadFactory> RefBackend::CreateWorkloadFactory() const
+IBackendInternal::IWorkloadFactoryPtr RefBackend::CreateWorkloadFactory() const
{
return std::make_unique<RefWorkloadFactory>();
}
-void RefBackend::Destroy(IBackend* backend)
-{
- delete boost::polymorphic_downcast<RefBackend*>(backend);
-}
-
} // namespace armnn
\ No newline at end of file
static const BackendId& GetIdStatic();
const BackendId& GetId() const override { return GetIdStatic(); }
- std::unique_ptr<IWorkloadFactory> CreateWorkloadFactory() const override;
-
- static void Destroy(IBackend* backend);
+ IWorkloadFactoryPtr CreateWorkloadFactory() const override;
};
} // namespace armnn
\ No newline at end of file
{
}
-bool RefWorkloadFactory::IsLayerSupported(const Layer& layer, boost::optional<DataType> dataType,
+bool RefWorkloadFactory::IsLayerSupported(const Layer& layer,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported)
{
return IWorkloadFactory::IsLayerSupported(Compute::CpuRef, layer, dataType, outReasonIfUnsupported);
//
#pragma once
+#include <armnn/Optional.hpp>
#include <backends/WorkloadFactory.hpp>
#include <backends/OutputHandler.hpp>
#include <boost/core/ignore_unused.hpp>
-#include <boost/optional.hpp>
+
namespace armnn
{
virtual Compute GetCompute() const override { return Compute::CpuRef; }
- static bool IsLayerSupported(const Layer& layer, boost::optional<DataType> dataType,
+ static bool IsLayerSupported(const Layer& layer,
+ Optional<DataType> dataType,
std::string& outReasonIfUnsupported);
virtual bool SupportsSubTensors() const override { return false; }
[&called]()
{
called = true;
- return armnn::IBackendUniquePtr(nullptr, nullptr);
+ return armnn::IBackendInternalUniquePtr(nullptr);
}
);
[&called]()
{
called = true;
- return armnn::IBackendUniquePtr(nullptr, nullptr);
+ return armnn::IBackendInternalUniquePtr(nullptr);
}
);
InferenceTestImage.cpp)
add_library_ex(inferenceTest STATIC ${inference_test_sources})
target_include_directories(inferenceTest PRIVATE ../src/armnnUtils)
-target_include_directories(inferenceTest PRIVATE ../src/backends)
+target_include_directories(inferenceTest PRIVATE ../src)
if(BUILD_CAFFE_PARSER)
macro(CaffeParserTest testName sources)
add_executable_ex(${testName} ${sources})
target_include_directories(${testName} PRIVATE ../src/armnnUtils)
- target_include_directories(${testName} PRIVATE ../src/backends)
+ target_include_directories(${testName} PRIVATE ../src)
set_target_properties(${testName} PROPERTIES COMPILE_FLAGS "${CAFFE_PARSER_TEST_ADDITIONAL_COMPILE_FLAGS}")
target_link_libraries(${testName} inferenceTest)
macro(TfParserTest testName sources)
add_executable_ex(${testName} ${sources})
target_include_directories(${testName} PRIVATE ../src/armnnUtils)
- target_include_directories(${testName} PRIVATE ../src/backends)
+ target_include_directories(${testName} PRIVATE ../src)
target_link_libraries(${testName} inferenceTest)
target_link_libraries(${testName} armnnTfParser)
macro(TfLiteParserTest testName sources)
add_executable_ex(${testName} ${sources})
target_include_directories(${testName} PRIVATE ../src/armnnUtils)
- target_include_directories(${testName} PRIVATE ../src/backends)
+ target_include_directories(${testName} PRIVATE ../src)
target_link_libraries(${testName} inferenceTest)
target_link_libraries(${testName} armnnTfLiteParser)
macro(OnnxParserTest testName sources)
add_executable_ex(${testName} ${sources})
target_include_directories(${testName} PRIVATE ../src/armnnUtils)
- target_include_directories(${testName} PRIVATE ../src/backends)
+ target_include_directories(${testName} PRIVATE ../src)
target_link_libraries(${testName} inferenceTest)
target_link_libraries(${testName} armnnOnnxParser)
add_executable_ex(ExecuteNetwork ${ExecuteNetwork_sources})
target_include_directories(ExecuteNetwork PRIVATE ../src/armnn)
target_include_directories(ExecuteNetwork PRIVATE ../src/armnnUtils)
- target_include_directories(ExecuteNetwork PRIVATE ../src/backends)
+ target_include_directories(ExecuteNetwork PRIVATE ../src)
if (BUILD_CAFFE_PARSER)
target_link_libraries(ExecuteNetwork armnnCaffeParser)
#include <armnnOnnxParser/IOnnxParser.hpp>
#endif
-#include <BackendRegistry.hpp>
+#include <backends/BackendRegistry.hpp>
#include <boost/exception/exception.hpp>
#include <boost/exception/diagnostic_information.hpp>