set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CLCACHE_PROGRAM}")
endif()
+if(WIN32)
+ find_program(SCCACHE sccache)
+ if(SCCACHE)
+ set(CMAKE_C_COMPILER_LAUNCHER ${SCCACHE})
+ set(CMAKE_CXX_COMPILER_LAUNCHER ${SCCACHE})
+ set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT Embedded)
+ cmake_policy(SET CMP0141 NEW)
+ endif()
+endif()
+
project(HailoRT)
# Prevent in-tree building
CONTEXT_SWITCH_DEFS__ACTION_TYPE_CHANGE_BOUNDARY_INPUT_BATCH,
CONTEXT_SWITCH_DEFS__ACTION_TYPE_PAUSE_VDMA_CHANNEL,
CONTEXT_SWITCH_DEFS__ACTION_TYPE_RESUME_VDMA_CHANNEL,
+ CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_INPUT,
+ CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_OUTPUT,
+ CONTEXT_SWITCH_DEFS__ACTION_TYPE_WAIT_FOR_CACHE_UPDATED,
/* Must be last */
CONTEXT_SWITCH_DEFS__ACTION_TYPE_COUNT
uint8_t connected_d2h_packed_vdma_channel_id;
} CONTEXT_SWITCH_DEFS__activate_ddr_buffer_input_data_t;
+typedef struct {
+ uint8_t packed_vdma_channel_id;
+ uint8_t stream_index;
+ uint8_t network_index;
+ CONTEXT_SWITCH_DEFS__stream_reg_info_t stream_reg_info;
+ CONTROL_PROTOCOL__host_buffer_info_t host_buffer_info;
+ uint32_t initial_credit_size;
+} CONTEXT_SWITCH_DEFS__activate_cache_input_data_t;
+
typedef struct {
uint8_t packed_vdma_channel_id;
uint8_t stream_index;
uint32_t buffered_rows_count;
} CONTEXT_SWITCH_DEFS__activate_ddr_buffer_output_data_t;
+typedef struct {
+ uint8_t packed_vdma_channel_id;
+ uint8_t stream_index;
+ uint8_t network_index;
+ CONTEXT_SWITCH_DEFS__stream_reg_info_t stream_reg_info;
+ CONTROL_PROTOCOL__host_buffer_info_t host_buffer_info;
+} CONTEXT_SWITCH_DEFS__activate_cache_output_data_t;
+
typedef struct {
uint8_t packed_vdma_channel_id;
CONTROL_PROTOCOL__host_buffer_info_t host_buffer_info;
/* Value to represent an operation should be performed on all streams. */
#define CONTROL_PROTOCOL__ALL_DATAFLOW_MANAGERS (0xFF)
-#define CONTROL_PROTOCOL__MAX_CONTEXT_SIZE (3072)
+#define CONTROL_PROTOCOL__MAX_CONTEXT_SIZE (4096)
#define CONTROL_PROTOCOL__OPCODES_VARIABLES \
CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_IDENTIFY, true, CPU_ID_APP_CPU)\
CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_SET_SLEEP_STATE, false, CPU_ID_APP_CPU)\
CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CHANGE_HW_INFER_STATUS, false, CPU_ID_CORE_CPU)\
CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_SIGNAL_DRIVER_DOWN, false, CPU_ID_CORE_CPU)\
+ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_INIT_CACHE_INFO, false, CPU_ID_CORE_CPU)\
+ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_GET_CACHE_INFO, false, CPU_ID_CORE_CPU)\
+ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_UPDATE_CACHE_READ_OFFSET, false, CPU_ID_CORE_CPU)\
+ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_SIGNAL_CACHE_UPDATED, false, CPU_ID_CORE_CPU)\
typedef enum {
#define CONTROL_PROTOCOL__OPCODE_X(name, is_critical, cpu_id) name,
#pragma warning(pop)
#endif
+typedef struct {
+ uint32_t cache_size;
+ uint32_t current_read_offset;
+ int32_t write_offset_delta;
+} CONTROL_PROTOCOL__context_switch_cache_info_t;
+
+typedef struct {
+ uint32_t cache_info_length;
+ CONTROL_PROTOCOL__context_switch_cache_info_t cache_info;
+} CONTROL_PROTOCOL__context_switch_init_cache_info_request_t;
+
+typedef struct {
+ uint32_t cache_info_length;
+ CONTROL_PROTOCOL__context_switch_cache_info_t cache_info;
+} CONTROL_PROTOCOL__context_switch_get_cache_info_response_t;
+
+typedef struct {
+ uint32_t read_offset_delta_length;
+ int32_t read_offset_delta;
+} CONTROL_PROTOCOL__context_switch_update_cache_read_offset_request_t;
typedef CONTROL_PROTOCOL__read_memory_request_t CONTROL_PROTOCOL__read_user_config_request_t;
typedef CONTROL_PROTOCOL__read_memory_response_t CONTROL_PROTOCOL__read_user_config_response_t;
CONTROL_PROTOCOL__get_overcurrent_state_response_t get_overcurrent_state_response;
CONTROL_PROTOCOL__get_hw_consts_response_t get_hw_consts_response;
CONTROL_PROTOCOL__change_hw_infer_status_response_t change_hw_infer_status_response;
+ CONTROL_PROTOCOL__context_switch_get_cache_info_response_t context_switch_get_cache_info_response;
// Note: This array is larger than any legal request:
// * Functions in this module won't write more than CONTROL_PROTOCOL__MAX_CONTROL_LENGTH bytes
CONTROL_PROTOCOL__sensor_set_generic_i2c_slave_request_t sensor_set_generic_i2c_slave_request;
CONTROL_PROTOCOL__context_switch_set_network_group_header_request_t context_switch_set_network_group_header_request;
CONTROL_PROTOCOL__context_switch_set_context_info_request_t context_switch_set_context_info_request;
+ CONTROL_PROTOCOL__context_switch_init_cache_info_request_t context_switch_init_cache_info_request;
+ CONTROL_PROTOCOL__context_switch_update_cache_read_offset_request_t context_switch_update_cache_read_offset_request;
CONTROL_PROTOCOL__idle_time_set_measurement_request_t idle_time_set_measurement_request;
CONTROL_PROTOCOL__download_context_action_list_request_t download_context_action_list_request;
CONTROL_PROTOCOL__change_context_switch_status_request_t change_context_switch_status_request;
#include "status.h"
#include "stdfloat.h"
+#pragma pack(push, 1)
+
/**
* @brief The d2h event manager structures relevant in the host
*/
HEALTH_MONITOR_CLOCK_CHANGED_EVENT_ID,
HW_INFER_MANAGER_INFER_DONE,
CONTEXT_SWITCH_RUN_TIME_ERROR,
+ START_UPDATE_CACHE_OFFSET_ID,
D2H_EVENT_ID_COUNT /* Must be last*/
} D2H_EVENT_ID_t;
#define D2H_EVENT_CONTEXT_SWITCH_RUN_TIME_ERROR_EVENT_PARAMETER_COUNT (5)
+typedef struct {
+ uint64_t cache_id_bitmask;
+} D2H_EVENT_start_update_cache_offset_message_t;
+
+#define D2H_EVENT_START_UPDATE_CACHE_OFFSET_PARAMETER_COUNT (1)
+
/* D2H_EVENT__message_parameters_t should be in the same order as hailo_notification_message_parameters_t */
typedef union {
D2H_EVENT_rx_error_event_message_t rx_error_event;
D2H_EVENT_health_monitor_clock_changed_event_message_t health_monitor_clock_changed_event;
D2H_EVENT_hw_infer_mamager_infer_done_message_t hw_infer_manager_infer_done_event;
D2H_EVENT_context_switch_run_time_error_event_message_t context_switch_run_time_error_event;
+ D2H_EVENT_start_update_cache_offset_message_t start_update_cache_offset_event;
} D2H_EVENT__message_parameters_t;
typedef struct {
uint8_t buffer[D2H_EVENT_MAX_SIZE];
} D2H_event_buffer_t;
+#pragma pack(pop)
+
/**********************************************************************
* Public Functions
**********************************************************************/
#define FIRMWARE_HEADER_MAGIC_HAILO8 (0x1DD89DE0)
#define FIRMWARE_HEADER_MAGIC_HAILO15 (0xE905DAAB)
-// TODO - HRT-11344 : change fw magic to pluto specific
-#define FIRMWARE_HEADER_MAGIC_PLUTO (0xE905DAAB)
+#define FIRMWARE_HEADER_MAGIC_PLUTO (0xF94739AB)
typedef enum {
FIRMWARE_HEADER_VERSION_INITIAL = 0,
FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_HW_INFER_STATE_LENGTH)\
FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_CHANNELS_INFO_LENGTH)\
FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_BATCH_COUNT_LENGTH)\
+ FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_CACHE_INFO_LENGTH)\
+ FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_READ_OFFSET_DELTA_LENGTH)\
\
FIRMWARE_MODULE__X(FIRMWARE_MODULE__POWER_MEASUREMENT)\
FIRMWARE_STATUS__X(HAILO_POWER_MEASUREMENT_STATUS_POWER_INIT_ERROR)\
FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_WRITE_DATA_BY_TYPE_ACTION_INVALID_MEMORY_SPACE)\
FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_REACHED_TIMEOUT_WHILE_WAITING_FOR_BATCH_SWITCH_CONTEXT_TO_END)\
FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_INVALID_EXTERNAL_ACTION_LIST_ADDRESS)\
+ FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_INVALID_CACHE_SIZE)\
+ FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_INVALID_READ_OFFSET_SIZE)\
\
FIRMWARE_MODULE__X(FIRMWARE_MODULE__D2H_EVENT_MANAGER)\
FIRMWARE_STATUS__X(HAILO_D2H_EVENT_MANAGER_STATUS_MESSAGE_HIGH_PRIORITY_QUEUE_CREATE_FAILED)\
build/
dist/
/external/
-cmake/external/*/
option(HAILO_BUILD_PYBIND "Build Python binding" OFF)
option(HAILO_BUILD_EMULATOR "Build hailort for emulator" OFF)
option(HAILO_BUILD_UT "Build Unit Tests" OFF)
-option(HAILO_BUILD_DMABUF_TESTS "Build DMA buffer tests. Relevant only if HAILO_BUILD_UT is ON" OFF)
option(HAILO_BUILD_HW_DEBUG_TOOL "Build hw debug tool" OFF)
option(HAILO_BUILD_GSTREAMER "Compile gstreamer plugins" OFF)
option(HAILO_BUILD_EXAMPLES "Build examples" OFF)
option(HAILO_BUILD_PROFILER "Build hailort profiler" ON)
option(HAILO_COMPILE_WARNING_AS_ERROR "Add compilation flag for treating compilation warnings as errors" OFF)
option(HAILO_SUPPORT_PACKAGING "Create HailoRT package (internal)" OFF)
+option(HAILO_BUILD_DOC "Build doc" OFF)
if (HAILO_COMPILE_WARNING_AS_ERROR)
if(WIN32)
# Set firmware version
add_definitions( -DFIRMWARE_VERSION_MAJOR=4 )
-add_definitions( -DFIRMWARE_VERSION_MINOR=17 )
-add_definitions( -DFIRMWARE_VERSION_REVISION=1 )
+add_definitions( -DFIRMWARE_VERSION_MINOR=18 )
+add_definitions( -DFIRMWARE_VERSION_REVISION=0 )
if(HAILO_BUILD_SERVICE)
add_definitions( -DHAILO_SUPPORT_MULTI_PROCESS )
endif()
set(COMMON_INC_DIR ${PROJECT_SOURCE_DIR}/common/include)
set(DRIVER_INC_DIR ${PROJECT_SOURCE_DIR}/hailort/drivers/common)
set(RPC_DIR ${PROJECT_SOURCE_DIR}/hailort/rpc)
+set(HRPC_DIR ${PROJECT_SOURCE_DIR}/hailort/hrpc)
+set(HRPC_PROTOCOL_DIR ${PROJECT_SOURCE_DIR}/hailort/hrpc_protocol)
+set(HAILORT_SERVICE_DIR ${PROJECT_SOURCE_DIR}/hailort/hailort_service)
+set(HAILORT_SERVER_DIR ${PROJECT_SOURCE_DIR}/hailort/hailort_server)
if(CMAKE_SYSTEM_NAME STREQUAL QNX)
include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/pevents.cmake)
endif()
add_subdirectory(common)
+add_subdirectory(hrpc)
+add_subdirectory(hrpc_protocol)
add_subdirectory(libhailort)
add_subdirectory(hailortcli)
if(HAILO_BUILD_HW_DEBUG_TOOL)
if(CMAKE_SYSTEM_NAME STREQUAL QNX)
add_subdirectory(drivers/qnx)
endif()
+
+add_subdirectory(hailort_server)
| pevents | Mahmoud Al-Qudsi | MIT License | master | Cloned entire package | https://github.com/neosmart/pevents.git |
| grpc | Google Inc. | Apache License 2.0 | 1.46.3 | Cloned entire package | https://github.com/grpc/grpc |
| stb | Sean Barrett | MIT License | 0.97 | Copied only the file `stb/stb_image_resize.h` | https://github.com/nothings/stb |
-| eigen | | Mozilla Public License 2.0 | 3.4.0 | Cloned entire package | https://gitlab.com/libeigen/eigen |
\ No newline at end of file
+| eigen | | Mozilla Public License 2.0 | 3.4.0 | Cloned entire package | https://gitlab.com/libeigen/eigen |
namespace hailort
{
-typedef struct {
- volatile int head;
- volatile int tail;
- int size;
- int size_mask;
-} circbuf_t;
-
-//TODO: Do not change the behavior of this module. see PLDA descs impl..
-//TODO: optimize macros
-#ifndef MIN
-#define MIN(x,y) (((x) < (y)) ? (x) : (y))
-#endif
#ifdef _WIN32
#define _CB_FETCH(x) (InterlockedOr((LONG volatile*)(&x), (LONG)0))
#define _CB_SET(x, value) (InterlockedExchange((LONG volatile*)(&x), (LONG)(value)))
#define _CB_SET(x, value) ((void)__sync_lock_test_and_set(&(x), value))
#endif
-#define CB_INIT(circbuf, s) \
- (circbuf).head = 0; \
- (circbuf).tail = 0; \
- (circbuf).size = static_cast<int>(s); \
- (circbuf).size_mask = static_cast<int>((s) - 1)
-#define CB_RESET(circbuf) \
- (circbuf).head = 0; \
- (circbuf).tail = 0
-#define CB_HEAD(x) _CB_FETCH((x).head)
-#define CB_TAIL(x) _CB_FETCH((x).tail)
-#define CB_SIZE(x) _CB_FETCH((x).size)
-#define CB_ENQUEUE(circbuf, value) _CB_SET((circbuf).head, ((circbuf).head + (value)) & ((circbuf).size_mask))
-#define CB_DEQUEUE(circbuf, value) _CB_SET((circbuf).tail, ((circbuf).tail + (value)) & ((circbuf).size_mask))
-#define CB_AVAIL(circbuf, head, tail) ((((circbuf).size)-1+(tail)-(head)) & ((circbuf).size_mask))
-#define CB_AVAIL_CONT(circbuf, head, tail) \
- MIN(CB_AVAIL((circbuf), (head), (tail)), (circbuf).size - (head))
-#define CB_PROG(circbuf, head, tail) ((((circbuf).size)+(head)-(tail)) & ((circbuf).size_mask))
-#define CB_PROG_CONT(circbuf, head, tail) \
- MIN(CB_PROG((circbuf), (head), (tail)), (circbuf).size - (tail))
+// Note: We use tag dispatching to select the right implementation for power of 2 size
+// There's a minor performance gain for power of 2 size, as we can use a mask instead of modulo
+// * If a CircularBuffer/Array with the IsPow2Tag, then the size must be a power of 2.
+// * If a CircularBuffer/Array with the IsNotPow2Tag, then the size may be any positive integer (we simply won't
+// use the mask optimization for modulo operation, even if the size is a power of 2).
+struct IsPow2Tag {};
+struct IsNotPow2Tag {};
+template <typename Pow2Tag>
+struct CircularBuffer
+{
+public:
+ CircularBuffer(int s) :
+ m_head(0),
+ m_tail(0),
+ m_size(s),
+ m_size_mask(s - 1)
+ {
+ check_size(s, Pow2Tag());
+ }
+
+ void reset()
+ {
+ m_head = 0;
+ m_tail = 0;
+ }
+
+ void enqueue(int value)
+ {
+ _CB_SET(m_head, modulo(m_head + value, Pow2Tag()));
+ }
+
+ void set_head(int value)
+ {
+ _CB_SET(m_head, value);
+ }
+
+ void dequeue(int value)
+ {
+ _CB_SET(m_tail, modulo(m_tail + value, Pow2Tag()));
+ }
+
+ void set_tail(int value)
+ {
+ _CB_SET(m_tail, value);
+ }
+
+ int avail(int head, int tail) const
+ {
+ return modulo(m_size - 1 + tail - head, Pow2Tag());
+ }
+
+ int prog(int head, int tail) const
+ {
+ return modulo(m_size + head - tail, Pow2Tag());
+ }
+
+ int head() const
+ {
+ return _CB_FETCH(m_head);
+ }
+
+ int tail() const
+ {
+ return _CB_FETCH(m_tail);
+ }
+
+ int size() const
+ {
+ return m_size;
+ }
+
+ int size_mask() const
+ {
+ return m_size_mask;
+ }
+
+private:
+ int modulo(int val, IsPow2Tag) const
+ {
+ return val & m_size_mask;
+ }
+
+ int modulo(int val, IsNotPow2Tag) const
+ {
+ return val % m_size;
+ }
+
+ void check_size(size_t size, IsPow2Tag)
+ {
+ assert(0 != size);
+ assert(is_powerof2(size));
+
+ (void)size; // For release
+ }
+
+ void check_size(size_t size, IsNotPow2Tag)
+ {
+ assert(0 != size);
+
+ (void)size; // For release
+ }
+
+ volatile int m_head;
+ volatile int m_tail;
+ const int m_size;
+ // For power of 2 size, we can use a mask instead of modulo
+ const int m_size_mask;
+};
template<typename T>
// TODO: implement more functionalities, better move semantic handle
// TODO: support consts methods (front(), empty()), right now CB_* macros requires non const pointer to head+tail
-template<typename T, typename Container = std::vector<T>>
+template<typename T, typename Pow2Tag = IsPow2Tag, typename Container = std::vector<T>>
class CircularArray final
{
public:
-
static_assert(std::is_default_constructible<T>::value, "CircularArray object must be default constructible");
// Based on https://en.cppreference.com/w/cpp/iterator/iterator
{
public:
explicit iterator(int index, CircularArray &array) : m_array(array), m_index(index) {}
- iterator& operator++() { m_index = ((m_index + 1) & m_array.m_circ.size_mask); return *this; }
+ iterator& operator++() { increment(Pow2Tag()); return *this; }
iterator operator++(int) { iterator retval = *this; ++(*this); return retval; }
bool operator==(iterator other) const { return m_index == other.m_index; }
bool operator!=(iterator other) const { return !(*this == other); }
T &operator*() const { return m_array.m_array[m_index]; }
+
private:
+ void increment(IsPow2Tag) { m_index = (m_index + 1) & m_array.m_circ.size_mask(); }
+ void increment(IsNotPow2Tag) { m_index = (m_index + 1) % m_array.m_circ.size(); }
+
CircularArray &m_array;
int m_index;
};
// Ctor for Container=std::vector
template <typename C=Container,
class = typename std::enable_if_t<std::is_same<C, std::vector<T>>::value>>
- CircularArray(size_t storage_size)
+ CircularArray(size_t storage_size) :
+ m_circ(static_cast<int>(storage_size))
{
- // storage size must be a power of 2
- assert(is_powerof2(storage_size));
- CB_INIT(m_circ, storage_size);
m_array.resize(storage_size);
}
// Ctor for Container=std::array
template <typename C=Container,
class = typename std::enable_if_t<is_std_array<C>::value>>
- CircularArray(size_t storage_size, int = 0)
+ CircularArray(size_t storage_size, int = 0) :
+ m_circ(static_cast<int>(storage_size))
{
- // storage size must be a power of 2
- assert(is_powerof2(storage_size));
assert(storage_size <= std::tuple_size<C>::value);
- CB_INIT(m_circ, storage_size);
}
void push_back(T &&element)
{
assert(!full());
- m_array[CB_HEAD(m_circ)] = std::move(element);
- CB_ENQUEUE(m_circ, 1);
+ m_array[m_circ.head()] = std::move(element);
+ m_circ.enqueue(1);
}
void push_back(const T& element)
{
assert(!full());
- m_array[CB_HEAD(m_circ)] = element;
- CB_ENQUEUE(m_circ, 1);
+ m_array[m_circ.head()] = element;
+ m_circ.enqueue(1);
}
void pop_front()
{
assert(!empty());
// Clear previous front
- m_array[CB_TAIL(m_circ)] = T();
- CB_DEQUEUE(m_circ, 1);
+ m_array[m_circ.tail()] = T();
+ m_circ.dequeue(1);
}
T &front()
{
assert(!empty());
- return m_array[CB_TAIL(m_circ)];
+ return m_array[m_circ.tail()];
}
void reset()
bool empty() const
{
- return CB_HEAD(m_circ) == CB_TAIL(m_circ);
+ return m_circ.head() == m_circ.tail();
}
bool full() const
{
- return 0 == CB_AVAIL(m_circ, CB_HEAD(m_circ), CB_TAIL(m_circ));
+ return 0 == m_circ.avail(m_circ.head(), m_circ.tail());
}
size_t size() const
{
- return CB_PROG(m_circ, CB_HEAD(m_circ), CB_TAIL(m_circ));
+ return m_circ.prog(m_circ.head(), m_circ.tail());
}
size_t capacity() const
{
- return CB_SIZE(m_circ) - 1;
+ return m_circ.size() - 1;
}
iterator begin()
{
- return iterator(CB_TAIL(m_circ), *this);
+ return iterator(m_circ.tail(), *this);
}
iterator end()
{
- return iterator(CB_HEAD(m_circ), *this);
+ return iterator(m_circ.head(), *this);
}
private:
- circbuf_t m_circ;
+ CircularBuffer<Pow2Tag> m_circ;
Container m_array;
};
hailo_status WaitOrShutdown::wait(std::chrono::milliseconds timeout)
{
- auto index = m_waitable_group.wait_any(timeout);
- if (index.status() == HAILO_TIMEOUT) {
- return index.status();
- }
- CHECK_EXPECTED_AS_STATUS(index);
-
- assert(index.value() <= WAITABLE_INDEX);
- return (index.value() == SHUTDOWN_INDEX) ? HAILO_SHUTDOWN_EVENT_SIGNALED : HAILO_SUCCESS;
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_TIMEOUT, const auto index, m_waitable_group.wait_any(timeout));
+ assert(index <= WAITABLE_INDEX);
+ return (index == SHUTDOWN_INDEX) ? HAILO_SHUTDOWN_EVENT_SIGNALED : HAILO_SUCCESS;
}
hailo_status WaitOrShutdown::signal()
{
+ // Cannot signal a WaitOrShutdown which has only shutdown event
+ CHECK_NOT_NULL(m_waitable, HAILO_INVALID_OPERATION);
return m_waitable->signal();
}
// Note the order - consistent with SHUTDOWN_INDEX, WAITABLE_INDEX.
std::vector<std::reference_wrapper<Waitable>> waitables;
waitables.emplace_back(std::ref(*shutdown_event));
- waitables.emplace_back(std::ref(*waitable));
+
+ if (nullptr != waitable) {
+ waitables.emplace_back(std::ref(*waitable));
+ }
+
return waitables;
}
s.seekg(beg_pos, s.beg);
CHECK_AS_EXPECTED(s.good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed");
- auto total_size = static_cast<uint64_t>(size - beg_pos);
+ auto total_size = static_cast<size_t>(size - beg_pos);
CHECK_AS_EXPECTED(total_size <= std::numeric_limits<size_t>::max(), HAILO_FILE_OPERATION_FAILURE,
"File size {} is too big", total_size);
return Expected<size_t>(static_cast<size_t>(total_size));
std::ifstream file(file_path, std::ios::in | std::ios::binary);
CHECK_AS_EXPECTED(file.good(), HAILO_OPEN_FILE_FAILURE, "Error opening file {}", file_path);
- auto file_size = get_istream_size(file);
- CHECK_EXPECTED(file_size, "Failed to get file size");
-
- auto buffer = Buffer::create(file_size.value(), output_buffer_params);
- CHECK_EXPECTED(buffer, "Failed to allocate file buffer ({} bytes}", file_size.value());
+ TRY(const auto file_size, get_istream_size(file), "Failed to get file size");
+ TRY(auto buffer, Buffer::create(file_size, output_buffer_params),
+ "Failed to allocate file buffer ({} bytes}", file_size);
// Read the data
- file.read(reinterpret_cast<char*>(buffer->data()), buffer->size());
+ file.read(reinterpret_cast<char*>(buffer.data()), buffer.size());
CHECK_AS_EXPECTED(file.good(), HAILO_FILE_OPERATION_FAILURE, "Failed reading file {}", file_path);
- return buffer.release();
+ return buffer;
+}
+
+Expected<std::shared_ptr<FileReader>> SeekableBytesReader::create_reader(const std::string &file_path)
+{
+ auto ptr = make_shared_nothrow<FileReader>(file_path);
+ CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+ return ptr;
+}
+
+Expected<std::shared_ptr<BufferReader>> SeekableBytesReader::create_reader(const MemoryView &memview)
+{
+ auto ptr = make_shared_nothrow<BufferReader>(memview);
+ CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+ return ptr;
+}
+
+FileReader::FileReader(const std::string &file_path) : m_file_path(file_path) {}
+
+hailo_status FileReader::read(uint8_t *buffer, size_t n)
+{
+ assert(nullptr != m_fstream);
+ (void)m_fstream->read(reinterpret_cast<char*>(buffer), n);
+ return m_fstream->good() ? HAILO_SUCCESS : HAILO_FILE_OPERATION_FAILURE;
+}
+
+hailo_status FileReader::read_from_offset(size_t offset, MemoryView &dst, size_t size)
+{
+ assert(nullptr != m_fstream);
+
+ auto beg_pos = m_fstream->tellg();
+ (void)m_fstream->seekg(offset);
+ CHECK(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed");
+
+ (void)m_fstream->read(reinterpret_cast<char*>(dst.data()), size);
+ CHECK(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::read() failed");
+
+ (void)m_fstream->seekg(beg_pos);
+ CHECK(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed");
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status FileReader::open()
+{
+ if (nullptr == m_fstream) { // The first call to open creates the ifstream object
+ m_fstream = std::make_shared<std::ifstream>(m_file_path, std::ios::in | std::ios::binary);
+ return m_fstream->good() ? HAILO_SUCCESS : HAILO_OPEN_FILE_FAILURE;
+ }
+ m_fstream->open(m_file_path, std::ios::in | std::ios::binary);
+ return m_fstream->good() ? HAILO_SUCCESS : HAILO_OPEN_FILE_FAILURE;
+}
+
+bool FileReader::is_open() const
+{
+ return m_fstream->is_open();
+}
+
+hailo_status FileReader::seek(size_t position)
+{
+ assert(nullptr != m_fstream);
+ (void)m_fstream->seekg(position, m_fstream->beg);
+ return m_fstream->good() ? HAILO_SUCCESS : HAILO_FILE_OPERATION_FAILURE;
+}
+
+Expected<size_t> FileReader::tell()
+{
+ assert(nullptr != m_fstream);
+ auto offset = m_fstream->tellg();
+ return m_fstream->good() ? Expected<size_t>(static_cast<size_t>(offset)) : make_unexpected(HAILO_FILE_OPERATION_FAILURE);
+}
+
+hailo_status FileReader::close()
+{
+ assert(nullptr != m_fstream);
+ m_fstream->close();
+ return m_fstream->good() ? HAILO_SUCCESS : HAILO_CLOSE_FAILURE;
+}
+
+Expected<size_t> FileReader::get_size()
+{
+ assert(nullptr != m_fstream);
+
+ auto beg_pos = m_fstream->tellg();
+ CHECK_AS_EXPECTED(-1 != beg_pos, HAILO_FILE_OPERATION_FAILURE, "ifstream::tellg() failed");
+
+ (void)m_fstream->seekg(0, m_fstream->end);
+ CHECK_AS_EXPECTED(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed");
+
+ auto file_size = m_fstream->tellg();
+ CHECK_AS_EXPECTED(-1 != file_size, HAILO_FILE_OPERATION_FAILURE, "ifstream::tellg() failed");
+
+ (void)m_fstream->seekg(beg_pos, m_fstream->beg);
+ CHECK_AS_EXPECTED(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed");
+
+ return static_cast<size_t>(file_size);
+}
+
+std::shared_ptr<std::ifstream> FileReader::get_fstream() const
+{
+ return m_fstream;
+}
+
+Expected<size_t> FileReader::calculate_remaining_size()
+{
+ assert(nullptr != m_fstream);
+ auto remaining_size = get_istream_size(*m_fstream);
+ CHECK_AS_EXPECTED(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "FileReader::calculate_remaining_size() failed");
+ return remaining_size;
+}
+
+Expected<bool> FileReader::good() const
+{
+ assert(nullptr != m_fstream);
+ return m_fstream->good();
+}
+
+BufferReader::BufferReader(const MemoryView &memview) : m_memview(memview) {}
+
+hailo_status BufferReader::read(uint8_t *buffer, size_t n)
+{
+ assert(m_seek_offset + n <= m_memview.size());
+ memcpy(buffer, m_memview.data() + m_seek_offset, n);
+ m_seek_offset += n;
+ return HAILO_SUCCESS;
+}
+
+hailo_status BufferReader::read_from_offset(size_t offset, MemoryView &dst, size_t size)
+{
+ memcpy(dst.data(), m_memview.data() + offset, size);
+ return HAILO_SUCCESS;
+}
+
+hailo_status BufferReader::open()
+{
+ // In case we use the buffer, we don't need to check if the file is open
+ return HAILO_SUCCESS;
+}
+
+bool BufferReader::is_open() const
+{
+ // In case we use the buffer, we don't need to check if the file is open
+ return true;
+}
+
+hailo_status BufferReader::seek(size_t position)
+{
+ assert(position < m_memview.size());
+ m_seek_offset = position;
+ return HAILO_SUCCESS;
+}
+
+Expected<size_t> BufferReader::tell()
+{
+ return Expected<size_t>(m_seek_offset);
+}
+
+hailo_status BufferReader::close()
+{
+ return HAILO_SUCCESS;
+}
+
+Expected<size_t> BufferReader::get_size()
+{
+ return Expected<size_t>(m_memview.size());
+}
+
+Expected<size_t> BufferReader::calculate_remaining_size()
+{
+ return m_memview.size() - m_seek_offset;
+}
+
+Expected<bool> BufferReader::good() const
+{
+ return true;
+}
+
+const MemoryView BufferReader::get_memview() const
+{
+ return m_memview;
}
} /* namespace hailort */
Expected<Buffer> read_binary_file(const std::string &file_path,
const BufferStorageParams &output_buffer_params = {});
+class FileReader;
+class BufferReader;
+
+class SeekableBytesReader
+{
+public:
+ virtual ~SeekableBytesReader() = default;
+ virtual hailo_status read(uint8_t *buffer, size_t n) = 0;
+ virtual hailo_status read_from_offset(size_t offset, MemoryView &dst, size_t n) = 0;
+ virtual hailo_status open() = 0;
+ virtual bool is_open() const = 0;
+ virtual hailo_status seek(size_t position) = 0;
+ virtual Expected<size_t> tell() = 0;
+ virtual hailo_status close() = 0;
+ virtual Expected<size_t> get_size() = 0;
+ virtual Expected<bool> good() const = 0;
+ virtual Expected<size_t> calculate_remaining_size() = 0;
+ static Expected<std::shared_ptr<FileReader>> create_reader(const std::string &file_path);
+ static Expected<std::shared_ptr<BufferReader>> create_reader(const MemoryView &memview);
+};
+
+class FileReader : public SeekableBytesReader
+{
+public:
+ FileReader(const std::string &file_path);
+
+ virtual hailo_status read(uint8_t *buffer, size_t n);
+ virtual hailo_status read_from_offset(size_t offset, MemoryView &dst, size_t n);
+ virtual hailo_status open();
+ virtual bool is_open() const;
+ virtual hailo_status seek(size_t position);
+ virtual Expected<size_t> tell();
+ virtual hailo_status close();
+ virtual Expected<size_t> get_size();
+ virtual Expected<bool> good() const;
+ virtual Expected<size_t> calculate_remaining_size();
+
+ std::shared_ptr<std::ifstream> get_fstream() const;
+
+private:
+ std::shared_ptr<std::ifstream> m_fstream = nullptr;
+ std::string m_file_path;
+
+};
+
+class BufferReader : public SeekableBytesReader
+{
+public:
+ BufferReader(const MemoryView &memview);
+
+ virtual hailo_status read(uint8_t *buffer, size_t n);
+ virtual hailo_status read_from_offset(size_t offset, MemoryView &dst, size_t n);
+ virtual hailo_status open();
+ virtual bool is_open() const;
+ virtual hailo_status seek(size_t position);
+ virtual Expected<size_t> tell();
+ virtual hailo_status close();
+ virtual Expected<size_t> get_size();
+ virtual Expected<bool> good() const;
+ virtual Expected<size_t> calculate_remaining_size();
+
+ const MemoryView get_memview() const;
+
+private:
+ MemoryView m_memview;
+ size_t m_seek_offset = 0;
+};
+
} /* namespace hailort */
#endif /* _HAILO_FILE_UTILS_HPP_ */
FindFile &operator=(FindFile &&other) = delete;
FindFile(FindFile &&other);
- Filesystem::FileInfo get_cur_file_info();
+ Filesystem::FileInfo get_cur_file_info() const;
// Will return HAILO_INVALID_OPERATION when the iteration is complete or HAILO_FILE_OPERATION_FAILURE upon failure
hailo_status next_file();
#endif
#endif
+#if defined(__linux__) && !defined(__ANDROID__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstringop-overflow"
+#endif
#include <spdlog/spdlog.h>
#include <spdlog/fmt/ostr.h>
+#if defined(__linux__) && !defined(__ANDROID__)
+#pragma GCC diagnostic pop
+#endif
inline std::ostream& operator<<(std::ostream& os, const hailo_status& status)
{
struct ifreq ifr = {};
/* Create socket */
- auto socket = Socket::create(AF_INET, SOCK_DGRAM, 0);
- CHECK_EXPECTED(socket);
+ TRY(const auto socket, Socket::create(AF_INET, SOCK_DGRAM, 0));
/* Convert interface name to ip address */
ifr.ifr_addr.sa_family = AF_INET;
(void)strncpy(ifr.ifr_name, interface_name.c_str(), IFNAMSIZ-1);
- auto posix_rc = ioctl(socket->get_fd(), SIOCGIFADDR, &ifr);
+ auto posix_rc = ioctl(socket.get_fd(), SIOCGIFADDR, &ifr);
CHECK_AS_EXPECTED(posix_rc >= 0, HAILO_ETH_INTERFACE_NOT_FOUND,
"Interface was not found. ioctl with SIOCGIFADDR has failed. errno: {:#x}", errno);
{
const std::string dir_path_with_sep = has_suffix(dir_path, SEPARATOR) ? dir_path : dir_path + SEPARATOR;
- auto dir = DirWalker::create(dir_path_with_sep);
- CHECK_EXPECTED(dir);
+ TRY(auto dir, DirWalker::create(dir_path_with_sep));
std::vector<std::string> files;
struct dirent *entry = nullptr;
- while ((entry = dir->next_file()) != nullptr) {
+ while ((entry = dir.next_file()) != nullptr) {
if (entry->d_type != DT_REG) {
continue;
}
std::time_t curr_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
const std::string dir_path_with_sep = has_suffix(dir_path, SEPARATOR) ? dir_path : dir_path + SEPARATOR;
- auto dir = DirWalker::create(dir_path_with_sep);
- CHECK_EXPECTED(dir);
+ TRY(auto dir, DirWalker::create(dir_path_with_sep));
std::vector<std::string> files;
struct dirent *entry = nullptr;
- while ((entry = dir->next_file()) != nullptr) {
+ while ((entry = dir.next_file()) != nullptr) {
if (entry->d_type != DT_REG) {
continue;
}
const std::string file_path = dir_path_with_sep + std::string(entry->d_name);
- auto file_modified_time = get_file_modified_time(file_path);
- CHECK_EXPECTED(file_modified_time);
+ TRY(const auto file_modified_time, get_file_modified_time(file_path));
- auto time_diff_sec = std::difftime(curr_time, file_modified_time.value());
+ auto time_diff_sec = std::difftime(curr_time, file_modified_time);
auto time_diff_millisec = time_diff_sec * 1000;
if (time_diff_millisec <= static_cast<double>(time_interval.count())) {
files.emplace_back(file_path);
Expected<std::pair<int32_t, std::string>> Process::create_and_wait_for_output(const std::string &command_line, uint32_t max_output_size)
{
- auto popen_expected = PopenWrapper::create(command_line);
- CHECK_EXPECTED(popen_expected);
- const auto output_expected = popen_expected->read_stdout(max_output_size);
- CHECK_EXPECTED(output_expected);
- const auto process_exit_code = popen_expected->close();
- return std::make_pair(process_exit_code, output_expected.value());
+ TRY(auto popen, PopenWrapper::create(command_line));
+ TRY(const auto output, popen.read_stdout(max_output_size));
+ const auto process_exit_code = popen.close();
+ return std::make_pair(process_exit_code, output);
}
Expected<Process::PopenWrapper> Process::PopenWrapper::create(const std::string &command_line)
{
assert (nullptr != m_pipe);
- // We zero out the bufer so that output won't contain junk from the heap
- auto output = Buffer::create(max_output_size, 0);
- CHECK_EXPECTED(output);
+ // We zero out the buffer so that output won't contain junk from the heap
+ TRY(auto output, Buffer::create(max_output_size, 0));
- const auto num_read = fread(reinterpret_cast<char*>(output->data()), sizeof(uint8_t), output->size(), m_pipe);
- if (num_read != output->size()) {
+ const auto num_read = fread(reinterpret_cast<char*>(output.data()), sizeof(uint8_t), output.size(), m_pipe);
+ if (num_read != output.size()) {
if (feof(m_pipe)) {
// We remove the trailing newline we get from fread
- const auto output_as_str = output->to_string();
+ const auto output_as_str = output.to_string();
if (output_as_str[output_as_str.length() - 1] == '\n') {
return output_as_str.substr(0, num_read - 1);
}
} else {
// Truncate output
LOGGER__TRACE("Truncating output to {} chars long", max_output_size);
- return output->to_string();
+ return output.to_string();
}
}
Expected<Socket> Socket::create(int af, int type, int protocol)
{
- auto module_wrapper = SocketModuleWrapper::create();
- CHECK_EXPECTED(module_wrapper);
-
- auto socket_fd = create_socket_fd(af, type, protocol);
- CHECK_EXPECTED(socket_fd);
+ TRY(auto module_wrapper, SocketModuleWrapper::create());
+ TRY(const auto socket_fd, create_socket_fd(af, type, protocol));
- auto obj = Socket(module_wrapper.release(), socket_fd.release());
+ auto obj = Socket(std::move(module_wrapper), socket_fd);
return obj;
}
Expected<TrafficControlUtil> TrafficControlUtil::create(const std::string &ip, uint16_t port, uint32_t rate_bytes_per_sec)
{
- auto interface_name = EthernetUtils::get_interface_from_board_ip(ip);
- CHECK_EXPECTED(interface_name, "get_interface_name failed with status {}", interface_name.status());
+ TRY(const auto interface_name, EthernetUtils::get_interface_from_board_ip(ip), "get_interface_name failed");
+ TRY(const auto board_id, ip_to_board_id(ip), "ip_to_board_id failed");
+ TRY(const auto is_sudo_needed, check_is_sudo_needed(), "check_is_sudo_needed failed");
- auto board_id = ip_to_board_id(ip);
- CHECK_EXPECTED(board_id, "ip_to_board_id failed with status {}", board_id.status());
-
- auto is_sudo_needed = check_is_sudo_needed();
- CHECK_EXPECTED(is_sudo_needed, "check_is_sudo_needed failed with status {}", is_sudo_needed.status());
-
- return TrafficControlUtil(ip, interface_name.release(), board_id.release(), port, port_to_port_id(port),
- rate_bytes_per_sec, is_sudo_needed.release());
+ return TrafficControlUtil(ip, interface_name, board_id, port, port_to_port_id(port),
+ rate_bytes_per_sec, is_sudo_needed);
}
TrafficControlUtil::TrafficControlUtil(const std::string& board_address, const std::string& interface_name,
Expected<bool> TrafficControlUtil::check_is_sudo_needed()
{
- const auto result = Process::create_and_wait_for_output("id -u", MAX_COMMAND_OUTPUT_LENGTH);
- CHECK_EXPECTED(result);
+ TRY(const auto result, Process::create_and_wait_for_output("id -u", MAX_COMMAND_OUTPUT_LENGTH));
// If the user id is zero then we don't need to add `sudo` to our commands
- return std::move(result->second != "0");
+ return std::move(result.second != "0");
}
hailo_status TrafficControlUtil::run_command(const std::string &commnad, bool add_sudo,
const std::vector<std::string> &allowed_errors, bool ignore_fails)
{
// Note: we redirect stderr to stdout
- const auto result = Process::create_and_wait_for_output(
+ TRY(const auto result, Process::create_and_wait_for_output(
add_sudo ? "sudo " + commnad + " 2>&1" : commnad + " 2>&1",
- MAX_COMMAND_OUTPUT_LENGTH);
- CHECK_EXPECTED_AS_STATUS(result);
+ MAX_COMMAND_OUTPUT_LENGTH));
- const uint32_t exit_code = result->first;
+ const uint32_t exit_code = result.first;
if (0 == exit_code) {
return HAILO_SUCCESS;
}
- std::string cmd_output = result->second;
+ std::string cmd_output = result.second;
// No output = everything was OK
bool is_output_valid = cmd_output.empty();
if ((!is_output_valid) && (!allowed_errors.empty())) {
Expected<TrafficControl> TrafficControl::create(const std::string &ip, uint16_t port, uint32_t rate_bytes_per_sec)
{
- auto tc_util = TrafficControlUtil::create(ip, port, rate_bytes_per_sec);
- CHECK_EXPECTED(tc_util);
+ TRY(auto tc_util, TrafficControlUtil::create(ip, port, rate_bytes_per_sec));
hailo_status rate_set_status = HAILO_UNINITIALIZED;
- TrafficControl tc(tc_util.release(), rate_set_status);
+ TrafficControl tc(std::move(tc_util), rate_set_status);
CHECK_SUCCESS_AS_EXPECTED(rate_set_status, "Failed setting rate limit with status {}", rate_set_status);
return tc;
return make_unexpected(HAILO_UNEXPECTED_INTERFACE_INFO_FAILURE);
}
- auto interface_info_buffer = Buffer::create(required_size, 0);
- CHECK_EXPECTED(interface_info_buffer);
+ TRY(auto interface_info_buffer, Buffer::create(required_size, 0));
ret_value = GetAdaptersAddresses(IPV4, UNICAST_ONLY, RESERVED,
- interface_info_buffer->as_pointer<IP_ADAPTER_ADDRESSES>(), &required_size);
+ interface_info_buffer.as_pointer<IP_ADAPTER_ADDRESSES>(), &required_size);
if (ret_value == ERROR_NO_DATA) {
LOGGER__ERROR("No IPv4 interfaces found");
return make_unexpected(HAILO_NO_IPV4_INTERFACES_FOUND);
}
NetworkInterfaces interfaces;
- PIP_ADAPTER_ADDRESSES interface_info = interface_info_buffer->as_pointer<IP_ADAPTER_ADDRESSES>();
+ PIP_ADAPTER_ADDRESSES interface_info = interface_info_buffer.as_pointer<IP_ADAPTER_ADDRESSES>();
while (interface_info != nullptr) {
PIP_ADAPTER_UNICAST_ADDRESS first_unicast_address = interface_info->FirstUnicastAddress;
continue;
}
- auto ip = Buffer::create(IPV4_STRING_MAX_LENGTH);
- CHECK_EXPECTED(ip);
+ TRY(auto ip, Buffer::create(IPV4_STRING_MAX_LENGTH));
const auto result = Socket::ntop(AF_INET, &(reinterpret_cast<sockaddr_in *>(address_struct)->sin_addr),
- ip->as_pointer<char>(), EthernetUtils::MAX_INTERFACE_SIZE);
+ ip.as_pointer<char>(), EthernetUtils::MAX_INTERFACE_SIZE);
if (result != HAILO_SUCCESS) {
LOGGER__DEBUG("Failed converting unicast address to string (result={}). Skipping.", result);
continue;
continue;
}
interfaces.emplace_back(interface_info->IfIndex, interface_info->AdapterName,
- friendly_name_ansi.value(), ip->to_string());
+ friendly_name_ansi.value(), ip.to_string());
interface_info = interface_info->Next;
}
return make_unexpected(HAILO_UNEXPECTED_ARP_TABLE_FAILURE);
}
- auto ip_net_table_buffer = Buffer::create(required_size, 0);
- CHECK_EXPECTED(ip_net_table_buffer);
- ret_value = GetIpNetTable(ip_net_table_buffer->as_pointer<MIB_IPNETTABLE>(), &required_size, SORTED);
+ TRY(auto ip_net_table_buffer, Buffer::create(required_size, 0));
+ ret_value = GetIpNetTable(ip_net_table_buffer.as_pointer<MIB_IPNETTABLE>(), &required_size, SORTED);
if (ret_value == ERROR_NO_DATA) {
LOGGER__ERROR("No IPv4 interfaces found");
return make_unexpected(HAILO_NO_IPV4_INTERFACES_FOUND);
}
std::unordered_map<uint32_t, MacAddress> result;
- const PMIB_IPNETTABLE ip_net_table = ip_net_table_buffer->as_pointer<MIB_IPNETTABLE>();
+ const PMIB_IPNETTABLE ip_net_table = ip_net_table_buffer.as_pointer<MIB_IPNETTABLE>();
for (uint32_t i = 0; i < ip_net_table->dwNumEntries; i++) {
if (ip_net_table->table[i].dwIndex != interface_index) {
continue;
Expected<std::string> EthernetUtils::get_interface_from_board_ip(const std::string &board_ip)
{
- auto network_interfaces = NetworkInterface::get_all_interfaces();
- CHECK_EXPECTED(network_interfaces);
+ TRY(const auto network_interfaces, NetworkInterface::get_all_interfaces());
struct in_addr board_ip_struct{};
auto status = Socket::pton(AF_INET, board_ip.c_str(), &board_ip_struct);
CHECK_SUCCESS_AS_EXPECTED(status, "Invalid board ip address {}", board_ip);
- for (const auto& network_interface : network_interfaces.value()) {
- auto arp_table = ArpTable::create(network_interface.index());
- CHECK_EXPECTED(arp_table);
-
- const auto mac_address = arp_table->get_mac_address(static_cast<uint32_t>(board_ip_struct.S_un.S_addr));
+ for (const auto &network_interface : network_interfaces) {
+ TRY(const auto arp_table, ArpTable::create(network_interface.index()));
+ const auto mac_address = arp_table.get_mac_address(static_cast<uint32_t>(board_ip_struct.S_un.S_addr));
if (mac_address) {
return network_interface.friendly_name();
}
Expected<std::string> EthernetUtils::get_ip_from_interface(const std::string &interface_name)
{
- auto network_interfaces = NetworkInterface::get_all_interfaces();
- CHECK_EXPECTED(network_interfaces);
+ TRY(const auto network_interfaces, NetworkInterface::get_all_interfaces());
- for (const auto& network_interface : network_interfaces.value()) {
+ for (const auto &network_interface : network_interfaces) {
if (network_interface.friendly_name() == interface_name) {
return network_interface.ip();
}
m_find_data(other.m_find_data)
{}
-Filesystem::FileInfo Filesystem::FindFile::get_cur_file_info()
+Filesystem::FileInfo Filesystem::FindFile::get_cur_file_info() const
{
return {m_find_data.cFileName, m_find_data.dwFileAttributes};
}
{
const std::string dir_path_with_sep = has_suffix(dir_path, SEPARATOR) ? dir_path : dir_path + SEPARATOR;
- auto dir = FindFile::create(dir_path_with_sep);
- CHECK_EXPECTED(dir);
+ TRY(auto dir, FindFile::create(dir_path_with_sep));
std::vector<std::string> files;
- auto file_info = dir->get_cur_file_info();
+ auto file_info = dir.get_cur_file_info();
if (is_regular_or_readonly_file(file_info.attrs)) {
files.emplace_back(file_info.path);
}
hailo_status status = HAILO_UNINITIALIZED;
while (true) {
- status = dir->next_file();
+ status = dir.next_file();
if (HAILO_INVALID_OPERATION == status) {
// We're done
break;
continue;
}
- file_info = dir->get_cur_file_info();
+ file_info = dir.get_cur_file_info();
if (is_regular_or_readonly_file(file_info.attrs)) {
files.emplace_back(dir_path_with_sep + file_info.path);
}
Expected<Socket> Socket::create(int af, int type, int protocol)
{
- auto module_wrapper = SocketModuleWrapper::create();
- CHECK_EXPECTED(module_wrapper);
-
- auto socket_fd = create_socket_fd(af, type, protocol);
- CHECK_EXPECTED(socket_fd);
+ TRY(auto module_wrapper, SocketModuleWrapper::create());
+ TRY(const auto socket_fd, create_socket_fd(af, type, protocol));
- auto obj = Socket(module_wrapper.release(), socket_fd.release());
+ auto obj = Socket(std::move(module_wrapper), socket_fd);
return std::move(obj);
}
CHECK_ARG_NOT_NULL(src);
CHECK_ARG_NOT_NULL(dst);
- auto module_wrapper = SocketModuleWrapper::create();
- CHECK_EXPECTED_AS_STATUS(module_wrapper);
+ TRY(const auto module_wrapper, SocketModuleWrapper::create());
const char *inet_result = inet_ntop(af, src, dst, size);
CHECK(nullptr != inet_result, HAILO_ETH_FAILURE, "Failed inet_ntop. WSALE={}", WSAGetLastError());
CHECK_ARG_NOT_NULL(src);
CHECK_ARG_NOT_NULL(dst);
- auto module_wrapper = SocketModuleWrapper::create();
- CHECK_EXPECTED_AS_STATUS(module_wrapper);
+ TRY(const auto module_wrapper, SocketModuleWrapper::create());
inet_result = inet_pton(af, src, dst);
if (1 != inet_result) {
m_module_wrapper(std::move(other.m_module_wrapper)), m_socket_fd(std::exchange(other.m_socket_fd, INVALID_SOCKET))
{};
- socket_t get_fd() { return m_socket_fd; }
+ socket_t get_fd() const { return m_socket_fd; }
static hailo_status ntop(int af, const void *src, char *dst, socklen_t size);
static hailo_status pton(int af, const char *src, void *dst);
Expected<uint8_t> StringUtils::to_uint8(const std::string &str, int base)
{
- auto number = to_uint32(str, base);
- CHECK_EXPECTED(number);
+ TRY(const auto number, to_uint32(str, base));
- CHECK_AS_EXPECTED(((number.value() >= std::numeric_limits<uint8_t>::min()) && (number.value() <= std::numeric_limits<uint8_t>::max())),
+ CHECK_AS_EXPECTED(((number >= std::numeric_limits<uint8_t>::min()) && (number <= std::numeric_limits<uint8_t>::max())),
HAILO_INVALID_ARGUMENT, "Failed to convert string {} to uint8_t.", str);
- return static_cast<uint8_t>(number.value());
+ return static_cast<uint8_t>(number);
}
std::string StringUtils::to_hex_string(const uint8_t *array, size_t size, bool uppercase, const std::string &delimiter)
#include "hailo/hailort.h"
#include "hailo/expected.hpp"
+#include "hailo/buffer.hpp"
#include "common/logger_macros.hpp"
#include <spdlog/fmt/bundled/core.h>
#include <map>
#include <set>
#include <unordered_set>
+#include <cstdint>
+#include <cstddef>
+#include <fstream>
namespace hailort
#define IS_FIT_IN_UINT16(number) ((std::numeric_limits<uint16_t>::max() >= ((int32_t)(number))) && (std::numeric_limits<uint16_t>::min() <= ((int32_t)(number))))
#define IS_FIT_IN_UINT32(number) ((std::numeric_limits<uint32_t>::max() >= ((int64_t)(number))) && (std::numeric_limits<uint32_t>::min() <= ((int64_t)(number))))
+static const uint32_t POLYNOMIAL = 0xEDB88320;
+
+static const size_t MB = 1024 * 1024;
+
template <typename T>
static inline bool contains(const std::vector<T> &container, const T &value)
{
#define CHECK_GRPC_STATUS_AS_EXPECTED(status) _CHECK_GRPC_STATUS(status, make_unexpected(HAILO_RPC_FAILED), SERVICE_WARNING_MSG)
#endif
+// Macros that check status. If status is 'valid_error', return without printing error to the prompt.
+#define CHECK_EXPECTED_WITH_ACCEPTABLE_STATUS(valid_error, exp, ...) if (valid_error == (exp).status()) {return make_unexpected(valid_error);} CHECK_SUCCESS(exp, __VA_ARGS__);
+
+
#define __HAILO_CONCAT(x, y) x ## y
#define _HAILO_CONCAT(x, y) __HAILO_CONCAT(x, y)
CHECK_EXPECTED(expected_var_name, __VA_ARGS__); \
var_decl = expected_var_name.release()
+#define _TRY_V(expected_var_name, var_decl, expr, ...) \
+ auto expected_var_name = (expr); \
+ CHECK_EXPECTED(expected_var_name, __VA_ARGS__); \
+ var_decl = expected_var_name.value()
+
/**
* The TRY macro is used to allow easier validation and access for variables returned as Expected<T>.
* If the expression returns an Expected<T> with status HAILO_SUCCESS, the macro will release the expected and assign
*/
#define TRY(var_decl, expr, ...) _TRY(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__)
+/**
+ * Same us TRY macro but instead of returning released value, it will return the value itself.
+*/
+// TODO: HRT-13624: Remove after 'expected' implementation is fixed
+#define TRY_V(var_decl, expr, ...) _TRY_V(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__)
+
+#define _TRY_WITH_ACCEPTABLE_STATUS(valid_error, expected_var_name, var_decl, expr, ...) \
+ auto expected_var_name = (expr); \
+ CHECK_EXPECTED_WITH_ACCEPTABLE_STATUS(valid_error, expected_var_name, __VA_ARGS__); \
+ var_decl = expected_var_name.release()
+
+#define TRY_WITH_ACCEPTABLE_STATUS(valid_error, var_decl, expr, ...) _TRY_WITH_ACCEPTABLE_STATUS(valid_error, _HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__)
+
#ifndef _MSC_VER
#define IGNORE_DEPRECATION_WARNINGS_BEGIN _Pragma("GCC diagnostic push") \
_Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
return ((nullptr != env_var) && (strncmp(env_var, required_value.c_str(), required_value.size()) == 0));
}
+static inline Expected<std::string> get_env_variable(const std::string &env_var_name)
+{
+ const auto env_var = std::getenv(env_var_name.c_str());
+ // Using ifs instead of CHECKs to avoid printing the error message
+ if (nullptr == env_var) {
+ return make_unexpected(HAILO_NOT_FOUND);
+ }
+
+ const auto result = std::string(env_var);
+ if (result.empty()) {
+ return make_unexpected(HAILO_NOT_FOUND);
+ }
+
+ return Expected<std::string>(result);
+}
+
+class CRC32 {
+public:
+ CRC32() {
+ generate_table();
+ }
+
+ uint32_t calculate(std::ifstream &s, size_t buffer_size) const {
+ auto beg_pos = s.tellg(); // Saves current position
+ uint32_t crc = 0xFFFFFFFF;
+ std::vector<char> buffer(MB);
+
+ size_t total_bytes_read = 0;
+
+ while (total_bytes_read < buffer_size) {
+ size_t bytes_to_read = std::min(buffer_size - total_bytes_read, MB);
+ s.read(buffer.data(), bytes_to_read);
+
+ size_t bytes_read = s.gcount();
+ total_bytes_read += bytes_read;
+ for (size_t i = 0; i < bytes_read; ++i) {
+ crc = (crc >> 8) ^ table[(crc ^ static_cast<uint8_t>(buffer[i])) & 0xFF];
+ }
+ }
+
+ s.seekg(beg_pos, std::ios::beg); // Return to the original position
+ return crc ^ 0xFFFFFFFF;
+ }
+
+ uint32_t calculate(const MemoryView &buffer) const {
+ uint32_t crc = 0xFFFFFFFF;
+ auto data = buffer.data();
+
+ for (size_t i = 0; i < buffer.size(); ++i) {
+ crc = (crc >> 8) ^ table[(crc ^ data[i]) & 0xFF];
+ }
+
+ return crc ^ 0xFFFFFFFF;
+ }
+
+ static Expected<uint32_t> calc_crc_on_buffer(const MemoryView &buffer)
+ {
+ CRC32 crcCalculator;
+ return crcCalculator.calculate(buffer);
+ }
+
+ static Expected<uint32_t> calc_crc_on_stream(std::ifstream &s, size_t size)
+ {
+ CRC32 crcCalculator;
+ return crcCalculator.calculate(s, size);
+ }
+
+private:
+ uint32_t table[256];
+
+ void generate_table() {
+ for (uint32_t i = 0; i < 256; ++i) {
+ uint32_t crc = i;
+ for (uint32_t j = 0; j < 8; ++j) {
+ crc = (crc & 1) ? (crc >> 1) ^ POLYNOMIAL : (crc >> 1);
+ }
+ table[i] = crc;
+ }
+ }
+};
+
+class BufferUtils final
+{
+public:
+ BufferUtils() = delete;
+
+ static void summarize_buffer(const Buffer& buffer, std::ostream& os)
+ {
+ os << "Buffer addr = " << static_cast<const void *>(buffer.data()) << ", size = " << buffer.size() << std::endl;
+
+ if (buffer.size() == 0) {
+ os << "Buffer is empty" << std::endl;
+ return;
+ }
+
+ size_t range_start = 0;
+ uint8_t current_value = buffer[0];
+ for (size_t i = 1; i < buffer.size(); ++i) {
+ if (buffer[i] != current_value) {
+ print_range(range_start, i, current_value, os);
+ current_value = buffer[i];
+ range_start = i;
+ }
+ }
+
+ // Print the last range
+ print_range(range_start, buffer.size(), current_value, os);
+ }
+
+private:
+ static void print_range(size_t range_start, size_t range_end_exclusive, uint8_t value, std::ostream& os)
+ {
+ const auto message = fmt::format("[0x{:08X}:0x{:08X}] - 0x{:02X} ({} bytes)",
+ range_start, range_end_exclusive - 1, static_cast<int>(value), range_end_exclusive - range_start);
+ os << message << std::endl;
+ }
+};
+
} /* namespace hailort */
#endif /* HAILO_UTILS_H_ */
\ No newline at end of file
#ifndef _HAILO_IOCTL_COMMON_H_
#define _HAILO_IOCTL_COMMON_H_
+#define HAILO_DRV_VER_MAJOR 4
+#define HAILO_DRV_VER_MINOR 18
+#define HAILO_DRV_VER_REVISION 0
+
+#define _STRINGIFY_EXPANDED( x ) #x
+#define _STRINGIFY_NUMBER( x ) _STRINGIFY_EXPANDED(x)
+#define HAILO_DRV_VER _STRINGIFY_NUMBER(HAILO_DRV_VER_MAJOR) "." _STRINGIFY_NUMBER(HAILO_DRV_VER_MINOR) "." _STRINGIFY_NUMBER(HAILO_DRV_VER_REVISION)
+
// This value is not easily changeable.
// For example: the channel interrupts ioctls assume we have up to 32 channels
#define INVALID_DRIVER_HANDLE_VALUE ((uintptr_t)-1)
// Used by windows and unix driver to raise the right CPU control handle to the FW. The same as in pcie_service FW
-#define FW_ACCESS_CORE_CPU_CONTROL_SHIFT (1)
-#define FW_ACCESS_CORE_CPU_CONTROL_MASK (1 << FW_ACCESS_CORE_CPU_CONTROL_SHIFT)
-#define FW_ACCESS_CONTROL_INTERRUPT_SHIFT (0)
-#define FW_ACCESS_APP_CPU_CONTROL_MASK (1 << FW_ACCESS_CONTROL_INTERRUPT_SHIFT)
-#define FW_ACCESS_DRIVER_SHUTDOWN_SHIFT (2)
-#define FW_ACCESS_DRIVER_SHUTDOWN_MASK (1 << FW_ACCESS_DRIVER_SHUTDOWN_SHIFT)
+#define FW_ACCESS_CORE_CPU_CONTROL_SHIFT (1)
+#define FW_ACCESS_CORE_CPU_CONTROL_MASK (1 << FW_ACCESS_CORE_CPU_CONTROL_SHIFT)
+#define FW_ACCESS_CONTROL_INTERRUPT_SHIFT (0)
+#define FW_ACCESS_APP_CPU_CONTROL_MASK (1 << FW_ACCESS_CONTROL_INTERRUPT_SHIFT)
+#define FW_ACCESS_DRIVER_SHUTDOWN_SHIFT (2)
+#define FW_ACCESS_DRIVER_SHUTDOWN_MASK (1 << FW_ACCESS_DRIVER_SHUTDOWN_SHIFT)
+#define FW_ACCESS_SOC_CONNECT_SHIFT (3)
+#define FW_ACCESS_SOC_CONNECT_MASK (1 << FW_ACCESS_SOC_CONNECT_SHIFT)
+
+#define INVALID_VDMA_CHANNEL (0xff)
-#define INVALID_VDMA_CHANNEL (0xff)
#if !defined(__cplusplus) && defined(NTDDI_VERSION)
#include <wdm.h>
#define INT_MAX 0x7FFFFFFF
#endif // !defined(INT_MAX)
+#if !defined(ECONNRESET)
+#define ECONNRESET 104 /* Connection reset by peer */
+#endif // !defined(ECONNRESET)
// {d88d31f1-fede-4e71-ac2a-6ce0018c1501}
-DEFINE_GUID (GUID_DEVINTERFACE_HailoKM,
+DEFINE_GUID (GUID_DEVINTERFACE_HailoKM_NNC,
0xd88d31f1,0xfede,0x4e71,0xac,0x2a,0x6c,0xe0,0x01,0x8c,0x15,0x01);
-#define HAILO_GENERAL_IOCTL_MAGIC 0
-#define HAILO_VDMA_IOCTL_MAGIC 1
-#define HAILO_NON_LINUX_IOCTL_MAGIC 2
+// {7f16047d-64b8-207a-0092-e970893970a2}
+DEFINE_GUID (GUID_DEVINTERFACE_HailoKM_SOC,
+ 0x7f16047d,0x64b8,0x207a,0x00,0x92,0xe9,0x70,0x89,0x39,0x70,0xa2);
+
+#define HAILO_GENERAL_IOCTL_MAGIC 0
+#define HAILO_VDMA_IOCTL_MAGIC 1
+#define HAILO_SOC_IOCTL_MAGIC 2
+#define HAILO_PCI_EP_IOCTL_MAGIC 3
+#define HAILO_NNC_IOCTL_MAGIC 4
#define HAILO_IOCTL_COMPATIBLE CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_ANY_ACCESS)
#define _IOWR_ _IOWR
#define _IO_ _IO
-#define HAILO_GENERAL_IOCTL_MAGIC 'g'
-#define HAILO_VDMA_IOCTL_MAGIC 'v'
-#define HAILO_NON_LINUX_IOCTL_MAGIC 'w'
+#define HAILO_GENERAL_IOCTL_MAGIC 'g'
+#define HAILO_VDMA_IOCTL_MAGIC 'v'
+#define HAILO_SOC_IOCTL_MAGIC 's'
+#define HAILO_NNC_IOCTL_MAGIC 'n'
+#define HAILO_PCI_EP_IOCTL_MAGIC 'p'
#elif defined(__QNX__) // #ifdef _MSC_VER
#include <devctl.h>
#define _IO_ __DION
#define HAILO_GENERAL_IOCTL_MAGIC _DCMD_ALL
#define HAILO_VDMA_IOCTL_MAGIC _DCMD_MISC
-#define HAILO_NON_LINUX_IOCTL_MAGIC _DCMD_PROC
#else // #ifdef _MSC_VER
#error "unsupported platform!"
HAILO_DMA_MAX_ENUM = INT_MAX,
};
+// Enum that states what type of buffer we are working with in the driver
+// TODO: HRT-13580 - Add specific type for user allocated and for driver allocated
+enum hailo_dma_buffer_type {
+ HAILO_DMA_USER_PTR_BUFFER = 0,
+ HAILO_DMA_DMABUF_BUFFER = 1,
+
+ /** Max enum value to maintain ABI Integrity */
+ HAILO_DMA_BUFFER_MAX_ENUM = INT_MAX,
+};
+
// Enum that determines if buffer should be allocated from user space or from driver
enum hailo_allocation_mode {
HAILO_ALLOCATION_MODE_USERSPACE = 0,
HAILO_ALLOCATION_MODE_MAX_ENUM = INT_MAX,
};
+enum hailo_vdma_interrupts_domain {
+ HAILO_VDMA_INTERRUPTS_DOMAIN_NONE = 0,
+ HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE = (1 << 0),
+ HAILO_VDMA_INTERRUPTS_DOMAIN_HOST = (1 << 1),
+
+ /** Max enum value to maintain ABI Integrity */
+ HAILO_VDMA_INTERRUPTS_DOMAIN_MAX_ENUM = INT_MAX,
+};
+
/* structure used in ioctl HAILO_VDMA_BUFFER_MAP */
struct hailo_vdma_buffer_map_params {
#if defined(__linux__) || defined(_MSC_VER)
- void* user_address; // in
+ uintptr_t user_address; // in
#elif defined(__QNX__)
shm_handle_t shared_memory_handle; // in
#else
#endif // __linux__
size_t size; // in
enum hailo_dma_data_direction data_direction; // in
+ enum hailo_dma_buffer_type buffer_type; // in
uintptr_t allocated_buffer_handle; // in
size_t mapped_handle; // out
};
uintptr_t desc_handle; // in
};
-/* structure used in ioctl HAILO_NON_LINUX_DESC_LIST_MMAP */
-struct hailo_non_linux_desc_list_mmap_params {
- uintptr_t desc_handle; // in
- size_t size; // in
- void* user_address; // out
-};
-
/* structure used in ioctl HAILO_DESC_LIST_BIND_VDMA_BUFFER */
-struct hailo_desc_list_bind_vdma_buffer_params {
+struct hailo_desc_list_program_params {
size_t buffer_handle; // in
size_t buffer_size; // in
size_t buffer_offset; // in
uintptr_t desc_handle; // in
uint8_t channel_index; // in
uint32_t starting_desc; // in
+ bool should_bind; // in
+ enum hailo_vdma_interrupts_domain last_interrupts_domain; // in
+ bool is_debug; // in
};
-/* structure used in ioctl HAILO_VDMA_INTERRUPTS_ENABLE */
-struct hailo_vdma_interrupts_enable_params {
+/* structure used in ioctl HAILO_VDMA_ENABLE_CHANNELS */
+struct hailo_vdma_enable_channels_params {
uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in
bool enable_timestamps_measure; // in
};
-/* structure used in ioctl HAILO_VDMA_INTERRUPTS_DISABLE */
-struct hailo_vdma_interrupts_disable_params {
+/* structure used in ioctl HAILO_VDMA_DISABLE_CHANNELS */
+struct hailo_vdma_disable_channels_params {
uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in
};
uint8_t engine_index;
uint8_t channel_index;
bool is_active; // If not activate, num_processed is ignored.
- uint16_t host_num_processed;
+ uint8_t transfers_completed; // Number of transfers completed.
uint8_t host_error; // Channel errors bits on source side
uint8_t device_error; // Channel errors bits on dest side
bool validation_success; // If the validation of the channel was successful
HAILO_TRANSFER_MEMORY_DMA_ENGINE1,
HAILO_TRANSFER_MEMORY_DMA_ENGINE2,
+ // PCIe EP driver memories
+ HAILO_TRANSFER_MEMORY_PCIE_EP_CONFIG = 0x400,
+ HAILO_TRANSFER_MEMORY_PCIE_EP_BRIDGE,
+
/** Max enum value to maintain ABI Integrity */
HAILO_TRANSFER_MEMORY_MAX_ENUM = INT_MAX,
};
HAILO_BOARD_TYPE_HAILO8 = 0,
HAILO_BOARD_TYPE_HAILO15,
HAILO_BOARD_TYPE_PLUTO,
+ HAILO_BOARD_TYPE_HAILO10H,
+ HAILO_BOARD_TYPE_HAILO10H_LEGACY,
HAILO_BOARD_TYPE_COUNT,
/** Max enum value to maintain ABI Integrity */
HAILO_BOARD_TYPE_MAX_ENUM = INT_MAX
};
+enum hailo_accelerator_type {
+ HAILO_ACCELERATOR_TYPE_NNC,
+ HAILO_ACCELERATOR_TYPE_SOC,
+
+ /** Max enum value to maintain ABI Integrity */
+ HAILO_ACCELERATOR_TYPE_MAX_ENUM = INT_MAX
+};
+
enum hailo_dma_type {
HAILO_DMA_TYPE_PCIE,
HAILO_DMA_TYPE_DRAM,
+ HAILO_DMA_TYPE_PCI_EP,
/** Max enum value to maintain ABI Integrity */
HAILO_DMA_TYPE_MAX_ENUM = INT_MAX,
uint32_t size; // in
};
-enum hailo_vdma_interrupts_domain {
- HAILO_VDMA_INTERRUPTS_DOMAIN_NONE = 0,
- HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE = (1 << 0),
- HAILO_VDMA_INTERRUPTS_DOMAIN_HOST = (1 << 1),
-
- /** Max enum value to maintain ABI Integrity */
- HAILO_VDMA_INTERRUPTS_DOMAIN_MAX_ENUM = INT_MAX,
-};
-
// We allow maximum 2 buffers per transfer since we may have an extra buffer
// to make sure each buffer is aligned to page size.
#define HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER (2)
// more info (e.g desc complete status)
uint32_t descs_programed; // out, amount of descriptors programed.
+ int launch_transfer_status; // out, status of the launch transfer call. (only used in case of error)
+};
+
+/* structure used in ioctl HAILO_SOC_CONNECT */
+struct hailo_soc_connect_params {
+ uint8_t input_channel_index; // out
+ uint8_t output_channel_index; // out
+ uintptr_t input_desc_handle; // in
+ uintptr_t output_desc_handle; // in
+};
+
+/* structure used in ioctl HAILO_SOC_CLOSE */
+struct hailo_soc_close_params {
+ uint8_t input_channel_index; // in
+ uint8_t output_channel_index; // in
+};
+
+/* structure used in ioctl HAILO_PCI_EP_ACCEPT */
+struct hailo_pci_ep_accept_params {
+ uint8_t input_channel_index; // out
+ uint8_t output_channel_index; // out
+ uintptr_t input_desc_handle; // in
+ uintptr_t output_desc_handle; // in
+};
+
+/* structure used in ioctl HAILO_PCI_EP_CLOSE */
+struct hailo_pci_ep_close_params {
+ uint8_t input_channel_index; // in
+ uint8_t output_channel_index; // in
};
#ifdef _MSC_VER
ULONG_PTR Value;
union {
struct hailo_memory_transfer_params MemoryTransfer;
- struct hailo_vdma_interrupts_enable_params VdmaInterruptsEnable;
- struct hailo_vdma_interrupts_disable_params VdmaInterruptsDisable;
+ struct hailo_vdma_enable_channels_params VdmaEnableChannels;
+ struct hailo_vdma_disable_channels_params VdmaDisableChannels;
struct hailo_vdma_interrupts_read_timestamp_params VdmaInterruptsReadTimestamps;
struct hailo_vdma_interrupts_wait_params VdmaInterruptsWait;
struct hailo_vdma_buffer_sync_params VdmaBufferSync;
struct hailo_vdma_buffer_unmap_params VdmaBufferUnmap;
struct hailo_desc_list_create_params DescListCreate;
struct hailo_desc_list_release_params DescListReleaseParam;
- struct hailo_desc_list_bind_vdma_buffer_params DescListBind;
+ struct hailo_desc_list_program_params DescListProgram;
struct hailo_d2h_notification D2HNotification;
struct hailo_device_properties DeviceProperties;
struct hailo_driver_info DriverInfo;
- struct hailo_non_linux_desc_list_mmap_params DescListMmap;
struct hailo_read_log_params ReadLog;
struct hailo_mark_as_in_use_params MarkAsInUse;
struct hailo_vdma_launch_transfer_params LaunchTransfer;
+ struct hailo_soc_connect_params ConnectParams;
+ struct hailo_soc_close_params SocCloseParams;
+ struct hailo_pci_ep_accept_params AcceptParams;
+ struct hailo_pci_ep_close_params PciEpCloseParams;
} Buffer;
};
#endif // _MSC_VER
enum hailo_general_ioctl_code {
HAILO_MEMORY_TRANSFER_CODE,
- HAILO_FW_CONTROL_CODE,
- HAILO_READ_NOTIFICATION_CODE,
- HAILO_DISABLE_NOTIFICATION_CODE,
HAILO_QUERY_DEVICE_PROPERTIES_CODE,
HAILO_QUERY_DRIVER_INFO_CODE,
- HAILO_READ_LOG_CODE,
- HAILO_RESET_NN_CORE_CODE,
// Must be last
HAILO_GENERAL_IOCTL_MAX_NR,
};
#define HAILO_MEMORY_TRANSFER _IOWR_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_MEMORY_TRANSFER_CODE, struct hailo_memory_transfer_params)
-#define HAILO_FW_CONTROL _IOWR_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_FW_CONTROL_CODE, struct hailo_fw_control)
-#define HAILO_READ_NOTIFICATION _IOW_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_READ_NOTIFICATION_CODE, struct hailo_d2h_notification)
-#define HAILO_DISABLE_NOTIFICATION _IO_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_DISABLE_NOTIFICATION_CODE)
#define HAILO_QUERY_DEVICE_PROPERTIES _IOW_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_QUERY_DEVICE_PROPERTIES_CODE, struct hailo_device_properties)
#define HAILO_QUERY_DRIVER_INFO _IOW_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_QUERY_DRIVER_INFO_CODE, struct hailo_driver_info)
-#define HAILO_READ_LOG _IOWR_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_READ_LOG_CODE, struct hailo_read_log_params)
-#define HAILO_RESET_NN_CORE _IO_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_RESET_NN_CORE_CODE)
enum hailo_vdma_ioctl_code {
- HAILO_VDMA_INTERRUPTS_ENABLE_CODE,
- HAILO_VDMA_INTERRUPTS_DISABLE_CODE,
+ HAILO_VDMA_ENABLE_CHANNELS_CODE,
+ HAILO_VDMA_DISABLE_CHANNELS_CODE,
HAILO_VDMA_INTERRUPTS_WAIT_CODE,
HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE,
HAILO_VDMA_BUFFER_MAP_CODE,
HAILO_VDMA_BUFFER_SYNC_CODE,
HAILO_DESC_LIST_CREATE_CODE,
HAILO_DESC_LIST_RELEASE_CODE,
- HAILO_DESC_LIST_BIND_VDMA_BUFFER_CODE,
+ HAILO_DESC_LIST_PROGRAM_CODE,
HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE,
HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE,
HAILO_MARK_AS_IN_USE_CODE,
HAILO_VDMA_IOCTL_MAX_NR,
};
-#define HAILO_VDMA_INTERRUPTS_ENABLE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_ENABLE_CODE, struct hailo_vdma_interrupts_enable_params)
-#define HAILO_VDMA_INTERRUPTS_DISABLE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_DISABLE_CODE, struct hailo_vdma_interrupts_disable_params)
+#define HAILO_VDMA_ENABLE_CHANNELS _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_ENABLE_CHANNELS_CODE, struct hailo_vdma_enable_channels_params)
+#define HAILO_VDMA_DISABLE_CHANNELS _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_DISABLE_CHANNELS_CODE, struct hailo_vdma_disable_channels_params)
#define HAILO_VDMA_INTERRUPTS_WAIT _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_WAIT_CODE, struct hailo_vdma_interrupts_wait_params)
#define HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE, struct hailo_vdma_interrupts_read_timestamp_params)
-#define HAILO_VDMA_BUFFER_MAP _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_MAP_CODE, struct hailo_vdma_buffer_map_params)
-#define HAILO_VDMA_BUFFER_UNMAP _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_UNMAP_CODE, struct hailo_vdma_buffer_unmap_params)
-#define HAILO_VDMA_BUFFER_SYNC _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_SYNC_CODE, struct hailo_vdma_buffer_sync_params)
+#define HAILO_VDMA_BUFFER_MAP _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_MAP_CODE, struct hailo_vdma_buffer_map_params)
+#define HAILO_VDMA_BUFFER_UNMAP _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_UNMAP_CODE, struct hailo_vdma_buffer_unmap_params)
+#define HAILO_VDMA_BUFFER_SYNC _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_SYNC_CODE, struct hailo_vdma_buffer_sync_params)
-#define HAILO_DESC_LIST_CREATE _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_CREATE_CODE, struct hailo_desc_list_create_params)
-#define HAILO_DESC_LIST_RELEASE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_RELEASE_CODE, struct hailo_desc_list_release_params)
-#define HAILO_DESC_LIST_BIND_VDMA_BUFFER _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_BIND_VDMA_BUFFER_CODE, struct hailo_desc_list_bind_vdma_buffer_params)
+#define HAILO_DESC_LIST_CREATE _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_CREATE_CODE, struct hailo_desc_list_create_params)
+#define HAILO_DESC_LIST_RELEASE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_RELEASE_CODE, struct hailo_desc_list_release_params)
+#define HAILO_DESC_LIST_PROGRAM _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_PROGRAM_CODE, struct hailo_desc_list_program_params)
-#define HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE, struct hailo_allocate_low_memory_buffer_params)
-#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE, struct hailo_free_low_memory_buffer_params)
+#define HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE, struct hailo_allocate_low_memory_buffer_params)
+#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE, struct hailo_free_low_memory_buffer_params)
-#define HAILO_MARK_AS_IN_USE _IOW_(HAILO_VDMA_IOCTL_MAGIC, HAILO_MARK_AS_IN_USE_CODE, struct hailo_mark_as_in_use_params)
+#define HAILO_MARK_AS_IN_USE _IOW_(HAILO_VDMA_IOCTL_MAGIC, HAILO_MARK_AS_IN_USE_CODE, struct hailo_mark_as_in_use_params)
-#define HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE, struct hailo_allocate_continuous_buffer_params)
-#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE, struct hailo_free_continuous_buffer_params)
+#define HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE, struct hailo_allocate_continuous_buffer_params)
+#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE, struct hailo_free_continuous_buffer_params)
-#define HAILO_VDMA_LAUNCH_TRANSFER _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LAUNCH_TRANSFER_CODE, struct hailo_vdma_launch_transfer_params)
+#define HAILO_VDMA_LAUNCH_TRANSFER _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LAUNCH_TRANSFER_CODE, struct hailo_vdma_launch_transfer_params)
+enum hailo_nnc_ioctl_code {
+ HAILO_FW_CONTROL_CODE,
+ HAILO_READ_NOTIFICATION_CODE,
+ HAILO_DISABLE_NOTIFICATION_CODE,
+ HAILO_READ_LOG_CODE,
+ HAILO_RESET_NN_CORE_CODE,
-enum hailo_non_linux_ioctl_code {
- HAILO_NON_LINUX_DESC_LIST_MMAP_CODE,
+ // Must be last
+ HAILO_NNC_IOCTL_MAX_NR
+};
+
+#define HAILO_FW_CONTROL _IOWR_(HAILO_NNC_IOCTL_MAGIC, HAILO_FW_CONTROL_CODE, struct hailo_fw_control)
+#define HAILO_READ_NOTIFICATION _IOW_(HAILO_NNC_IOCTL_MAGIC, HAILO_READ_NOTIFICATION_CODE, struct hailo_d2h_notification)
+#define HAILO_DISABLE_NOTIFICATION _IO_(HAILO_NNC_IOCTL_MAGIC, HAILO_DISABLE_NOTIFICATION_CODE)
+#define HAILO_READ_LOG _IOWR_(HAILO_NNC_IOCTL_MAGIC, HAILO_READ_LOG_CODE, struct hailo_read_log_params)
+#define HAILO_RESET_NN_CORE _IO_(HAILO_NNC_IOCTL_MAGIC, HAILO_RESET_NN_CORE_CODE)
+
+enum hailo_soc_ioctl_code {
+ HAILO_SOC_IOCTL_CONNECT_CODE,
+ HAILO_SOC_IOCTL_CLOSE_CODE,
// Must be last
- HAILO_NON_LINUX_IOCTL_MAX_NR,
+ HAILO_SOC_IOCTL_MAX_NR,
};
-#define HAILO_NON_LINUX_DESC_LIST_MMAP _IOWR_(HAILO_NON_LINUX_IOCTL_MAGIC, HAILO_NON_LINUX_DESC_LIST_MMAP_CODE, struct hailo_non_linux_desc_list_mmap_params)
+#define HAILO_SOC_CONNECT _IOWR_(HAILO_SOC_IOCTL_MAGIC, HAILO_SOC_IOCTL_CONNECT_CODE, struct hailo_soc_connect_params)
+#define HAILO_SOC_CLOSE _IOR_(HAILO_SOC_IOCTL_MAGIC, HAILO_SOC_IOCTL_CLOSE_CODE, struct hailo_soc_close_params)
+
+
+enum hailo_pci_ep_ioctl_code {
+ HAILO_PCI_EP_ACCEPT_CODE,
+ HAILO_PCI_EP_CLOSE_CODE,
+
+ // Must be last
+ HAILO_PCI_EP_IOCTL_MAX_NR,
+};
+#define HAILO_PCI_EP_ACCEPT _IOWR_(HAILO_PCI_EP_IOCTL_MAGIC, HAILO_PCI_EP_ACCEPT_CODE, struct hailo_pci_ep_accept_params)
+#define HAILO_PCI_EP_CLOSE _IOR_(HAILO_PCI_EP_IOCTL_MAGIC, HAILO_PCI_EP_CLOSE_CODE, struct hailo_pci_ep_close_params)
#endif /* _HAILO_IOCTL_COMMON_H_ */
+++ /dev/null
-#ifndef _HAILO_PCIE_VERSION_H_
-#define _HAILO_PCIE_VERSION_H_
-
-#include "..\..\common\hailo_pcie_version.h"
-
-#define STRINGIFY_EXPANDED( x ) #x
-#define STRINGIFY_NUMBER( x ) STRINGIFY_EXPANDED(x)
-#define HAILO_DRV_VER STRINGIFY_NUMBER(HAILO_DRV_VER_MAJOR) "." STRINGIFY_NUMBER(HAILO_DRV_VER_MINOR) "." STRINGIFY_NUMBER(HAILO_DRV_VER_REVISION)
-
-#endif /* _HAILO_PCIE_VERSION_H_ */
--- /dev/null
+Copyright 2024 (C) Hailo Technologies Ltd. ("Hailo")
+All rights reserved.
+
+By downloading, installing, copying, or otherwise using the software,
+you agree to be bound by the terms of this license.
+
+Hailo hereby grants You a limited, non-exclusive, non-assignable, non-transferable, revocable, non-sublicensable, license to (a) use the Licensed Software, (b) Redistribution of this software in its binary form only, both (a) and (b) solely: (i) according to the Documentation and this license ; (ii) for the purpose of developing applications for use in Your system(s) ("End-User Product(s)), and in conjunction with Hailo's proprietary products previously purchased by You from Hailo ("Products") which are utilizing the Licensed Software;
+
+Restrictions. You agree that except as expressly permitted by Hailo under this license, you will not, nor allow any third party on Your behalf to (a) license, rent, lease, sublicense, loan, sell or otherwise allow any access the Licensed Software; (b) modify, alter, copy, transfer, emulate or create any derivative works of the Licensed Software or of any part thereof; (c) reverse engineer, decompile, decode, decrypt, disassemble, or in any way attempt to derive source code, designs, or otherwise discover the underlying Intellectual Property or technology of the Licensed Software or any part thereof; (d) remove, alter or obscure any copyright, trademark or other proprietary rights notice, on or in, the Licensed Software and/or the Documentation; (e) use the Licensed Software for any benchmarking to be publicly published or for competing development activities, (f) distribute without this license and/or not in the Licensed software binary form (g) publish or disclose to any third party any technical features, quality, performance or benchmark test, or comparative analyses relating to the Licensed Software ("Technical Data"), or (h) utilize the Licensed Software with any product(s) other than End User Products in conjunction with the Product(s) as described hereunder and under the Documentation, or (i) use the Licensed Software in any manner that would cause the Licensed Software and/or the Product to become subject to an Open Source License. "Open Source License" includes, without limitation, a software license that requires as a condition of use, modification, and/or distribution, of such software that the Licensed Software be (1) disclosed or distributed in source code form; (2) be licensed under other and/or terms other than this license;
+
+Hailo Technologies Ltd. disclaims any warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose.
+This software is provided on an "AS IS" basis, and Hailo has no obligation to provide maintenance, support, updates, enhancements, or modifications
--- /dev/null
+cmake_minimum_required(VERSION 3.0.0)
+
+include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/spdlog.cmake)
+include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/readerwriterqueue.cmake)
+
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
+set(HAILORT_SERVER_SOURCES
+ hailort_server.cpp
+ ${HRPC_CPP_SOURCES}
+ ${HRPC_PROTOCOL_CPP_SOURCES}
+ ${HAILORT_COMMON_OS_DIR}/os_utils.cpp
+ ${HAILORT_SERVICE_DIR}/cng_buffer_pool.cpp
+ ${HAILORT_COMMON_DIR}/common/event_internal.cpp
+ ${HAILO_FULL_OS_DIR}/event.cpp # TODO HRT-10681: move to common
+ ${DRIVER_OS_DIR}/driver_os_specific.cpp
+ ${HAILO_OS_DIR}/file_descriptor.cpp
+ ${HAILO_OS_DIR}/mmap_buffer.cpp
+ ${HAILORT_SRC_DIR}/vdma/pcie_session.cpp
+ ${HAILORT_SRC_DIR}/vdma/memory/descriptor_list.cpp
+ ${HAILORT_SRC_DIR}/vdma/memory/mapped_buffer.cpp
+ ${HAILORT_SRC_DIR}/vdma/memory/dma_able_buffer.cpp
+ ${HAILORT_SRC_DIR}/vdma/memory/vdma_edge_layer.cpp
+ ${HAILORT_SRC_DIR}/vdma/driver/hailort_driver.cpp
+ ${HAILORT_SRC_DIR}/vdma/channel/interrupts_dispatcher.cpp
+ ${HAILORT_SRC_DIR}/vdma/channel/transfer_launcher.cpp
+ ${HAILORT_SRC_DIR}/vdma/channel/boundary_channel.cpp
+ ${HAILORT_SRC_DIR}/vdma/channel/channels_group.cpp
+ ${HAILORT_SRC_DIR}/stream_common/transfer_common.cpp
+)
+if(WIN32)
+ # hailort_driver.cpp in windows depends on string_conversion
+ # dma_able_buffer.cpp in windows depends on virtual_alloc_guard
+ set(HAILORT_SERVER_SOURCES ${HAILORT_SERVER_SOURCES}
+ ${HAILORT_COMMON_OS_DIR}/string_conversion.cpp
+ ${HAILO_FULL_OS_DIR}/virtual_alloc_guard.cpp)
+endif()
+
+add_executable(hailort_server ${HAILORT_SERVER_SOURCES})
+target_include_directories(hailort_server PRIVATE
+ ${HAILORT_SRC_DIR}
+ ${COMMON_INC_DIR}
+ ${DRIVER_INC_DIR}
+)
+target_compile_options(hailort_server PRIVATE ${HAILORT_COMPILE_OPTIONS})
+set_property(TARGET hailort_server PROPERTY CXX_STANDARD 14)
+set_property(TARGET hailort_server PROPERTY INSTALL_RPATH "$ORIGIN" "../lib/") # Link with a relative libhailort
+target_link_libraries(hailort_server PRIVATE
+ libhailort
+ Threads::Threads
+ rpc_proto
+ spdlog::spdlog
+ readerwriterqueue
+)
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file hailo_server.cpp
+ * @brief Hailo Server
+ **/
+
+#include "hailort_server.hpp"
+#include "hrpc/server.hpp"
+#include "hailo/vdevice.hpp"
+#include "hailo/infer_model.hpp"
+#include "hrpc_protocol/serializer.hpp"
+#include "net_flow/ops/nms_post_process.hpp"
+#include "hailort_service/service_resource_manager.hpp"
+
+#include <spdlog/spdlog.h>
+#include <spdlog/sinks/stdout_color_sinks.h>
+
+using namespace hailort;
+
+// TODO: These macros should be merged with the grpc macros, also change them to TRY
+#define CHECK_EXPECTED_AS_HRPC_STATUS(_exepcted, T) \
+ do { \
+ if (!_exepcted) { \
+ LOGGER__ERROR("CHECK_EXPECTED_AS_HRPC_STATUS failed, status: {}", _exepcted.status()); \
+ auto reply = T::serialize_reply(_exepcted.status()); \
+ if (reply) return reply; \
+ LOGGER__CRITICAL("Failed to create reply with status: {}", reply.status()); \
+ return make_unexpected(HAILO_INTERNAL_FAILURE); \
+ } \
+ } while (0)
+#define CHECK_SUCCESS_AS_HRPC_STATUS(_status, T) \
+ do { \
+ if (_status != HAILO_SUCCESS) { \
+ LOGGER__ERROR("CHECK_SUCCESS_AS_HRPC_STATUS failed, status: {}", _status); \
+ auto reply = T::serialize_reply(_status); \
+ if (reply) return reply; \
+ LOGGER__CRITICAL("Failed to create reply with status: {}", reply.status()); \
+ return make_unexpected(HAILO_INTERNAL_FAILURE); \
+ } \
+ } while (0)
+
+#define __HAILO_CONCAT(x, y) x ## y
+#define _HAILO_CONCAT(x, y) __HAILO_CONCAT(x, y)
+
+#define _TRY_AS_HRPC_STATUS(expected_var_name, var_decl, expr, ...) \
+ auto expected_var_name = (expr); \
+ CHECK_EXPECTED_AS_HRPC_STATUS(expected_var_name, __VA_ARGS__); \
+ var_decl = expected_var_name.release()
+
+#define TRY_AS_HRPC_STATUS(var_decl, expr, ...) _TRY_AS_HRPC_STATUS(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__)
+
+#ifdef NDEBUG
+#define LOGGER_PATTERN ("[%n] [%^%l%$] %v")
+#else
+#define LOGGER_PATTERN ("[%Y-%m-%d %X.%e] [%P] [%t] [%n] [%^%l%$] [%s:%#] [%!] %v")
+#endif
+#define BUFFER_POOL_SIZE (10) // TODO: this may hurt performance, should be configurable
+
+struct InferModelInfo
+{
+ std::unordered_map<std::string, size_t> input_streams_sizes;
+ std::unordered_map<std::string, size_t> output_streams_sizes;
+ std::vector<std::string> inputs_names;
+ std::vector<std::string> outputs_names;
+};
+
+void init_logger(const std::string &name)
+{
+ auto console_sink = make_shared_nothrow<spdlog::sinks::stderr_color_sink_mt>();
+ console_sink->set_level(spdlog::level::info);
+ console_sink->set_pattern(LOGGER_PATTERN);
+ spdlog::set_default_logger(make_shared_nothrow<spdlog::logger>(name, console_sink));
+}
+
+void hrpc::HailoRTServer::cleanup_infer_model_hef_buffers(const std::vector<uint32_t> &infer_model_handles)
+{
+ for (const auto &infer_model_handle : infer_model_handles) {
+ auto hef_buffers_iter = m_hef_buffers_per_infer_model.find(infer_model_handle);
+ if (m_hef_buffers_per_infer_model.end() != hef_buffers_iter) {
+ m_hef_buffers_per_infer_model.erase(infer_model_handle);
+ }
+ }
+}
+
+void hrpc::HailoRTServer::cleanup_cim_buffer_pools(const std::vector<uint32_t> &cim_handles)
+{
+ for (const auto &cim_handle : cim_handles) {
+ auto buffer_pool_iter = m_buffer_pool_per_cim.find(cim_handle);
+ if (m_buffer_pool_per_cim.end() != buffer_pool_iter) {
+ m_buffer_pool_per_cim.erase(cim_handle);
+ }
+ }
+}
+
+hailo_status hrpc::HailoRTServer::cleanup_client_resources(RpcConnection client_connection)
+{
+ std::set<uint32_t> pids = {SINGLE_CLIENT_PID};
+ auto cim_handles = ServiceResourceManager<ConfiguredInferModel>::get_instance().resources_handles_by_pids(pids);
+ (void)ServiceResourceManager<ConfiguredInferModel>::get_instance().release_by_pid(SINGLE_CLIENT_PID);
+ cleanup_cim_buffer_pools(cim_handles);
+
+ auto infer_model_handles = ServiceResourceManager<InferModel>::get_instance().resources_handles_by_pids(pids);
+ (void)ServiceResourceManager<InferModelInfo>::get_instance().release_by_pid(SINGLE_CLIENT_PID);
+ (void)ServiceResourceManager<InferModel>::get_instance().release_by_pid(SINGLE_CLIENT_PID);
+ cleanup_infer_model_hef_buffers(infer_model_handles);
+ m_infer_model_to_info_id.clear();
+
+ (void)ServiceResourceManager<VDevice>::get_instance().release_by_pid(SINGLE_CLIENT_PID);
+ CHECK_SUCCESS(client_connection.close());
+ return HAILO_SUCCESS;
+}
+
+Expected<std::unique_ptr<hrpc::HailoRTServer>> hrpc::HailoRTServer::create_unique()
+{
+ TRY(auto connection_context, ConnectionContext::create_shared(true));
+ auto res = make_unique_nothrow<HailoRTServer>(connection_context);
+ CHECK_NOT_NULL(res, HAILO_OUT_OF_HOST_MEMORY);
+ return res;
+}
+
+int main()
+{
+ init_logger("HailoRT-Server");
+ TRY(auto server, hrpc::HailoRTServer::create_unique());
+ hrpc::Dispatcher dispatcher;
+
+ // TODO: add a server implementation class, with resources heiracrhy and more
+ auto &infer_model_to_info_id = server->get_infer_model_to_info_id();
+ auto &buffer_pool_per_cim = server->get_buffer_pool_per_cim();
+
+ // Because the infer model is created with a hef buffer, we need to keep the buffer until the configure stage.
+ // Here I keep it until the infer model is destroyed
+ auto &hef_buffers = server->get_hef_buffers();
+
+ dispatcher.register_action(HailoRpcActionID::VDEVICE__CREATE,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ TRY_AS_HRPC_STATUS(auto vdevice_params, CreateVDeviceSerializer::deserialize_request(request), CreateVDeviceSerializer);
+ TRY_AS_HRPC_STATUS(auto vdevice, VDevice::create(vdevice_params), CreateVDeviceSerializer);
+
+ auto &manager = ServiceResourceManager<VDevice>::get_instance();
+ auto id = manager.register_resource(SINGLE_CLIENT_PID, std::move(vdevice));
+ auto reply = CreateVDeviceSerializer::serialize_reply(HAILO_SUCCESS, id);
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::VDEVICE__DESTROY,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &manager = ServiceResourceManager<VDevice>::get_instance();
+ TRY_AS_HRPC_STATUS(auto vdevice_handle, DestroyVDeviceSerializer::deserialize_request(request), DestroyVDeviceSerializer);
+ (void)manager.release_resource(vdevice_handle, SINGLE_CLIENT_PID);
+ TRY_AS_HRPC_STATUS(auto reply, DestroyVDeviceSerializer::serialize_reply(HAILO_SUCCESS), DestroyVDeviceSerializer);
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::VDEVICE__CREATE_INFER_MODEL,
+ [&hef_buffers] (const MemoryView &request, hrpc::ServerContextPtr server_context) -> Expected<Buffer> {
+ TRY_AS_HRPC_STATUS(auto tuple, CreateInferModelSerializer::deserialize_request(request), CreateInferModelSerializer);
+ auto vdevice_handle = std::get<0>(tuple);
+ uint64_t hef_size = std::get<1>(tuple);
+
+ assert(hef_size <= SIZE_MAX);
+ TRY_AS_HRPC_STATUS(auto hef_buffer, Buffer::create(static_cast<size_t>(hef_size), BufferStorageParams::create_dma()), CreateInferModelSerializer);
+
+ auto status = server_context->connection().read_buffer(MemoryView(hef_buffer));
+ CHECK_SUCCESS_AS_HRPC_STATUS(status, CreateInferModelSerializer);
+
+ auto &vdevice_manager = ServiceResourceManager<VDevice>::get_instance();
+ auto lambda = [view = MemoryView(hef_buffer)] (std::shared_ptr<VDevice> vdevice) {
+ return vdevice->create_infer_model(view);
+ };
+ auto infer_model = vdevice_manager.execute<Expected<std::shared_ptr<InferModel>>>(vdevice_handle, lambda);
+ CHECK_EXPECTED_AS_HRPC_STATUS(infer_model, CreateInferModelSerializer);
+
+ auto &infer_model_manager = ServiceResourceManager<InferModel>::get_instance();
+ auto infer_model_id = infer_model_manager.register_resource(SINGLE_CLIENT_PID, std::move(infer_model.release()));
+ hef_buffers.emplace(infer_model_id, std::move(hef_buffer));
+
+ TRY_AS_HRPC_STATUS(auto reply, CreateInferModelSerializer::serialize_reply(HAILO_SUCCESS, infer_model_id), CreateInferModelSerializer);
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::INFER_MODEL__DESTROY,
+ [&hef_buffers] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &manager = ServiceResourceManager<InferModel>::get_instance();
+ TRY_AS_HRPC_STATUS(auto infer_model_handle, DestroyInferModelSerializer::deserialize_request(request), DestroyInferModelSerializer);
+ hef_buffers.erase(infer_model_handle);
+ (void)manager.release_resource(infer_model_handle, SINGLE_CLIENT_PID);
+ TRY_AS_HRPC_STATUS(auto reply, DestroyInferModelSerializer::serialize_reply(HAILO_SUCCESS), DestroyInferModelSerializer);
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::INFER_MODEL__CREATE_CONFIGURED_INFER_MODEL,
+ [&buffer_pool_per_cim, &infer_model_to_info_id]
+ (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &infer_model_manager = ServiceResourceManager<InferModel>::get_instance();
+
+ TRY_AS_HRPC_STATUS(auto request_params, CreateConfiguredInferModelSerializer::deserialize_request(request), CreateConfiguredInferModelSerializer);
+ const auto &infer_model_handle = request_params.infer_model_handle;
+ const auto &vdevice_handle = request_params.vdevice_handle;
+
+ auto lambda = [&request_params] (std::shared_ptr<InferModel> infer_model) -> Expected<ConfiguredInferModel> {
+ const auto &input_streams_formats = request_params.input_streams_params;
+ const auto &output_streams_formats = request_params.output_streams_params;
+ for (const auto &input_stream_format : input_streams_formats) {
+ TRY(auto input, infer_model->input(input_stream_format.first));
+
+ input.set_format_order(static_cast<hailo_format_order_t>(input_stream_format.second.format_order));
+ input.set_format_type(static_cast<hailo_format_type_t>(input_stream_format.second.format_type));
+ if (INVALID_NMS_CONFIG != input_stream_format.second.nms_score_threshold) {
+ input.set_nms_score_threshold(input_stream_format.second.nms_score_threshold);
+ }
+ if (INVALID_NMS_CONFIG != input_stream_format.second.nms_iou_threshold) {
+ input.set_nms_iou_threshold(input_stream_format.second.nms_iou_threshold);
+ }
+ if (static_cast<uint32_t>(INVALID_NMS_CONFIG) != input_stream_format.second.nms_max_proposals_per_class) {
+ input.set_nms_max_proposals_per_class(input_stream_format.second.nms_max_proposals_per_class);
+ }
+ if (static_cast<uint32_t>(INVALID_NMS_CONFIG) != input_stream_format.second.nms_max_accumulated_mask_size) {
+ input.set_nms_max_accumulated_mask_size(input_stream_format.second.nms_max_accumulated_mask_size);
+ }
+ }
+
+ for (const auto &output_stream_format : output_streams_formats) {
+ TRY(auto output, infer_model->output(output_stream_format.first));
+ output.set_format_order(static_cast<hailo_format_order_t>(output_stream_format.second.format_order));
+ output.set_format_type(static_cast<hailo_format_type_t>(output_stream_format.second.format_type));
+ if (INVALID_NMS_CONFIG != output_stream_format.second.nms_score_threshold) {
+ output.set_nms_score_threshold(output_stream_format.second.nms_score_threshold);
+ }
+ if (INVALID_NMS_CONFIG != output_stream_format.second.nms_iou_threshold) {
+ output.set_nms_iou_threshold(output_stream_format.second.nms_iou_threshold);
+ }
+ if (static_cast<uint32_t>(INVALID_NMS_CONFIG) != output_stream_format.second.nms_max_proposals_per_class) {
+ output.set_nms_max_proposals_per_class(output_stream_format.second.nms_max_proposals_per_class);
+ }
+ if (static_cast<uint32_t>(INVALID_NMS_CONFIG) != output_stream_format.second.nms_max_accumulated_mask_size) {
+ output.set_nms_max_accumulated_mask_size(output_stream_format.second.nms_max_accumulated_mask_size);
+ }
+ }
+
+ infer_model->set_batch_size(request_params.batch_size);
+ infer_model->set_power_mode(request_params.power_mode);
+ infer_model->set_hw_latency_measurement_flags(request_params.latency_flag);
+
+ return infer_model->configure();
+ };
+
+ auto configured_infer_model = infer_model_manager.execute<Expected<ConfiguredInferModel>>(infer_model_handle, lambda);
+ CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model, CreateConfiguredInferModelSerializer);
+
+ TRY_AS_HRPC_STATUS(auto async_queue_size, configured_infer_model->get_async_queue_size(), CreateConfiguredInferModelSerializer);
+ auto set_model_info_lambda = [] (std::shared_ptr<InferModel> infer_model) -> Expected<std::shared_ptr<InferModelInfo>> {
+ auto infer_model_info = make_shared_nothrow<InferModelInfo>();
+ CHECK_NOT_NULL_AS_EXPECTED(infer_model_info, HAILO_OUT_OF_HOST_MEMORY);
+
+ for (const auto &input : infer_model->inputs()) {
+ infer_model_info->input_streams_sizes.emplace(input.name(), input.get_frame_size());
+ infer_model_info->inputs_names.push_back(input.name());
+ }
+ for (const auto &output : infer_model->outputs()) {
+ infer_model_info->output_streams_sizes.emplace(output.name(), output.get_frame_size());
+ infer_model_info->outputs_names.push_back(output.name());
+ }
+ return infer_model_info;
+ };
+ auto model_info = infer_model_manager.execute<Expected<std::shared_ptr<InferModelInfo>>>(infer_model_handle, set_model_info_lambda);
+ CHECK_EXPECTED_AS_HRPC_STATUS(model_info, CreateConfiguredInferModelSerializer);
+
+ auto &infer_model_infos_manager = ServiceResourceManager<InferModelInfo>::get_instance();
+ auto infer_model_info_id = infer_model_infos_manager.register_resource(SINGLE_CLIENT_PID, std::move(model_info.release()));
+
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+ auto cim_id = cim_manager.register_resource(SINGLE_CLIENT_PID,
+ std::move(make_shared_nothrow<ConfiguredInferModel>(configured_infer_model.release())));
+
+ auto buffer_pool = ServiceNetworkGroupBufferPool::create(vdevice_handle);
+ CHECK_EXPECTED_AS_HRPC_STATUS(buffer_pool, CreateConfiguredInferModelSerializer);
+
+ auto buffer_pool_ptr = buffer_pool.release();
+ auto get_infer_model_info_lambda = [] (std::shared_ptr<InferModelInfo> infer_model_info) {
+ return *infer_model_info;
+ };
+ auto infer_model_info = infer_model_infos_manager.execute<Expected<InferModelInfo>>(infer_model_info_id, get_infer_model_info_lambda);
+ CHECK_EXPECTED_AS_HRPC_STATUS(infer_model_info, CreateConfiguredInferModelSerializer);
+
+ for (const auto &input_name : infer_model_info->inputs_names) {
+ auto status = buffer_pool_ptr->allocate_pool(input_name, HAILO_DMA_BUFFER_DIRECTION_D2H,
+ infer_model_info->input_streams_sizes[input_name], BUFFER_POOL_SIZE);
+ CHECK_SUCCESS_AS_HRPC_STATUS(status, CreateConfiguredInferModelSerializer);
+ }
+ for (const auto &output_name : infer_model_info->outputs_names) {
+ auto status = buffer_pool_ptr->allocate_pool(output_name, HAILO_DMA_BUFFER_DIRECTION_H2D,
+ infer_model_info->output_streams_sizes[output_name], BUFFER_POOL_SIZE);
+ CHECK_SUCCESS_AS_HRPC_STATUS(status, CreateConfiguredInferModelSerializer);
+ }
+ buffer_pool_per_cim.emplace(cim_id, buffer_pool_ptr);
+
+ infer_model_to_info_id[infer_model_handle] = infer_model_info_id;
+ TRY_AS_HRPC_STATUS(auto reply,
+ CreateConfiguredInferModelSerializer::serialize_reply(HAILO_SUCCESS, cim_id, static_cast<uint32_t>(async_queue_size)),
+ CreateConfiguredInferModelSerializer);
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__DESTROY,
+ [&buffer_pool_per_cim] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+ TRY_AS_HRPC_STATUS(auto configured_infer_model_handle, DestroyConfiguredInferModelSerializer::deserialize_request(request), DestroyInferModelSerializer);
+
+ auto shutdown_lambda = [] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ configured_infer_model->shutdown();
+ return HAILO_SUCCESS;
+ };
+ manager.execute<hailo_status>(configured_infer_model_handle, shutdown_lambda);
+ buffer_pool_per_cim.erase(configured_infer_model_handle);
+ (void)manager.release_resource(configured_infer_model_handle, SINGLE_CLIENT_PID);
+ TRY_AS_HRPC_STATUS(auto reply, DestroyConfiguredInferModelSerializer::serialize_reply(HAILO_SUCCESS), DestroyInferModelSerializer);
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_TIMEOUT,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+ TRY_AS_HRPC_STATUS(auto tuple, SetSchedulerTimeoutSerializer::deserialize_request(request), SetSchedulerTimeoutSerializer);
+ const auto &configured_infer_model_handle = std::get<0>(tuple);
+ const auto &timeout = std::get<1>(tuple);
+ auto lambda = [timeout] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->set_scheduler_timeout(timeout);
+ };
+ auto status = cim_manager.execute<hailo_status>(configured_infer_model_handle, lambda);
+ TRY_AS_HRPC_STATUS(auto reply, SetSchedulerTimeoutSerializer::serialize_reply(status), SetSchedulerTimeoutSerializer);
+
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_THRESHOLD,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+ TRY_AS_HRPC_STATUS(auto tuple, SetSchedulerThresholdSerializer::deserialize_request(request), SetSchedulerThresholdSerializer);
+ const auto &configured_infer_model_handle = std::get<0>(tuple);
+ const auto &threshold = std::get<1>(tuple);
+ auto lambda = [threshold] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->set_scheduler_threshold(threshold);
+ };
+ auto status = cim_manager.execute<hailo_status>(configured_infer_model_handle, lambda);
+ TRY_AS_HRPC_STATUS(auto reply, SetSchedulerThresholdSerializer::serialize_reply(status), SetSchedulerThresholdSerializer);
+
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_PRIORITY,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+ TRY_AS_HRPC_STATUS(auto tuple, SetSchedulerPrioritySerializer::deserialize_request(request), SetSchedulerPrioritySerializer);
+ const auto &configured_infer_model_handle = std::get<0>(tuple);
+ const auto &priority = std::get<1>(tuple);
+ auto lambda = [priority] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->set_scheduler_priority(static_cast<uint8_t>(priority));
+ };
+ auto status = cim_manager.execute<hailo_status>(configured_infer_model_handle, lambda);
+ TRY_AS_HRPC_STATUS(auto reply, SetSchedulerPrioritySerializer::serialize_reply(status), SetSchedulerPrioritySerializer);
+
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__GET_HW_LATENCY_MEASUREMENT,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+
+ auto configured_infer_model_handle = GetHwLatencyMeasurementSerializer::deserialize_request(request);
+ CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model_handle, GetHwLatencyMeasurementSerializer);
+
+ auto lambda = [] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->get_hw_latency_measurement();
+ };
+
+ auto latency_measurement_result = cim_manager.execute<Expected<LatencyMeasurementResult>>(configured_infer_model_handle.value(), lambda);
+ if (HAILO_NOT_AVAILABLE == latency_measurement_result.status()) {
+ return GetHwLatencyMeasurementSerializer::serialize_reply(HAILO_NOT_AVAILABLE);
+ }
+ CHECK_EXPECTED_AS_HRPC_STATUS(latency_measurement_result, GetHwLatencyMeasurementSerializer);
+
+ uint32_t avg_hw_latency = static_cast<uint32_t>(latency_measurement_result.value().avg_hw_latency.count());
+ TRY_AS_HRPC_STATUS(auto reply, GetHwLatencyMeasurementSerializer::serialize_reply(latency_measurement_result.status(), avg_hw_latency), GetHwLatencyMeasurementSerializer);
+
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__ACTIVATE,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+
+ auto configured_infer_model_handle = ActivateSerializer::deserialize_request(request);
+ CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model_handle, ActivateSerializer);
+
+ auto lambda = [] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->activate();
+ };
+
+ auto status = cim_manager.execute<hailo_status>(configured_infer_model_handle.value(), lambda);
+ TRY_AS_HRPC_STATUS(auto reply, ActivateSerializer::serialize_reply(status), ActivateSerializer);
+
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__DEACTIVATE,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+
+ auto configured_infer_model_handle = DeactivateSerializer::deserialize_request(request);
+ CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model_handle, DeactivateSerializer);
+
+ auto lambda = [] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->deactivate();
+ };
+
+ auto status = cim_manager.execute<hailo_status>(configured_infer_model_handle.value(), lambda);
+ TRY_AS_HRPC_STATUS(auto reply, DeactivateSerializer::serialize_reply(status), DeactivateSerializer);
+
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__SHUTDOWN,
+ [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected<Buffer> {
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+
+ auto configured_infer_model_handle = ShutdownSerializer::deserialize_request(request);
+ CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model_handle, ShutdownSerializer);
+
+ auto lambda = [] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->shutdown();
+ };
+
+ auto status = cim_manager.execute<hailo_status>(configured_infer_model_handle.value(), lambda);
+ TRY_AS_HRPC_STATUS(auto reply, ShutdownSerializer::serialize_reply(status), ShutdownSerializer);
+
+ return reply;
+ });
+ dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__RUN_ASYNC,
+ [&infer_model_to_info_id, &buffer_pool_per_cim]
+ (const MemoryView &request, hrpc::ServerContextPtr server_context) -> Expected<Buffer> {
+ auto &cim_manager = ServiceResourceManager<ConfiguredInferModel>::get_instance();
+ auto bindings_lambda = [] (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->create_bindings();
+ };
+ TRY_AS_HRPC_STATUS(auto request_tuple, RunAsyncSerializer::deserialize_request(request), RunAsyncSerializer);
+ auto configured_infer_model_handle = std::get<0>(request_tuple);
+ auto infer_model_handle = std::get<1>(request_tuple);
+ auto callback_id = std::get<2>(request_tuple);
+
+ auto bindings = cim_manager.execute<Expected<ConfiguredInferModel::Bindings>>(configured_infer_model_handle, bindings_lambda);
+ CHECK_EXPECTED_AS_HRPC_STATUS(bindings, RunAsyncSerializer);
+
+ auto infer_model_info_lambda = [] (std::shared_ptr<InferModelInfo> infer_model_info) {
+ return *infer_model_info;
+ };
+ auto &infer_model_infos_manager = ServiceResourceManager<InferModelInfo>::get_instance();
+ auto infer_model_info = infer_model_infos_manager.execute<Expected<InferModelInfo>>(infer_model_to_info_id[infer_model_handle],
+ infer_model_info_lambda);
+ CHECK_EXPECTED_AS_HRPC_STATUS(infer_model_info, RunAsyncSerializer);
+
+ std::vector<BufferPtr> inputs; // TODO: add infer vector pool
+ inputs.reserve(infer_model_info->inputs_names.size());
+ for (const auto &input_name : infer_model_info->inputs_names) {
+ TRY_AS_HRPC_STATUS(auto input, bindings->input(input_name), RunAsyncSerializer);
+
+ TRY_AS_HRPC_STATUS(auto buffer_ptr, buffer_pool_per_cim[configured_infer_model_handle]->acquire_buffer(input_name),
+ RunAsyncSerializer);
+
+ auto status = server_context->connection().read_buffer(MemoryView(*buffer_ptr));
+ CHECK_SUCCESS_AS_HRPC_STATUS(status, RunAsyncSerializer);
+
+ inputs.emplace_back(buffer_ptr);
+ status = input.set_buffer(MemoryView(*buffer_ptr));
+ CHECK_SUCCESS_AS_HRPC_STATUS(status, RunAsyncSerializer);
+ }
+
+ std::vector<BufferPtr> outputs; // TODO: add infer vector pool
+ outputs.reserve(infer_model_info->outputs_names.size());
+ for (const auto &output_name : infer_model_info->outputs_names) {
+ TRY_AS_HRPC_STATUS(auto buffer_ptr, buffer_pool_per_cim[configured_infer_model_handle]->acquire_buffer(output_name),
+ RunAsyncSerializer);
+
+ auto output = bindings->output(output_name);
+ CHECK_EXPECTED_AS_HRPC_STATUS(output, RunAsyncSerializer);
+
+ auto status = output->set_buffer(MemoryView(buffer_ptr->data(), buffer_ptr->size()));
+ CHECK_SUCCESS_AS_HRPC_STATUS(status, RunAsyncSerializer);
+
+ outputs.emplace_back(buffer_ptr);
+ }
+
+ auto infer_lambda =
+ [bindings = bindings.release(), callback_id, server_context, inputs, outputs, &buffer_pool_per_cim, configured_infer_model_handle,
+ infer_model_info]
+ (std::shared_ptr<ConfiguredInferModel> configured_infer_model) {
+ return configured_infer_model->run_async(bindings,
+ [callback_id, server_context, inputs, outputs, &buffer_pool_per_cim, configured_infer_model_handle, infer_model_info]
+ (const AsyncInferCompletionInfo &completion_info) {
+ auto status = server_context->trigger_callback(callback_id, completion_info.status, [outputs, completion_info] (hrpc::RpcConnection connection) -> hailo_status {
+ if (HAILO_SUCCESS == completion_info.status) {
+ for (auto output : outputs) {
+ auto status = connection.write_buffer(MemoryView(*output));
+ CHECK_SUCCESS(status);
+ }
+ }
+ return HAILO_SUCCESS;
+ });
+
+ // HAILO_COMMUNICATION_CLOSED means the client disconnected. Server doesn't need to restart in this case.
+ if ((status != HAILO_SUCCESS) && (status != HAILO_COMMUNICATION_CLOSED)) {
+ LOGGER__CRITICAL("Error {} returned from connection.write(). Server Should restart!", status);
+ }
+
+ for (uint32_t i = 0; i < inputs.size(); i++) {
+ status = buffer_pool_per_cim[configured_infer_model_handle]->return_to_pool(infer_model_info->inputs_names[i], inputs[i]);
+ if (status != HAILO_SUCCESS) {
+ LOGGER__CRITICAL("return_to_pool failed for input {}, status = {}. Server should restart!", infer_model_info->inputs_names[i], status);
+ return;
+ }
+ }
+ for (uint32_t i = 0; i < outputs.size(); i++) {
+ status = buffer_pool_per_cim[configured_infer_model_handle]->return_to_pool(infer_model_info->outputs_names[i], outputs[i]);
+ if (status != HAILO_SUCCESS) {
+ LOGGER__CRITICAL("return_to_pool failed for output {}, status = {}. Server should restart!", infer_model_info->outputs_names[i], status);
+ return;
+ }
+ }
+ });
+ };
+ auto job = cim_manager.execute<Expected<AsyncInferJob>>(configured_infer_model_handle, infer_lambda);
+ CHECK_EXPECTED_AS_HRPC_STATUS(job, RunAsyncSerializer);
+
+ job->detach();
+
+ TRY_AS_HRPC_STATUS(auto reply, RunAsyncSerializer::serialize_reply(HAILO_SUCCESS), RunAsyncSerializer);
+ return reply;
+ });
+
+ server->set_dispatcher(dispatcher);
+ auto status = server->serve();
+ if (status != HAILO_SUCCESS) {
+ LOGGER__ERROR("Error in serve, status = {}", status);
+ return status;
+ }
+
+ return 0;
+}
--- /dev/null
+#ifndef HAILORT_SERVER_HPP_
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file hailort_server.hpp
+ * @brief RPC Hailort Server Header
+ **/
+
+#define HAILORT_SERVER_HPP_
+
+#include "hrpc/server.hpp"
+#include "hailort_service/cng_buffer_pool.hpp"
+
+namespace hrpc
+{
+
+using infer_model_handle_t = uint32_t;
+
+class Server;
+class HailoRTServer : public Server {
+public:
+ static Expected<std::unique_ptr<HailoRTServer>> create_unique();
+ explicit HailoRTServer(std::shared_ptr<ConnectionContext> connection_context) : Server(connection_context) {};
+
+ std::unordered_map<uint32_t, uint32_t> &get_infer_model_to_info_id() { return m_infer_model_to_info_id; };
+ std::unordered_map<uint32_t, std::shared_ptr<ServiceNetworkGroupBufferPool>> &get_buffer_pool_per_cim() { return m_buffer_pool_per_cim; };
+ std::unordered_map<infer_model_handle_t, Buffer> &get_hef_buffers() { return m_hef_buffers_per_infer_model; };
+
+private:
+
+ std::unordered_map<uint32_t, uint32_t> m_infer_model_to_info_id;
+ std::unordered_map<uint32_t, std::shared_ptr<ServiceNetworkGroupBufferPool>> m_buffer_pool_per_cim;
+ std::unordered_map<infer_model_handle_t, Buffer> m_hef_buffers_per_infer_model;
+ virtual hailo_status cleanup_client_resources(RpcConnection client_connection) override;
+ void cleanup_cim_buffer_pools(const std::vector<uint32_t> &cim_handles);
+ void cleanup_infer_model_hef_buffers(const std::vector<uint32_t> &infer_model_handles);
+
+};
+
+} // namespace hrpc
+
+#endif // HAILORT_SERVER_HPP_
\ No newline at end of file
--- /dev/null
+#! /bin/bash
+
+### BEGIN INIT INFO
+# Provides: hailort_server
+# Required-Start: $local_fs $network
+# Required-Stop: $local_fs
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: hailort_server service
+# Description: Run hailort_server daemon
+### END INIT INFO
+
+# TODO: Remove this file once the hailort_server will use systemd
+
+# Carry out specific functions when asked to by the system
+case "$1" in
+ start)
+ echo "Starting hailort_server"
+ bash -c "cd /usr/bin && hailort_server &"
+ ;;
+
+ stop)
+ echo "Stopping hailort_server..."
+ bash -c "killall hailort_server"
+ ;;
+
+ restart)
+ $0 stop
+ sleep 1
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: /etc/init.d/hailort_server {start|stop|restart}"
+ exit 1
+ ;;
+esac
+
+exit 0
\ No newline at end of file
};
auto &vdevice_manager = ServiceResourceManager<VDevice>::get_instance();
- auto free_buffers_queue = SpscQueue<BufferPtr>::create(buffer_count, shutdown_event, DEFAULT_TRANSFER_TIMEOUT);
- CHECK_EXPECTED(free_buffers_queue);
+ TRY(auto free_buffers_queue,
+ SpscQueue<BufferPtr>::create(buffer_count, shutdown_event, DEFAULT_TRANSFER_TIMEOUT));
std::vector<AllocatedMappedBuffer> buffers;
buffers.reserve(buffer_count);
for (size_t i = 0; i < buffer_count; i++) {
- auto buffer = Buffer::create_shared(buffer_size, BufferStorageParams::create_dma());
- CHECK_EXPECTED(buffer);
+ TRY(auto buffer, Buffer::create_shared(buffer_size, BufferStorageParams::create_dma()));
- auto mapped_buffer = vdevice_manager.execute<Expected<DmaMappedBuffer>>(vdevice_handle, map_buffer_lambda, buffer.value());
- CHECK_EXPECTED(mapped_buffer);
+ TRY(auto mapped_buffer,
+ vdevice_manager.execute<Expected<DmaMappedBuffer>>(vdevice_handle, map_buffer_lambda, buffer));
- auto status = free_buffers_queue->enqueue(buffer.value());
+ auto status = free_buffers_queue.enqueue(buffer);
CHECK_SUCCESS(status);
- buffers.emplace_back(AllocatedMappedBuffer{ buffer.release(), mapped_buffer.release()});
+ buffers.emplace_back(AllocatedMappedBuffer{ buffer, std::move(mapped_buffer)});
}
auto buffer_pool_ptr = make_shared_nothrow<ServiceStreamBufferPool>(buffer_size, std::move(buffers),
- free_buffers_queue.release(), buffer_count);
+ std::move(free_buffers_queue), buffer_count);
CHECK_NOT_NULL_AS_EXPECTED(buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY);
return buffer_pool_ptr;
Expected<BufferPtr> ServiceStreamBufferPool::acquire_buffer()
{
- auto buffer = m_free_buffers_queue.dequeue(DEFAULT_TRANSFER_TIMEOUT);
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == buffer.status()) {
- return make_unexpected(buffer.status());
- }
- else if (HAILO_TIMEOUT == buffer.status()) {
- LOGGER__WARNING(
- "Failed to acquire buffer because the buffer pool is empty. This could be caused by uneven reading and writing speeds");
- return make_unexpected(buffer.status());
- }
- CHECK_EXPECTED(buffer);
-
- return buffer.release();
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto buffer,
+ m_free_buffers_queue.dequeue(DEFAULT_TRANSFER_TIMEOUT));
+ return buffer;
}
hailo_status ServiceStreamBufferPool::return_to_pool(BufferPtr buffer)
Expected<std::shared_ptr<ServiceNetworkGroupBufferPool>> ServiceNetworkGroupBufferPool::create(uint32_t vdevice_handle)
{
- auto shutdown_event_exp = Event::create_shared(Event::State::not_signalled);
- CHECK_EXPECTED(shutdown_event_exp);
- auto shutdown_event = shutdown_event_exp.release();
+ TRY(auto shutdown_event, Event::create_shared(Event::State::not_signalled));
auto cng_buffer_pool_ptr = make_shared_nothrow<ServiceNetworkGroupBufferPool>(shutdown_event, vdevice_handle);
CHECK_NOT_NULL_AS_EXPECTED(cng_buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY);
}
ServiceNetworkGroupBufferPool::ServiceNetworkGroupBufferPool(EventPtr shutdown_event, uint32_t vdevice_handle) :
- m_output_name_to_buffer_pool(), m_shutdown_event(shutdown_event), m_vdevice_handle(vdevice_handle)
+ m_stream_name_to_buffer_pool(), m_shutdown_event(shutdown_event), m_vdevice_handle(vdevice_handle)
{}
-hailo_status ServiceNetworkGroupBufferPool::allocate_pool(const std::string &name, size_t frame_size, size_t pool_size)
+hailo_status ServiceNetworkGroupBufferPool::allocate_pool(const std::string &name,
+ hailo_dma_buffer_direction_t direction, size_t frame_size, size_t pool_size)
{
- auto buffer_pool = ServiceStreamBufferPool::create(m_vdevice_handle, frame_size,
- pool_size, HAILO_DMA_BUFFER_DIRECTION_D2H, m_shutdown_event);
- CHECK_EXPECTED(buffer_pool);
+ TRY(auto buffer_pool, ServiceStreamBufferPool::create(m_vdevice_handle, frame_size,
+ pool_size, direction, m_shutdown_event));
std::unique_lock<std::mutex> lock(m_mutex);
- m_output_name_to_buffer_pool[name] = buffer_pool.release();
+ m_stream_name_to_buffer_pool[name] = buffer_pool;
return HAILO_SUCCESS;
}
-hailo_status ServiceNetworkGroupBufferPool::reallocate_pool(const std::string &name, size_t frame_size)
+hailo_status ServiceNetworkGroupBufferPool::reallocate_pool(const std::string &name,
+ hailo_dma_buffer_direction_t direction, size_t frame_size)
{
std::unique_lock<std::mutex> lock(m_mutex);
- auto pool_size = m_output_name_to_buffer_pool[name]->buffers_count();
- m_output_name_to_buffer_pool[name].reset();
+ auto pool_size = m_stream_name_to_buffer_pool[name]->buffers_count();
+ m_stream_name_to_buffer_pool[name].reset();
- auto buffer_pool = ServiceStreamBufferPool::create(m_vdevice_handle, frame_size,
- pool_size, HAILO_DMA_BUFFER_DIRECTION_D2H, m_shutdown_event);
- CHECK_EXPECTED(buffer_pool);
- m_output_name_to_buffer_pool[name] = buffer_pool.release();
+ TRY(auto buffer_pool, ServiceStreamBufferPool::create(m_vdevice_handle, frame_size,
+ pool_size, direction, m_shutdown_event));
+ m_stream_name_to_buffer_pool[name] = buffer_pool;
return HAILO_SUCCESS;
}
-Expected<BufferPtr> ServiceNetworkGroupBufferPool::acquire_buffer(const std::string &output_name)
+Expected<BufferPtr> ServiceNetworkGroupBufferPool::acquire_buffer(const std::string &stream_name)
{
- CHECK_AS_EXPECTED(contains(m_output_name_to_buffer_pool, output_name), HAILO_INTERNAL_FAILURE,
- "acquire_buffer() for output {} failed, output name does not exist in buffer pool", output_name);
+ CHECK_AS_EXPECTED(contains(m_stream_name_to_buffer_pool, stream_name), HAILO_INTERNAL_FAILURE,
+ "acquire_buffer() for stream {} failed, stream name does not exist in buffer pool", stream_name);
std::unique_lock<std::mutex> lock(m_mutex);
- auto buffer = m_output_name_to_buffer_pool.at(output_name)->acquire_buffer();
- CHECK_EXPECTED(buffer);
+ TRY(auto buffer, m_stream_name_to_buffer_pool.at(stream_name)->acquire_buffer());
- return buffer.release();
+ return buffer;
}
-hailo_status ServiceNetworkGroupBufferPool::return_to_pool(const std::string &output_name, BufferPtr buffer)
+hailo_status ServiceNetworkGroupBufferPool::return_to_pool(const std::string &stream_name, BufferPtr buffer)
{
- CHECK(contains(m_output_name_to_buffer_pool, output_name), HAILO_INTERNAL_FAILURE,
- "acquire_buffer() for output {} failed, output name does not exist in buffer pool", output_name);
+ CHECK(contains(m_stream_name_to_buffer_pool, stream_name), HAILO_INTERNAL_FAILURE,
+ "acquire_buffer() for stream {} failed, stream name does not exist in buffer pool", stream_name);
std::unique_lock<std::mutex> lock(m_mutex);
- auto status = m_output_name_to_buffer_pool.at(output_name)->return_to_pool(buffer);
+ auto status = m_stream_name_to_buffer_pool.at(stream_name)->return_to_pool(buffer);
CHECK_SUCCESS(status);
return HAILO_SUCCESS;
**/
/**
* @file cng_buffer_pool.hpp
- * @brief This model represents the buffer pools for the output reads for each network group. Used in async API
+ * @brief This model represents the buffer pools for the streams of each network group. Used in async API
**/
#ifndef _HAILO_CNG_BUFFER_POOL_HPP_
};
using BufferPoolPtr = std::shared_ptr<ServiceStreamBufferPool>;
-using output_name_t = std::string;
+using stream_name_t = std::string;
-// This object holds a buffer pool for each output streams of the network group.
+// This object holds a buffer pool for each stream of the network group.
// It is used to pre-allocate all the buffers necessary for the reads from the device.
// The buffers are reuseable, which also prevents allocation during inference.
// The buffers are mapped to the device during their creation, which prevent lazy mapping each frame inference.
public:
static Expected<std::shared_ptr<ServiceNetworkGroupBufferPool>> create(uint32_t vdevice_handle);
- hailo_status allocate_pool(const std::string &name, size_t frame_size, size_t pool_size);
+ hailo_status allocate_pool(const std::string &name, hailo_dma_buffer_direction_t direction, size_t frame_size, size_t pool_size);
// Used in order to reallocate the pool buffers with different frame_size
- hailo_status reallocate_pool(const std::string &name, size_t frame_size);
+ hailo_status reallocate_pool(const std::string &name, hailo_dma_buffer_direction_t direction, size_t frame_size);
ServiceNetworkGroupBufferPool(ServiceNetworkGroupBufferPool &&) = delete;
ServiceNetworkGroupBufferPool(const ServiceNetworkGroupBufferPool &) = delete;
virtual ~ServiceNetworkGroupBufferPool() = default;
ServiceNetworkGroupBufferPool(EventPtr shutdown_event, uint32_t vdevice_handle);
- Expected<BufferPtr> acquire_buffer(const std::string &output_name);
- hailo_status return_to_pool(const std::string &output_name, BufferPtr buffer);
+ Expected<BufferPtr> acquire_buffer(const std::string &stream_name);
+ hailo_status return_to_pool(const std::string &stream_name, BufferPtr buffer);
hailo_status shutdown();
private:
- std::unordered_map<output_name_t, BufferPoolPtr> m_output_name_to_buffer_pool;
+ std::unordered_map<stream_name_t, BufferPoolPtr> m_stream_name_to_buffer_pool;
EventPtr m_shutdown_event;
uint32_t m_vdevice_handle;
std::mutex m_mutex;
hailo_status HailoRtRpcService::create_buffer_pools_for_ng(uint32_t vdevice_handle, uint32_t ng_handle, uint32_t request_pid,
bool allocate_for_raw_streams)
{
- auto cng_buffer_pool = ServiceNetworkGroupBufferPool::create(vdevice_handle);
- CHECK_EXPECTED_AS_STATUS(cng_buffer_pool);
+ TRY(auto cng_buffer_pool, ServiceNetworkGroupBufferPool::create(vdevice_handle));
auto &cng_buffer_pool_manager = ServiceResourceManager<ServiceNetworkGroupBufferPool>::get_instance();
- auto cng_buffer_pool_handle = cng_buffer_pool_manager.register_resource(request_pid, cng_buffer_pool.release());
+ auto cng_buffer_pool_handle = cng_buffer_pool_manager.register_resource(request_pid, cng_buffer_pool);
CHECK(cng_buffer_pool_handle == ng_handle, HAILO_INTERNAL_FAILURE,
"cng_buffer_pool_handle = {} must be equal to network_group_handle ={}", cng_buffer_pool_handle, ng_handle);
if (allocate_for_raw_streams) {
// For Async API - The buffer size in the pool will be the stream's hw frame size as used in the infer_model pipeline
- auto min_buffer_pool_size = get_min_buffer_pool_size(ng_handle);
- CHECK_EXPECTED_AS_STATUS(min_buffer_pool_size);
+ TRY(const auto min_buffer_pool_size, get_min_buffer_pool_size(ng_handle));
+ TRY(const auto streams_infos, get_all_stream_infos(ng_handle));
- auto streams_infos = get_all_stream_infos(ng_handle);
- CHECK_EXPECTED_AS_STATUS(streams_infos);
-
- for (const auto &stream_info : streams_infos.value()) {
+ for (const auto &stream_info : streams_infos) {
if (stream_info.direction == HAILO_D2H_STREAM) {
auto allocate_lambda = [&](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool) {
- return cng_buffer_pool->allocate_pool(stream_info.name, stream_info.hw_frame_size, min_buffer_pool_size.value());
+ return cng_buffer_pool->allocate_pool(stream_info.name, HAILO_DMA_BUFFER_DIRECTION_D2H,
+ stream_info.hw_frame_size, min_buffer_pool_size);
};
CHECK_SUCCESS(cng_buffer_pool_manager.execute(ng_handle, allocate_lambda));
}
mem_view = MemoryView::create_const(data, proto_stream_transfer_request.data().size());
} else {
// The memory is not aligned to 8, therefore we need to copy the data into a buffer
- auto buffer_exp = Buffer::create_shared(data, proto_stream_transfer_request.data().size(),
- BufferStorageParams::create_dma());
- CHECK_EXPECTED(buffer_exp);
- buffer = buffer_exp.release();
+ TRY(buffer, Buffer::create_shared(data, proto_stream_transfer_request.data().size(),
+ BufferStorageParams::create_dma()));
mem_view = MemoryView(*buffer);
}
enqueue_cb_identifier(vdevice_handle, std::move(cb_identifier));
};
- named_buffers_callbacks.emplace(stream_name, std::make_pair(mem_view, transfer_done));
+ BufferRepresentation buffer_representation {};
+ buffer_representation.buffer_type = BufferType::VIEW;
+ buffer_representation.view = mem_view;
+
+ named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_representation, transfer_done));
return HAILO_SUCCESS;
}
{
// Prepare output buffer
auto &stream_name = proto_stream_transfer_request.stream_name();
- auto buffer_exp = acquire_buffer_from_cng_pool(ng_handle, stream_name);
- CHECK_EXPECTED(buffer_exp);
- auto buffer = buffer_exp.release();
+ TRY(auto buffer, acquire_buffer_from_cng_pool(ng_handle, stream_name));
// Prepare callback
auto cb_idx = proto_stream_transfer_request.cb_idx();
enqueue_cb_identifier(vdevice_handle, std::move(cb_identifier));
};
- named_buffers_callbacks.emplace(stream_name, std::make_pair(MemoryView(*buffer), transfer_done));
+ BufferRepresentation buffer_representation {};
+ buffer_representation.buffer_type = BufferType::VIEW;
+ buffer_representation.view = MemoryView(*buffer);
+
+ named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_representation, transfer_done));
return HAILO_SUCCESS;
}
auto lambda_acquire_buffer = [](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool, const std::string &output_name) {
return cng_buffer_pool->acquire_buffer(output_name);
};
- auto buffer = cng_buffer_pool_manager.execute<Expected<BufferPtr>>(ng_handle, lambda_acquire_buffer, output_name);
- CHECK_EXPECTED(buffer);
- return buffer.release();
+ TRY(auto buffer,
+ cng_buffer_pool_manager.execute<Expected<BufferPtr>>(
+ ng_handle, lambda_acquire_buffer, output_name)
+ );
+
+ return buffer;
}
grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_infer_async(grpc::ServerContext*,
nms_config_proto->set_background_removal(nms_config.background_removal);
nms_config_proto->set_background_removal_index(nms_config.background_removal_index);
nms_config_proto->set_cross_classes(nms_config.cross_classes);
+ nms_config_proto->set_bbox_only(nms_config.bbox_only);
}
switch (op_metadata.type()) {
auto &vstream_manager = ServiceResourceManager<OutputVStream>::get_instance();
for (size_t i = 0; i < vstreams.size(); i++) {
auto allocate_lambda = [&](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool) {
- return cng_buffer_pool->allocate_pool(vstreams[i].name(), vstreams[i].get_frame_size(), output_params.at(vstreams[i].name()).queue_size);
+ return cng_buffer_pool->allocate_pool(vstreams[i].name(), HAILO_DMA_BUFFER_DIRECTION_D2H,
+ vstreams[i].get_frame_size(), output_params.at(vstreams[i].name()).queue_size);
};
CHECK_SUCCESS_AS_RPC_STATUS(cng_buffer_pool_manager.execute(network_group_handle, allocate_lambda), reply);
auto buffer_exp = acquire_buffer_from_cng_pool(ng_handle, vstream_name.value());
CHECK_EXPECTED_AS_RPC_STATUS(buffer_exp, reply);
- auto buffer = buffer_exp.release();
+ auto buffer = buffer_exp.value();
auto lambda = [](std::shared_ptr<OutputVStream> output_vstream, MemoryView &buffer) {
return output_vstream->read(std::move(buffer));
return cng->get_all_stream_infos();
};
auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
- auto expected_stream_infos = manager.execute<Expected<std::vector<hailo_stream_info_t>>>(ng_handle, lambda);
- CHECK_EXPECTED(expected_stream_infos);
+ TRY(auto stream_infos,
+ manager.execute<Expected<std::vector<hailo_stream_info_t>>>(ng_handle, lambda));
- return expected_stream_infos.release();
+ return stream_infos;
}
Expected<std::vector<hailo_vstream_info_t>> HailoRtRpcService::get_all_vstream_infos(uint32_t ng_handle)
return cng->get_all_vstream_infos();
};
auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
- auto expected_vstream_infos = manager.execute<Expected<std::vector<hailo_vstream_info_t>>>(ng_handle, lambda);
- CHECK_EXPECTED(expected_vstream_infos);
+ TRY(auto vstream_infos,
+ manager.execute<Expected<std::vector<hailo_vstream_info_t>>>(ng_handle, lambda));
- return expected_vstream_infos.release();
+ return vstream_infos;
}
grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_all_stream_infos(grpc::ServerContext*,
return cng->get_min_buffer_pool_size();
};
auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
- auto min_buffer_pool_size = manager.execute<Expected<size_t>>(ng_handle, lambda);
- CHECK_EXPECTED(min_buffer_pool_size);
+ TRY(auto min_buffer_pool_size, manager.execute<Expected<size_t>>(ng_handle, lambda));
- return min_buffer_pool_size.release();
+ return min_buffer_pool_size;
}
grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_min_buffer_pool_size(grpc::ServerContext*,
return output_vstream->name();
};
auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
- auto name = manager.execute<Expected<std::string>>(vstream_handle, lambda);
- CHECK_EXPECTED(name);
+ TRY(auto name, manager.execute<Expected<std::string>>(vstream_handle, lambda));
- return name.release();
+ return name;
}
Expected<size_t> HailoRtRpcService::output_vstream_frame_size(uint32_t vstream_handle)
return output_vstream->get_frame_size();
};
auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
- auto frame_size = manager.execute<Expected<size_t>>(vstream_handle, lambda);
- CHECK_EXPECTED(frame_size);
+ TRY(auto frame_size, manager.execute<Expected<size_t>>(vstream_handle, lambda));
- return frame_size.release();
+ return frame_size;
}
grpc::Status HailoRtRpcService::OutputVStream_name(grpc::ServerContext*, const VStream_name_Request *request,
hailo_status HailoRtRpcService::update_buffer_size_in_pool(uint32_t vstream_handle, uint32_t network_group_handle)
{
- auto vstream_name = output_vstream_name(vstream_handle);
- CHECK_EXPECTED(vstream_name);
-
- auto frame_size = output_vstream_frame_size(vstream_handle);
- CHECK_EXPECTED(frame_size);
+ TRY(const auto vstream_name, output_vstream_name(vstream_handle));
+ TRY(const auto frame_size, output_vstream_frame_size(vstream_handle));
auto &cng_buffer_pool_manager = ServiceResourceManager<ServiceNetworkGroupBufferPool>::get_instance();
auto allocate_lambda = [&](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool) {
- return cng_buffer_pool->reallocate_pool(vstream_name.release(), frame_size.release());
+ return cng_buffer_pool->reallocate_pool(vstream_name, HAILO_DMA_BUFFER_DIRECTION_D2H, frame_size);
};
CHECK_SUCCESS(cng_buffer_pool_manager.execute(network_group_handle, allocate_lambda));
#include <shared_mutex>
#include <unordered_set>
+#define SINGLE_CLIENT_PID (0)
+
namespace hailort
{
K execute(uint32_t handle, Func &lambda, Args... args)
{
std::unique_lock<std::mutex> lock(m_mutex);
- auto resource_expected = resource_lookup(handle);
- CHECK_EXPECTED(resource_expected);
- auto resource = resource_expected.release();
-
+ TRY(auto resource, resource_lookup(handle));
assert(contains(m_resources_mutexes, handle));
std::shared_lock<std::shared_timed_mutex> resource_lock(m_resources_mutexes[handle]);
lock.unlock();
hailo_status execute(uint32_t handle, Func &lambda, Args... args)
{
std::unique_lock<std::mutex> lock(m_mutex);
- auto resource_expected = resource_lookup(handle);
- CHECK_EXPECTED_AS_STATUS(resource_expected);
- auto resource = resource_expected.release();
-
+ TRY(auto resource, resource_lookup(handle));
assert(contains(m_resources_mutexes, handle));
std::shared_lock<std::shared_timed_mutex> resource_lock(m_resources_mutexes[handle]);
lock.unlock();
Expected<uint32_t> dup_handle(uint32_t handle, uint32_t pid)
{
std::unique_lock<std::mutex> lock(m_mutex);
- auto resource_expected = resource_lookup(handle);
- CHECK_EXPECTED(resource_expected);
- auto resource = resource_expected.release();
-
+ TRY(auto resource, resource_lookup(handle));
assert(contains(m_resources_mutexes, handle));
std::unique_lock<std::shared_timed_mutex> resource_lock(m_resources_mutexes[handle]);
resource->pids.insert(pid);
{
std::unique_lock<std::shared_timed_mutex> resource_lock(m_resources_mutexes[handle]);
resource->pids.erase(pid);
- if (all_pids_dead(resource)) {
+ if ((SINGLE_CLIENT_PID == pid) || all_pids_dead(resource)) {
release_resource = true;
res = resource->resource;
m_resources.erase(handle);
public:
static Expected<std::unique_ptr<VDeviceCallbacksQueue>> create(uint32_t max_queue_size)
{
- auto shutdown_event_exp = Event::create_shared(Event::State::not_signalled);
- CHECK_EXPECTED(shutdown_event_exp);
- auto shutdown_event = shutdown_event_exp.release();
+ TRY(auto shutdown_event, Event::create_shared(Event::State::not_signalled));
- auto cb_ids_queue = SpscQueue<ProtoCallbackIdentifier>::create(max_queue_size, shutdown_event, HAILO_INFINITE_TIMEOUT);
- CHECK_EXPECTED(cb_ids_queue);
+ TRY(auto cb_ids_queue,
+ SpscQueue<ProtoCallbackIdentifier>::create(max_queue_size, shutdown_event, HAILO_INFINITE_TIMEOUT));
- auto queue_ptr = make_unique_nothrow<VDeviceCallbacksQueue>(cb_ids_queue.release(), shutdown_event);
+ auto queue_ptr = make_unique_nothrow<VDeviceCallbacksQueue>(std::move(cb_ids_queue), shutdown_event);
CHECK_AS_EXPECTED(nullptr != queue_ptr, HAILO_OUT_OF_HOST_MEMORY);
return queue_ptr;
Expected<ProtoCallbackIdentifier> dequeue()
{
- auto callback_id = m_callbacks_ids_queue.dequeue();
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == callback_id.status()) {
- return make_unexpected(callback_id.status());
- }
- else if (HAILO_TIMEOUT == callback_id.status()) {
- LOGGER__WARNING("Failed to dequeue callback_id because the queue is empty, status={}", HAILO_TIMEOUT);
- return make_unexpected(callback_id.status());
- }
- CHECK_EXPECTED(callback_id);
-
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto callback_id,
+ m_callbacks_ids_queue.dequeue());
return callback_id;
}
std::cout << "Starting Measurements..." << std::endl;
std::cout << "Measuring FPS in hw_only mode" << std::endl;
- auto hw_only_mode_info = hw_only_mode();
- CHECK_EXPECTED_AS_STATUS(hw_only_mode_info, "hw_only measuring failed");
+ TRY(auto hw_only_mode_info, hw_only_mode(), "hw_only measuring failed");
std::cout << "Measuring FPS " << (!m_not_measure_power ? "and Power " : "") << "in streaming mode" << std::endl;
- auto streaming_mode_info = fps_streaming_mode();
- CHECK_EXPECTED_AS_STATUS(streaming_mode_info, "FPS in streaming mode failed");
+ TRY(auto streaming_mode_info, fps_streaming_mode(), "FPS in streaming mode failed");
// TODO - HRT-6931 - measure latency only in the case of single device.
std::cout << "Measuring HW Latency" << std::endl;
- auto latency_info = latency();
- CHECK_EXPECTED_AS_STATUS(latency_info, "Latency measuring failed");
+ TRY(auto latency_info, latency(), "Latency measuring failed");
- assert(hw_only_mode_info->network_group_results().size() == streaming_mode_info->network_group_results().size());
- assert(latency_info->network_group_results().size() == streaming_mode_info->network_group_results().size());
+ assert(hw_only_mode_info.network_group_results().size() == streaming_mode_info.network_group_results().size());
+ assert(latency_info.network_group_results().size() == streaming_mode_info.network_group_results().size());
std::cout << std::endl;
std::cout << "=======" << std::endl;
std::cout << "Summary" << std::endl;
std::cout << "=======" << std::endl;
- for (auto &hw_only_res : hw_only_mode_info->network_group_results()) {
+ for (auto &hw_only_res : hw_only_mode_info.network_group_results()) {
auto network_group_name = hw_only_res.network_group_name();
- auto streaming_res = std::find_if(streaming_mode_info->network_group_results().begin(), streaming_mode_info->network_group_results().end(),
+ auto streaming_res = std::find_if(streaming_mode_info.network_group_results().begin(), streaming_mode_info.network_group_results().end(),
[network_group_name] (NetworkGroupInferResult &infer_results) { return (infer_results.network_group_name() == network_group_name); });
- CHECK(streaming_mode_info->network_group_results().end() != streaming_res, HAILO_INTERNAL_FAILURE, "Failed to fun streaming results for network group {}", network_group_name);
+ CHECK(streaming_mode_info.network_group_results().end() != streaming_res, HAILO_INTERNAL_FAILURE, "Failed to fun streaming results for network group {}", network_group_name);
- auto latency_res = std::find_if(latency_info->network_group_results().begin(), latency_info->network_group_results().end(),
+ auto latency_res = std::find_if(latency_info.network_group_results().begin(), latency_info.network_group_results().end(),
[network_group_name] (NetworkGroupInferResult &infer_results) { return (infer_results.network_group_name() == network_group_name); });
- CHECK(latency_info->network_group_results().end() != latency_res, HAILO_INTERNAL_FAILURE, "Failed to fun latency results for network group {}", network_group_name);
+ CHECK(latency_info.network_group_results().end() != latency_res, HAILO_INTERNAL_FAILURE, "Failed to fun latency results for network group {}", network_group_name);
std::cout << "FPS (hw_only) = " << hw_only_res.fps().value() <<std::endl;
std::cout << " (streaming) = " << streaming_res->fps().value() <<std::endl;
}
}
if (!m_not_measure_power) {
- for (const auto &pair : streaming_mode_info->m_power_measurements) {
+ for (const auto &pair : streaming_mode_info.m_power_measurements) {
std::cout << "Device " << pair.first << ":" << std::endl;
const auto &data = pair.second->data();
const auto &power_units = pair.second->power_units();
if (!m_csv_file_path.empty()){
m_params.csv_output = m_csv_file_path;
- auto printer = InferStatsPrinter::create(m_params, false);
- CHECK_EXPECTED_AS_STATUS(printer, "Failed to initialize infer stats printer");
- printer->print_benchmark_csv_header();
- printer->print_benchmark_csv(hw_only_mode_info.value(),
- streaming_mode_info.value(), latency_info.value());
+ TRY(auto printer, InferStatsPrinter::create(m_params, false), "Failed to initialize infer stats printer");
+ printer.print_benchmark_csv_header();
+ printer.print_benchmark_csv(hw_only_mode_info, streaming_mode_info, latency_info);
}
return HAILO_SUCCESS;
}
CHECK_SUCCESS(status,
"'board-config read' command should get a specific device-id.");
- auto buffer = device.read_board_config();
- CHECK_EXPECTED_AS_STATUS(buffer, "Failed reading board config from device");
+ TRY(auto buffer, device.read_board_config(), "Failed reading board config from device");
auto output_file = std::ofstream(m_output_file_path, std::ios::out | std::ios::binary);
CHECK(output_file.is_open(), HAILO_OPEN_FILE_FAILURE, "Failed opening output file {} with errno: {}", m_output_file_path, errno);
- output_file.write(reinterpret_cast<char*>(buffer->data()), buffer->size());
+ output_file.write(reinterpret_cast<char*>(buffer.data()), buffer.size());
CHECK(output_file.good(), HAILO_FILE_OPERATION_FAILURE, "Failed writing board config into file {}.", m_output_file_path);
return HAILO_SUCCESS;
hailo_status BoardConfigWriteSubcommand::execute_on_device(Device &device)
{
- auto buffer = read_binary_file(m_input_file_path);
- CHECK_EXPECTED_AS_STATUS(buffer);
-
- hailo_status status = device.write_board_config(MemoryView(buffer.value()));
+ TRY(auto buffer, read_binary_file(m_input_file_path));
+ hailo_status status = device.write_board_config(MemoryView(buffer));
CHECK_SUCCESS(status, "Failed writing board config to device.");
return HAILO_SUCCESS;
}
DeviceCommand::DeviceCommand(CLI::App *app) :
- Command(app)
+ Command(app),
+ m_show_stdout(true)
{
add_device_options(m_app, m_device_params);
}
+void DeviceCommand::pre_execute()
+{
+ // Do nothing by default
+}
+
hailo_status DeviceCommand::execute()
{
+ pre_execute();
+
auto devices = create_devices(m_device_params);
if (!devices) {
return devices.status();
{
auto status = HAILO_SUCCESS; // Best effort
for (auto &device : devices) {
- std::cout << "Executing on device: " << device->get_dev_id() << std::endl;
+ if (m_show_stdout) {
+ std::cout << "Executing on device: " << device->get_dev_id() << std::endl;
+ }
auto execute_status = execute_on_device(*device);
if (HAILO_SUCCESS != execute_status) {
std::cerr << "Failed to execute on device: " << device->get_dev_id() << ". status= " << execute_status << std::endl;
{
if ((1 != m_device_params.device_ids.size()) || contains(m_device_params.device_ids, std::string("*"))) {
// No specific device-id given, make sure there is only 1 device on the machine.
- auto scan_res = Device::scan();
- CHECK_EXPECTED_AS_STATUS(scan_res, "Failed to scan for devices");
- if (1 != scan_res->size()) {
+ TRY(auto scan_res, Device::scan(), "Failed to scan for devices");
+ if (1 != scan_res.size()) {
return HAILO_INVALID_OPERATION;
}
}
protected:
hailo_device_params m_device_params;
+ bool m_show_stdout; // Set to false in subclasses to disable this class' prints to stdout
+ virtual void pre_execute(); // Override this function to do any pre-execution setup
virtual hailo_status execute_on_device(Device &device) = 0;
hailo_status execute_on_devices(std::vector<std::unique_ptr<Device>> &devices);
hailo_status validate_specific_device_is_given();
hailo_status DownloadActionListCommand::execute(Device &device, const std::string &output_file_path,
const ConfiguredNetworkGroupVector &network_groups, const std::string &hef_file_path)
{
- auto expected_action_list_json = init_json_object(device, hef_file_path);
- CHECK_EXPECTED_AS_STATUS(expected_action_list_json);
- auto action_list_json = expected_action_list_json.value();
-
- auto network_groups_list_json = parse_network_groups(device, network_groups);
- CHECK_EXPECTED_AS_STATUS(network_groups_list_json);
- action_list_json["network_groups"] = network_groups_list_json.release();
+ TRY(auto action_list_json, init_json_object(device, hef_file_path));
+ TRY(action_list_json["network_groups"], parse_network_groups(device, network_groups));
return write_to_json(action_list_json, output_file_path);
}
hailo_status DownloadActionListCommand::execute(Device &device, std::shared_ptr<ConfiguredNetworkGroup> network_group,
uint16_t batch_size, ordered_json &action_list_json_param, double fps, uint32_t network_group_index)
{
- auto expected_network_groups_list_json = parse_network_group(device, network_group, network_group_index);
- CHECK_EXPECTED_AS_STATUS(expected_network_groups_list_json);
- auto network_groups_list_json = expected_network_groups_list_json.release();
+ TRY(auto network_groups_list_json, parse_network_group(device, network_group, network_group_index));
network_groups_list_json[0]["batch_size"] = batch_size;
network_groups_list_json[0]["fps"] = fps;
action_list_json_param["runs"] += network_groups_list_json[0];
Expected<ordered_json> DownloadActionListCommand::init_json_object(Device &device, const std::string &hef_file_path)
{
ordered_json action_list_json = {};
- auto curr_time = CliCommon::current_time_to_string();
- CHECK_EXPECTED(curr_time);
+ TRY(auto curr_time, CliCommon::current_time_to_string());
+ TRY(auto chip_arch, device.get_architecture());
- auto chip_arch = device.get_architecture();
- CHECK_EXPECTED(chip_arch);
unsigned int clock_cycle = 0;
// TODO - HRT-8046 Implement extended device info for hailo15
- if (HAILO_ARCH_HAILO15H == chip_arch.value()) {
+ if (HAILO_ARCH_HAILO15H == chip_arch) {
clock_cycle = HAILO15_VPU_CORE_CPU_DEFAULT_FREQ_MHZ;
} else {
- auto extended_info = device.get_extended_device_information();
- CHECK_EXPECTED(extended_info);
- clock_cycle = (extended_info->neural_network_core_clock_rate / NN_CORE_TO_TIMER_FREQ_FACTOR) / MHz;
+ TRY(auto extended_info, device.get_extended_device_information());
+ clock_cycle = (extended_info.neural_network_core_clock_rate / NN_CORE_TO_TIMER_FREQ_FACTOR) / MHz;
}
action_list_json["version"] = ACTION_LIST_FORMAT_VERSION();
- action_list_json["creation_time"] = curr_time.release();
+ action_list_json["creation_time"] = curr_time;
action_list_json["clock_cycle_MHz"] = clock_cycle;
action_list_json["hef"] = json({});
if (!hef_file_path.empty()) {
- auto hef_info = parse_hef_metadata(hef_file_path);
- CHECK_EXPECTED(hef_info);
- action_list_json["hef"] = hef_info.release();
+ TRY(action_list_json["hef"], parse_hef_metadata(hef_file_path));
}
action_list_json["runs"] = ordered_json::array();
CHECK_AS_EXPECTED(is_valid_hef(hef_file_path), HAILO_INTERNAL_FAILURE,
"Hef '{}' is not valid", hef_file_path);
- auto hef_md5 = calc_md5_hexdigest(hef_file_path);
- CHECK_EXPECTED(hef_md5);
+ TRY(auto hef_md5, calc_md5_hexdigest(hef_file_path));
ordered_json hef_info_json = {
{"path", hef_file_path},
- {"file_hash", hef_md5.release()}
+ {"file_hash", hef_md5}
};
return hef_info_json;
Expected<std::string> DownloadActionListCommand::calc_md5_hexdigest(const std::string &hef_file_path)
{
- auto hef_bin = read_binary_file(hef_file_path);
- CHECK_EXPECTED(hef_bin);
+ TRY(auto hef_bin, read_binary_file(hef_file_path));
MD5_CTX md5_ctx{};
MD5_SUM_t md5_sum{};
MD5_Init(&md5_ctx);
- MD5_Update(&md5_ctx, hef_bin->data(), hef_bin->size());
+ MD5_Update(&md5_ctx, hef_bin.data(), hef_bin.size());
MD5_Final(md5_sum, &md5_ctx);
const bool LOWERCASE = false;
data_json = *reinterpret_cast<CONTEXT_SWITCH_DEFS__activate_ddr_buffer_output_data_t *>(action);
action_length_local = sizeof(CONTEXT_SWITCH_DEFS__activate_ddr_buffer_output_data_t);
break;
+ case CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_INPUT:
+ data_json = *reinterpret_cast<CONTEXT_SWITCH_DEFS__activate_cache_input_data_t *>(action);
+ action_length_local = sizeof(CONTEXT_SWITCH_DEFS__activate_cache_input_data_t);
+ break;
+ case CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_OUTPUT:
+ data_json = *reinterpret_cast<CONTEXT_SWITCH_DEFS__activate_cache_output_data_t *>(action);
+ action_length_local = sizeof(CONTEXT_SWITCH_DEFS__activate_cache_output_data_t);
+ break;
+ case CONTEXT_SWITCH_DEFS__ACTION_TYPE_WAIT_FOR_CACHE_UPDATED:
+ data_json = json({});
+ action_length_local = 0;
+ break;
case CONTEXT_SWITCH_DEFS__ACTION_TYPE_CHANGE_VDMA_TO_STREAM_MAPPING:
data_json = *reinterpret_cast<CONTEXT_SWITCH_DEFS__change_vdma_to_stream_mapping_data_t *>(action);
action_length_local = sizeof(CONTEXT_SWITCH_DEFS__change_vdma_to_stream_mapping_data_t);
static const bool DONT_SET_SUB_ACTION_INDEX = false;
uint32_t action_data_length = 0;
- auto json = parse_action_data(base_address, &context_action_list[current_buffer_offset], current_buffer_offset, &action_data_length,
- action_header->action_type, time_stamp_local, 0, DONT_SET_SUB_ACTION_INDEX, is_repeated, num_repeated, sub_action_type);
- CHECK_EXPECTED(json);
+ TRY(auto json, parse_action_data(base_address, &context_action_list[current_buffer_offset], current_buffer_offset, &action_data_length,
+ action_header->action_type, time_stamp_local, 0, DONT_SET_SUB_ACTION_INDEX, is_repeated, num_repeated, sub_action_type));
*action_length = static_cast<uint32_t>(action_length_local + action_data_length);
*time_stamp = time_stamp_local;
- return json.release();
+ return json;
}
Expected<ordered_json> DownloadActionListCommand::parse_context(Device &device, uint32_t network_group_id,
uint32_t action_list_base_address = 0;
uint32_t batch_counter = 0;
- auto action_list = device.download_context_action_list(network_group_id, converted_context_type, context_index,
- &action_list_base_address, &batch_counter);
- CHECK_EXPECTED(action_list);
+ TRY(auto action_list, device.download_context_action_list(network_group_id, converted_context_type, context_index,
+ &action_list_base_address, &batch_counter));
// Needs to fit in 2 bytes due to firmware limitation of action list size
- CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(action_list->size()), HAILO_INTERNAL_FAILURE,
- "Action list size is expected to fit in 2B. actual size is {}", action_list->size());
+ CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(action_list.size()), HAILO_INTERNAL_FAILURE,
+ "Action list size is expected to fit in 2B. actual size is {}", action_list.size());
ordered_json context_json {
{"action_list_base_address", action_list_base_address},
- {"action_list_size", action_list->size() },
+ {"action_list_size", action_list.size() },
{"batch_counter", batch_counter},
{"context_name", context_name},
};
ordered_json action_list_json;
uint16_t current_buffer_offset = 0;
- while (current_buffer_offset < action_list->size()) {
+ while (current_buffer_offset < action_list.size()) {
bool is_repeated = false;
uint8_t num_repeated = 0;
CONTEXT_SWITCH_DEFS__ACTION_TYPE_t sub_action_type = CONTEXT_SWITCH_DEFS__ACTION_TYPE_COUNT;
uint32_t single_action_length = 0;
uint32_t timestamp = 0;
- auto action_json = parse_single_action(action_list_base_address, action_list->data(),
- current_buffer_offset, &single_action_length, &is_repeated, &num_repeated, &sub_action_type, ×tamp);
- CHECK_EXPECTED(action_json);
+ TRY(auto action_json, parse_single_action(action_list_base_address, action_list.data(),
+ current_buffer_offset, &single_action_length, &is_repeated, &num_repeated, &sub_action_type, ×tamp));
current_buffer_offset = (uint16_t)(current_buffer_offset + single_action_length);
- action_list_json.emplace_back(action_json.release());
+ action_list_json.emplace_back(std::move(action_json));
if (is_repeated) {
for (uint8_t index_in_repeated_block = 0; index_in_repeated_block < num_repeated; index_in_repeated_block++) {
uint32_t sub_action_length = 0;
- auto repeated_action_json = parse_single_repeated_action(action_list_base_address,
- action_list->data() + current_buffer_offset, current_buffer_offset, &sub_action_length,
- sub_action_type, timestamp, index_in_repeated_block);
- CHECK_EXPECTED(repeated_action_json);
+ TRY(auto repeated_action_json, parse_single_repeated_action(action_list_base_address,
+ action_list.data() + current_buffer_offset, current_buffer_offset, &sub_action_length,
+ sub_action_type, timestamp, index_in_repeated_block));
current_buffer_offset = (uint16_t)(current_buffer_offset + sub_action_length);
- action_list_json.emplace_back(repeated_action_json.release());
+ action_list_json.emplace_back(std::move(repeated_action_json));
}
}
}
- CHECK_AS_EXPECTED(current_buffer_offset == action_list->size(), HAILO_INTERNAL_FAILURE,
+ CHECK_AS_EXPECTED(current_buffer_offset == action_list.size(), HAILO_INTERNAL_FAILURE,
"PARSING ERROR ! Reached forbidden memory space");
context_json["actions"] = action_list_json;
Expected<ordered_json> DownloadActionListCommand::parse_network_groups(Device &device, const ConfiguredNetworkGroupVector &network_groups)
{
- const auto number_of_dynamic_contexts_per_network_group = device.get_number_of_dynamic_contexts_per_network_group();
- CHECK_EXPECTED(number_of_dynamic_contexts_per_network_group);
+ TRY(const auto number_of_dynamic_contexts_per_network_group, device.get_number_of_dynamic_contexts_per_network_group());
- auto number_of_network_groups = (uint32_t)number_of_dynamic_contexts_per_network_group->size();
+ auto number_of_network_groups = (uint32_t)number_of_dynamic_contexts_per_network_group.size();
ordered_json network_group_list_json;
for (uint32_t network_group_index = 0; network_group_index < number_of_network_groups; network_group_index++) {
auto &network_group = (network_group_index < network_groups.size()) ? network_groups[network_group_index] : nullptr;
- auto expected_json_file = parse_network_group(device, network_group, network_group_index);
- CHECK_EXPECTED(expected_json_file);
- network_group_list_json.emplace_back(expected_json_file.value());
+ TRY(auto json_file, parse_network_group(device, network_group, network_group_index));
+ network_group_list_json.emplace_back(std::move(json_file));
}
return network_group_list_json;
}
Expected<ordered_json> DownloadActionListCommand::parse_network_group(Device &device, const std::shared_ptr<ConfiguredNetworkGroup> network_group, uint32_t network_group_id)
{
- const auto number_of_dynamic_contexts_per_network_group = device.get_number_of_dynamic_contexts_per_network_group();
- CHECK_EXPECTED(number_of_dynamic_contexts_per_network_group);
+ TRY(const auto number_of_dynamic_contexts_per_network_group, device.get_number_of_dynamic_contexts_per_network_group());
ordered_json network_group_list_json;
// TODO: network_group_name via Hef::get_network_groups_names (HRT-5997)
network_group->get_deactivation_time_accumulator());
}
- auto activation_context_json = parse_context(device, network_group_id,
- CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION, 0, "activation");
- CHECK_EXPECTED(activation_context_json);
- network_group_json["contexts"].emplace_back(activation_context_json.release());
+ TRY(auto activation_context_json, parse_context(device, network_group_id,
+ CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION, 0, "activation"));
+ network_group_json["contexts"].emplace_back(std::move(activation_context_json));
- auto preliminary_context_json = parse_context(device, network_group_id,
- CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY, 0, "preliminary");
- CHECK_EXPECTED(preliminary_context_json);
- network_group_json["contexts"].emplace_back(preliminary_context_json.release());
+ TRY(auto preliminary_context_json, parse_context(device, network_group_id,
+ CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY, 0, "preliminary"));
+ network_group_json["contexts"].emplace_back(std::move(preliminary_context_json));
- const auto dynamic_contexts_count = number_of_dynamic_contexts_per_network_group.value()[network_group_id];
+ const auto dynamic_contexts_count = number_of_dynamic_contexts_per_network_group[network_group_id];
for (uint16_t context_index = 0; context_index < dynamic_contexts_count; context_index++) {
- auto context_json = parse_context(device, network_group_id,
+ TRY(auto context_json, parse_context(device, network_group_id,
CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC, context_index,
- fmt::format("dynamic_{}", context_index));
- CHECK_EXPECTED(context_json);
+ fmt::format("dynamic_{}", context_index)));
- network_group_json["contexts"].emplace_back(context_json.release());
+ network_group_json["contexts"].emplace_back(std::move(context_json));
}
- auto batch_switching_context_json = parse_context(device, network_group_id,
- CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING, 0, "batch_switching");
- CHECK_EXPECTED(batch_switching_context_json);
- network_group_json["contexts"].emplace_back(batch_switching_context_json.release());
+ TRY(auto batch_switching_context_json, parse_context(device, network_group_id,
+ CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING, 0, "batch_switching"));
+ network_group_json["contexts"].emplace_back(std::move(batch_switching_context_json));
network_group_list_json.emplace_back(network_group_json);
j["stream_index"] = data.stream_index;
}
+void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_cache_input_data_t &data)
+{
+ j = unpack_vdma_channel_id(data);
+ j["stream_index"] = data.stream_index;
+}
+
+void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_cache_output_data_t &data)
+{
+ j = unpack_vdma_channel_id(data);
+ j["stream_index"] = data.stream_index;
+}
+
+
// Needs to be backwards compatible, so we use "channel_index" instead of "vdma_channel_index".
void to_json(json& j, const CONTEXT_SWITCH_DEFS__fetch_cfg_channel_descriptors_action_data_t& data) {
uint8_t engine_index = 0;
{CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_INTER_CONTEXT_OUTPUT, "activate_inter_context_output"},
{CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_DDR_BUFFER_INPUT, "activate_ddr_buffer_input"},
{CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_DDR_BUFFER_OUTPUT, "activate_ddr_buffer_output"},
+ {CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_INPUT, "activate_cache_input"},
+ {CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_OUTPUT, "activate_cache_output"},
+ {CONTEXT_SWITCH_DEFS__ACTION_TYPE_WAIT_FOR_CACHE_UPDATED, "wait_for_cache_updated"},
{CONTEXT_SWITCH_DEFS__ACTION_TYPE_DEACTIVATE_VDMA_CHANNEL, "deactivate_vdma_channel"},
{CONTEXT_SWITCH_DEFS__ACTION_TYPE_VALIDATE_VDMA_CHANNEL, "validate_vdma_channel"},
{CONTEXT_SWITCH_DEFS__ACTION_TYPE_CHANGE_VDMA_TO_STREAM_MAPPING, "change_vdma_to_stream_mapping"},
void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_boundary_output_data_t &data);
void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_inter_context_output_data_t &data);
void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_ddr_buffer_output_data_t &data);
+void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_cache_input_data_t &data);
+void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_cache_output_data_t &data);
void to_json(json &j, const CONTEXT_SWITCH_DEFS__enable_lcu_action_default_data_t &data);
void to_json(json &j, const CONTEXT_SWITCH_DEFS__enable_lcu_action_non_default_data_t &data);
void to_json(json &j, const CONTEXT_SWITCH_DEFS__disable_lcu_action_data_t &data);
CHECK_SUCCESS(status,
"'fw-config read' command should get a specific device-id.");
- auto user_config_buffer = device.read_user_config();
- CHECK_EXPECTED_AS_STATUS(user_config_buffer, "Failed reading user config from device");
+ TRY(auto user_config_buffer, device.read_user_config(), "Failed reading user config from device");
status = FwConfigJsonSerializer::deserialize_config(
- *reinterpret_cast<USER_CONFIG_header_t*>(user_config_buffer->data()),
- user_config_buffer->size(), m_output_file);
+ *reinterpret_cast<USER_CONFIG_header_t*>(user_config_buffer.data()),
+ user_config_buffer.size(), m_output_file);
CHECK_SUCCESS(status);
return HAILO_SUCCESS;
hailo_status FwConfigWriteSubcommand::execute_on_device(Device &device)
{
- auto config_buffer = Buffer::create(FLASH_USER_CONFIG_SECTION_SIZE);
- CHECK_EXPECTED_AS_STATUS(config_buffer);
-
- auto config_size = FwConfigJsonSerializer::serialize_config(
- *reinterpret_cast<USER_CONFIG_header_t*>(config_buffer->data()), config_buffer->size(), m_input_file);
- CHECK_EXPECTED_AS_STATUS(config_size);
+ TRY(auto config_buffer, Buffer::create(FLASH_USER_CONFIG_SECTION_SIZE));
+ TRY(auto config_size, FwConfigJsonSerializer::serialize_config(
+ *reinterpret_cast<USER_CONFIG_header_t*>(config_buffer.data()), config_buffer.size(), m_input_file));
- // We only need to write config_size.value() bytes from config_buffer, so we "resize" the buffer
- CHECK(config_buffer->size() >= config_size.value(), HAILO_INTERNAL_FAILURE,
- "Unexpected config size {} (max_size={})", config_size.value(), config_buffer->size());
- auto resized_config_buffer = Buffer::create(config_buffer->data(), config_size.value());
- CHECK_EXPECTED_AS_STATUS(resized_config_buffer);
+ // We only need to write 'config_size' bytes from config_buffer, so we "resize" the buffer
+ CHECK(config_buffer.size() >= config_size, HAILO_INTERNAL_FAILURE,
+ "Unexpected config size {} (max_size={})", config_size, config_buffer.size());
+ TRY(auto resized_config_buffer, Buffer::create(config_buffer.data(), config_size));
- hailo_status status = device.write_user_config(MemoryView(resized_config_buffer.value()));
+ hailo_status status = device.write_user_config(MemoryView(resized_config_buffer));
CHECK_SUCCESS(status, "Failed writing user firmware configuration to device");
return HAILO_SUCCESS;
hailo_status FwConfigSerializeSubcommand::execute()
{
- auto config_buffer = Buffer::create(FLASH_USER_CONFIG_SECTION_SIZE);
- CHECK_EXPECTED_AS_STATUS(config_buffer);
+ TRY(auto config_buffer, Buffer::create(FLASH_USER_CONFIG_SECTION_SIZE));
- USER_CONFIG_header_t *config_header = reinterpret_cast<USER_CONFIG_header_t*>(config_buffer->data());
- auto config_size = FwConfigJsonSerializer::serialize_config(*config_header, config_buffer->size(), m_input_file);
- CHECK_EXPECTED_AS_STATUS(config_size);
+ USER_CONFIG_header_t *config_header = reinterpret_cast<USER_CONFIG_header_t*>(config_buffer.data());
+ TRY(auto config_size, FwConfigJsonSerializer::serialize_config(*config_header, config_buffer.size(), m_input_file));
std::ofstream ofs(m_output_file, std::ios::out | std::ios::binary);
CHECK(ofs.good(), HAILO_OPEN_FILE_FAILURE, "Failed opening file: {}, with errno: {}", m_output_file, errno);
- ofs.write(reinterpret_cast<char*>(config_header), config_size.value());
+ ofs.write(reinterpret_cast<char*>(config_header), config_size);
CHECK(ofs.good(), HAILO_FILE_OPERATION_FAILURE,
"Failed writing binary firmware configuration to file: {}, with errno: {}", m_output_file, errno);
hailo_status FwConfigJsonSerializer::deserialize_config(const USER_CONFIG_header_t &user_config_header, size_t config_size, const std::string &file_path)
{
try {
- auto categories = get_deserialize_vector();
- CHECK_EXPECTED_AS_STATUS(categories);
+ TRY(const auto categories, get_deserialize_vector());
ordered_json config_json;
size_t current_deserialized_data_size = 0;
uintptr_t current_entry_offset = (uintptr_t)(&(user_config_header.entries));
for (size_t i = 0; i < user_config_header.entry_count; i++) {
USER_CONFIG_ENTRY_t *config_entry = reinterpret_cast<USER_CONFIG_ENTRY_t*>(current_entry_offset);
- CHECK(config_entry->category < categories->size(), HAILO_INTERNAL_FAILURE,
- "Category id is out of bounds. Category id = {}, Max category id = {}", config_entry->category, (categories->size()-1));
+ CHECK(config_entry->category < categories.size(), HAILO_INTERNAL_FAILURE,
+ "Category id is out of bounds. Category id = {}, Max category id = {}", config_entry->category, (categories.size()-1));
- auto category = categories.value()[config_entry->category];
+ auto category = categories[config_entry->category];
CHECK(config_entry->entry_id < category.size(), HAILO_INTERNAL_FAILURE,
"Entry id is out of bounds. Entry id = {}, Max entry id = {}", config_entry->entry_id, (category.size() - 1));
entry_definition["size"].get<uint32_t>();
if (deserialize_as == "str") {
- auto str_val = deserialize_str(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(str_val);
- config_json[category_name][entry_name] = str_val.value();
+ TRY(config_json[category_name][entry_name], deserialize_str(entry_value, size));
}
else if (deserialize_as == "bool") {
- auto bool_val = deserialize_bool(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(bool_val);
- config_json[category_name][entry_name] = bool_val.value();
+ TRY(config_json[category_name][entry_name], deserialize_bool(entry_value, size));
}
else if (deserialize_as == "int") {
- auto int_val = deserialize_int(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(int_val);
- config_json[category_name][entry_name] = int_val.value();
+ TRY(config_json[category_name][entry_name], deserialize_int(entry_value, size));
}
else if (deserialize_as == "i2c_speed") {
- auto i2c_speed_val = deserialize_i2c_speed(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(i2c_speed_val);
- config_json[category_name][entry_name]["value"] = i2c_speed_val.value();
+ TRY(config_json[category_name][entry_name]["value"], deserialize_i2c_speed(entry_value, size));
}
else if (deserialize_as == "supported_aspm_states") {
- auto supported_aspm_states_val = deserialize_supported_aspm_states(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(supported_aspm_states_val);
- config_json[category_name][entry_name]["value"] = supported_aspm_states_val.value();
+ TRY(config_json[category_name][entry_name]["value"], deserialize_supported_aspm_states(entry_value, size));
}
else if (deserialize_as == "supported_aspm_l1_substates") {
- auto supported_aspm_l1_substates_val = deserialize_supported_aspm_l1_substates(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(supported_aspm_l1_substates_val);
- config_json[category_name][entry_name]["value"] = supported_aspm_l1_substates_val.value();
+ TRY(config_json[category_name][entry_name]["value"],
+ deserialize_supported_aspm_l1_substates(entry_value, size));
}
else if (deserialize_as == "ipv4") {
- auto ipv4_val = deserialize_ipv4(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(ipv4_val);
- config_json[category_name][entry_name]["value"] = ipv4_val.value();
+ TRY(config_json[category_name][entry_name]["value"], deserialize_ipv4(entry_value, size));
}
else if (deserialize_as == "mac_address") {
- auto mac_address_val = deserialize_mac_address(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(mac_address_val);
- config_json[category_name][entry_name]["value"] = mac_address_val.value();
+ TRY(config_json[category_name][entry_name]["value"], deserialize_mac_address(entry_value, size));
}
else if (deserialize_as == "clock_frequency") {
- auto clock_frequency_val = deserialize_clock_frequency(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(clock_frequency_val);
- config_json[category_name][entry_name]["value"] = clock_frequency_val.value();
+ TRY(config_json[category_name][entry_name]["value"],
+ deserialize_clock_frequency(entry_value, size));
}
else if (deserialize_as == "logger_level") {
- auto logger_level = deserialize_logger_level(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(logger_level);
- config_json[category_name][entry_name]["value"] = logger_level.value();
+ TRY(config_json[category_name][entry_name]["value"], deserialize_logger_level(entry_value, size));
}
else if (deserialize_as == "watchdog_mode") {
- auto watchdog_mode_val = deserialize_watchdog_mode(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(watchdog_mode_val);
- config_json[category_name][entry_name]["value"] = watchdog_mode_val.value();
+ TRY(config_json[category_name][entry_name]["value"], deserialize_watchdog_mode(entry_value, size));
}
else if (deserialize_as == "overcurrent_parameters_source") {
- auto overcurrent_parameters_source_val = deserialize_overcurrent_parameters_source(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(overcurrent_parameters_source_val);
- config_json[category_name][entry_name]["value"] = overcurrent_parameters_source_val.value();
+ TRY(config_json[category_name][entry_name]["value"],
+ deserialize_overcurrent_parameters_source(entry_value, size));
}
else if (deserialize_as == "temperature_parameters_source") {
- auto temperature_parameters_source_val = deserialize_temperature_parameters_source(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(temperature_parameters_source_val);
- config_json[category_name][entry_name]["value"] = temperature_parameters_source_val.value();
+ TRY(config_json[category_name][entry_name]["value"],
+ deserialize_temperature_parameters_source(entry_value, size));
}
else if (deserialize_as == "conversion_time") {
- auto conversion_time_val = deserialize_conversion_time(entry_value, size);
- CHECK_EXPECTED_AS_STATUS(conversion_time_val);
- config_json[category_name][entry_name]["value"] = conversion_time_val.value();
+ TRY(config_json[category_name][entry_name]["value"],
+ deserialize_conversion_time(entry_value, size));
}
else {
LOGGER__ERROR("Failed deserializing entry. Serialization format {} not found", deserialize_as);
Expected<json> FwConfigJsonSerializer::deserialize_bool(uint8_t *entry_value, uint32_t size)
{
- auto bool_val = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(bool_val);
- json bool_str = bool_val.value() ? true : false;
+ TRY(const auto bool_val, get_int_value<uint8_t>(entry_value, size));
+ json bool_str = bool_val ? true : false;
return bool_str;
}
Expected<json> FwConfigJsonSerializer::deserialize_supported_aspm_states(uint8_t *entry_value, uint32_t size)
{
- auto aspm_state = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(aspm_state);
+ TRY(const auto aspm_state, get_int_value<uint8_t>(entry_value, size));
- switch (static_cast<PCIE_CONFIG_SUPPOPRTED_ASPM_STATES_t>(aspm_state.value())) {
+ switch (static_cast<PCIE_CONFIG_SUPPOPRTED_ASPM_STATES_t>(aspm_state)) {
case ASPM_DISABLED:
return json("ASPM DISABLED");
case ASPM_L1_ONLY:
Expected<json> FwConfigJsonSerializer::deserialize_supported_aspm_l1_substates(uint8_t *entry_value, uint32_t size)
{
- auto aspm_l1_substate = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(aspm_l1_substate);
+ TRY(const auto aspm_l1_substate, get_int_value<uint8_t>(entry_value, size));
- switch (static_cast<PCIE_CONFIG_SUPPOPRTED_L1_ASPM_SUBSTATES_t>(aspm_l1_substate.value())) {
+ switch (static_cast<PCIE_CONFIG_SUPPOPRTED_L1_ASPM_SUBSTATES_t>(aspm_l1_substate)) {
case ASPM_L1_SUBSTATES_DISABLED:
return json("ASPM L1 SUBSTATES DISABLED");
case ASPM_L1_SUBSTATES_L11_ONLY:
Expected<json> FwConfigJsonSerializer::deserialize_clock_frequency(uint8_t *entry_value, uint32_t size)
{
- auto clock_frequency = get_int_value<uint32_t>(entry_value, size);
- CHECK_EXPECTED(clock_frequency);
+ TRY(const auto clock_frequency, get_int_value<uint32_t>(entry_value, size));
- switch (clock_frequency.value()) {
+ switch (clock_frequency) {
case SOC__NN_CLOCK_400MHz:
return json("400MHZ");
case SOC__NN_CLOCK_375MHz:
Expected<json> FwConfigJsonSerializer::deserialize_watchdog_mode(uint8_t *entry_value, uint32_t size)
{
- auto watchdog_mode = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(watchdog_mode);
+ TRY(const auto watchdog_mode, get_int_value<uint8_t>(entry_value, size));
- switch (static_cast<WD_SERVICE_wd_mode_t>(watchdog_mode.value())) {
+ switch (static_cast<WD_SERVICE_wd_mode_t>(watchdog_mode)) {
case WD_SERVICE_MODE_HW_SW:
return json("WD MODE HW SW");
case WD_SERVICE_MODE_HW_ONLY:
Expected<json> FwConfigJsonSerializer::deserialize_i2c_speed(uint8_t *entry_value, uint32_t size)
{
- auto i2c_speed = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(i2c_speed);
+ TRY(const auto i2c_speed, get_int_value<uint8_t>(entry_value, size));
- switch (static_cast<i2c_speed_mode_t>(i2c_speed.value())) {
+ switch (static_cast<i2c_speed_mode_t>(i2c_speed)) {
case I2C_SPEED_STANDARD:
return json("I2C SPEED STANDARD");
case I2C_SPEED_FAST:
Expected<json> FwConfigJsonSerializer::deserialize_logger_level(uint8_t *entry_value, uint32_t size)
{
- auto logger_level = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(logger_level);
+ TRY(const auto logger_level, get_int_value<uint8_t>(entry_value, size));
- switch (static_cast<FW_LOGGER_LEVEL_t>(logger_level.value())) {
+ switch (static_cast<FW_LOGGER_LEVEL_t>(logger_level)) {
case FW_LOGGER_LEVEL_TRACE:
return json("TRACE");
case FW_LOGGER_LEVEL_DEBUG:
Expected<json> FwConfigJsonSerializer::deserialize_overcurrent_parameters_source(uint8_t *entry_value, uint32_t size)
{
- auto overcurrent_parameters_source = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(overcurrent_parameters_source);
+ TRY(const auto overcurrent_parameters_source, get_int_value<uint8_t>(entry_value, size));
- switch (static_cast<OVERCURRENT_parameters_source_t>(overcurrent_parameters_source.value())) {
+ switch (static_cast<OVERCURRENT_parameters_source_t>(overcurrent_parameters_source)) {
case OVERCURRENT_PARAMETERS_SOURCE_FW_VALUES:
return json("FW VALUES");
case OVERCURRENT_PARAMETERS_SOURCE_USER_CONFIG_VALUES:
Expected<json> FwConfigJsonSerializer::deserialize_temperature_parameters_source(uint8_t *entry_value, uint32_t size)
{
- auto temperature_parameters_source = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(temperature_parameters_source);
+ TRY(const auto temperature_parameters_source, get_int_value<uint8_t>(entry_value, size));
- switch (static_cast<TEMPERATURE_PROTECTION_parameters_source_t>(temperature_parameters_source.value())) {
+ switch (static_cast<TEMPERATURE_PROTECTION_parameters_source_t>(temperature_parameters_source)) {
case TEMPERATURE_PROTECTION_PARAMETERS_SOURCE_FW_VALUES:
return json("FW VALUES");
case TEMPERATURE_PROTECTION_PARAMETERS_SOURCE_USER_CONFIG_VALUES:
Expected<json> FwConfigJsonSerializer::deserialize_conversion_time(uint8_t *entry_value, uint32_t size)
{
- auto conversion_time = get_int_value<uint32_t>(entry_value, size);
- CHECK_EXPECTED(conversion_time);
- auto conversion_time_value = static_cast<OVERCURRENT_conversion_time_us_t>(conversion_time.value());
+ TRY(const auto conversion_time, get_int_value<uint32_t>(entry_value, size));
+ auto conversion_time_value = static_cast<OVERCURRENT_conversion_time_us_t>(conversion_time);
if (conversion_time_value == OVERCURRENT_CONVERSION_PERIOD_140US ||
conversion_time_value == OVERCURRENT_CONVERSION_PERIOD_204US ||
switch (size) {
case sizeof(uint8_t):
{
- auto uint8_val = get_int_value<uint8_t>(entry_value, size);
- CHECK_EXPECTED(uint8_val);
- return json(uint8_val.value());
+ TRY(const auto uint8_val, get_int_value<uint8_t>(entry_value, size));
+ return json(uint8_val);
}
case sizeof(uint16_t):
{
- auto uint16_val = get_int_value<uint16_t>(entry_value, size);
- CHECK_EXPECTED(uint16_val);
- return json(uint16_val.value());
+ TRY(const auto uint16_val, get_int_value<uint16_t>(entry_value, size));
+ return json(uint16_val);
}
case sizeof(uint32_t):
{
- auto uint32_val = get_int_value<uint32_t>(entry_value, size);
- CHECK_EXPECTED(uint32_val);
- return json(uint32_val.value());
+ TRY(const auto uint32_val, get_int_value<uint32_t>(entry_value, size));
+ return json(uint32_val);
}
default:
LOGGER__ERROR("Failed deserializing int value");
size_t data_size = sizeof(USER_CONFIG_header_t);
try {
- auto config_json = FwConfigJsonSerializer::read_json_file(file_path);
- CHECK_EXPECTED(config_json);
+ TRY_V(const auto config_json, FwConfigJsonSerializer::read_json_file(file_path));
+ TRY(auto definitions, FwConfigJsonSerializer::get_serialize_map());
- auto definitions = FwConfigJsonSerializer::get_serialize_map();
- CHECK_EXPECTED(definitions);
-
- user_config_header.version = definitions.value()["version"]["value"].get<uint32_t>();
+ user_config_header.version = definitions["version"]["value"].get<uint32_t>();
user_config_header.magic = USER_CONFIG_MAGIC;
user_config_header.entry_count = 0;
uintptr_t current_entry_offset = (uintptr_t)(&(user_config_header.entries));
- for (auto &config_category : config_json->items()) {
+ for (auto &config_category : config_json.items()) {
for (auto &config_entry : config_category.value().items()) {
- ordered_json entry_definition = definitions.value()[config_category.key()][config_entry.key()];
+ ordered_json entry_definition = definitions[config_category.key()][config_entry.key()];
USER_CONFIG_ENTRY_t *curr_entry = (USER_CONFIG_ENTRY_t *)current_entry_offset;
curr_entry->entry_size = entry_definition.contains("length") ?
(entry_definition["length"].get<uint32_t>() * entry_definition["size"].get<uint32_t>()) :
CHECK(entry.entry_size >= str.length(), HAILO_INVALID_ARGUMENT,
"Failed serializing string value {}. String length must be equal or shorter than {}", str, entry.entry_size);
+ memset(&(entry.value), 0, entry.entry_size);
memcpy(&(entry.value), str.c_str(), str.length());
return HAILO_SUCCESS;
\r
static hailo_status print_extended_device_information(Device &device)\r
{\r
- auto extended_info_expected = device.get_extended_device_information();\r
- CHECK_EXPECTED_AS_STATUS(extended_info_expected, "Failed identify");\r
- auto device_info = extended_info_expected.release();\r
+ TRY(auto device_info, device.get_extended_device_information());\r
\r
// Print Board Extended information\r
std::cout << "Boot source: " << extended_device_information_boot_string(device_info.boot_source) << std::endl;\r
return "PLUTO";\r
case HAILO_ARCH_HAILO15M:\r
return "HAILO15M";\r
+ case HAILO_ARCH_HAILO10H:\r
+ return "HAILO10H";\r
default:\r
return "Unknown";\r
}\r
\r
hailo_status FwControlIdentifyCommand::execute_on_device(Device &device)\r
{\r
- auto identity_expected = device.identify();\r
- CHECK_EXPECTED_AS_STATUS(identity_expected, "Failed identify");\r
- auto identity = identity_expected.release();\r
+ TRY(const auto identity, device.identify());\r
\r
// Print board information\r
std::cout << "Identifying board" << std::endl;\r
FwLoggerCommand::FwLoggerCommand(CLI::App &parent_app) :
DeviceCommand(parent_app.add_subcommand("fw-logger", "Download fw logs to a file")),
- m_should_overwrite(false)
+ m_should_overwrite(false),
+ m_stdout(false),
+ m_continuos(false)
{
m_app->add_option("output_file", m_output_file, "File path to write binary firmware log into")
->required();
m_app->add_flag("--overwrite", m_should_overwrite, "Should overwrite the file or not");
+ m_app->add_flag("--stdout", m_stdout, "Write the output to stdout instead of a file");
+ m_app->add_flag("--continuos", m_continuos, "Write to file/stdout, until the process is killed");
}
-hailo_status write_logs_to_file(Device &device, std::ofstream &ofs, hailo_cpu_id_t cpu_id){
+hailo_status FwLoggerCommand::write_logs(Device &device, std::ostream *os, hailo_cpu_id_t cpu_id)
+{
auto still_has_logs = true;
static const auto buffer_size = AMOUNT_OF_BYTES_TO_READ;
-
- auto expected_buffer = Buffer::create(buffer_size);
- CHECK_EXPECTED_AS_STATUS(expected_buffer);
- Buffer buffer = expected_buffer.release();
- while(still_has_logs) {
- MemoryView response_view(buffer);
- auto response_size_expected = device.read_log(response_view, cpu_id);
- CHECK_EXPECTED_AS_STATUS(response_size_expected);
+ TRY(auto buffer, Buffer::create(buffer_size));
- auto response_size = response_size_expected.release();
+ while (still_has_logs || m_continuos) {
+ MemoryView response_view(buffer);
+ TRY(const auto response_size, device.read_log(response_view, cpu_id));
if (response_size == 0) {
still_has_logs = false;
- }
- else {
- ofs.write((char *)buffer.data(), response_size);
- CHECK(ofs.good(), HAILO_FILE_OPERATION_FAILURE,
+ } else {
+ os->write((char *)buffer.data(), response_size);
+ CHECK(os->good(), HAILO_FILE_OPERATION_FAILURE,
"Failed writing firmware logger to output file, with errno: {}", errno);
+ os->flush();
}
}
return HAILO_SUCCESS;
}
+void FwLoggerCommand::pre_execute()
+{
+ if (m_stdout) {
+ // We want only the binary data from the logger to be written to stdout
+ DeviceCommand::m_show_stdout = false;
+ }
+}
+
hailo_status FwLoggerCommand::execute_on_device(Device &device)
{
auto status = validate_specific_device_is_given();
CHECK_SUCCESS(status,
"'fw-logger' command should get a specific device-id");
-
- auto ofs_flags = std::ios::out | std::ios::binary;
- if (!m_should_overwrite){
- ofs_flags |= std::ios::app;
+ // Initialization dependency
+ std::ofstream ofs;
+ std::ostream *os = nullptr;
+ if (m_stdout) {
+ os = &std::cout;
+ } else {
+ auto ofs_flags = std::ios::out | std::ios::binary;
+ if (!m_should_overwrite){
+ ofs_flags |= std::ios::app;
+ }
+ ofs.open(m_output_file, ofs_flags);
+ CHECK(ofs.good(), HAILO_OPEN_FILE_FAILURE, "Failed opening file: {}, with errno: {}", m_output_file, errno);
+ os = &ofs;
}
- std::ofstream ofs(m_output_file, ofs_flags);
- CHECK(ofs.good(), HAILO_OPEN_FILE_FAILURE, "Failed opening file: {}, with errno: {}", m_output_file, errno);
-
if (Device::Type::ETH == device.get_type()) {
LOGGER__ERROR("Read FW log is not supported over Eth device");
return HAILO_INVALID_OPERATION;
}
-
+
if (Device::Type::INTEGRATED != device.get_type()) {
- status = write_logs_to_file(device, ofs, HAILO_CPU_ID_0);
+ status = write_logs(device, os, HAILO_CPU_ID_0);
if (status != HAILO_SUCCESS){
return status;
}
}
- status = write_logs_to_file(device, ofs, HAILO_CPU_ID_1);
+ status = write_logs(device, os, HAILO_CPU_ID_1);
if (status != HAILO_SUCCESS){
return status;
}
explicit FwLoggerCommand(CLI::App &parent_app);
protected:
+ virtual void pre_execute() override;
virtual hailo_status execute_on_device(Device &device) override;
private:
std::string m_output_file;
bool m_should_overwrite;
+ bool m_stdout;
+ bool m_continuos;
+
+ hailo_status write_logs(Device &device, std::ostream *os, hailo_cpu_id_t cpu_id);
};
#endif /* _HAILO_FW_LOGGER_COMMAND_COMMAND_HPP_ */
{
std::vector<std::unique_ptr<Device>> res;
- auto device_ids = get_device_ids(device_params);
- CHECK_EXPECTED(device_ids);
-
- for (auto device_id : device_ids.value()) {
- auto device = Device::create(device_id);
- CHECK_EXPECTED(device);
- res.emplace_back(device.release());
+ TRY(const auto device_ids, get_device_ids(device_params));
+ for (auto device_id : device_ids) {
+ TRY(auto device, Device::create(device_id));
+ res.emplace_back(std::move(device));
}
return res;
hailo_status HwInferEstimatorCommand::execute()
{
- auto devices = create_devices(m_params.vdevice_params.device_params);
- CHECK_EXPECTED_AS_STATUS(devices, "Failed creating device");
+ TRY(auto devices, create_devices(m_params.vdevice_params.device_params), "Failed creating device");
/* This function supports controls for multiple devices.
We validate there is only 1 device generated as we are on a single device flow */
- CHECK(1 == devices->size(), HAILO_INTERNAL_FAILURE, "Hw infer command support only one physical device");
- auto &device = devices.value()[0];
+ CHECK(1 == devices.size(), HAILO_INTERNAL_FAILURE, "Hw infer command support only one physical device");
+ auto &device = devices[0];
- auto hef = Hef::create(m_params.hef_path.c_str());
- CHECK_EXPECTED_AS_STATUS(hef, "Failed reading hef file {}", m_params.hef_path);
+ TRY(auto hef,
+ Hef::create(m_params.hef_path.c_str()), "Failed reading hef file {}", m_params.hef_path);
- auto interface = device->get_default_streams_interface();
- CHECK_EXPECTED_AS_STATUS(interface, "Failed to get default streams interface");
+ TRY(const auto interface, device->get_default_streams_interface(), "Failed to get default streams interface");
- auto configure_params = get_configure_params(m_params, hef.value(), interface.value());
- CHECK_EXPECTED_AS_STATUS(configure_params);
+ TRY(auto configure_params, get_configure_params(m_params, hef, interface));
/* Use Env var to configure all desc list with max depth */
setenv("HAILO_CONFIGURE_FOR_HW_INFER","Y",1);
- auto network_group_list = device->configure(hef.value(), configure_params.value());
- CHECK_EXPECTED_AS_STATUS(network_group_list, "Failed configure device from hef");
+ TRY(auto network_group_list,
+ device->configure(hef, configure_params), "Failed configure device from hef");
unsetenv("HAILO_CONFIGURE_FOR_HW_INFER");
- CHECK(1 == network_group_list->size(), HAILO_INVALID_OPERATION,
+ CHECK(1 == network_group_list.size(), HAILO_INVALID_OPERATION,
"HW Inference is not supported on HEFs with multiple network groups");
- auto network_group_ptr = network_group_list.value()[0];
+ auto network_group_ptr = network_group_list[0];
std::cout << "Starting HW infer Estimator..." << std::endl;
-
- auto results = network_group_ptr->run_hw_infer_estimator();
- CHECK_EXPECTED_AS_STATUS(results);
+ TRY(const auto results, network_group_ptr->run_hw_infer_estimator());
std::cout << std::endl;
std::cout << "======================" << std::endl;
std::cout << " Summary" << std::endl;
std::cout << "======================" << std::endl;
- std::cout << "Batch count: " << results->batch_count << std::endl;
- std::cout << "Total transfer size [KB]: " << (results->total_transfer_size / BYTES_TO_KILOBYTES) << std::endl;
- std::cout << "Total frames passed: " << results->total_frames_passed << std::endl;
- std::cout << "Total time [s]: " << results->time_sec << std::endl;
- std::cout << "Total FPS [1/s]: " << results->fps << std::endl;
- std::cout << "BW [Gbps]: " << results->BW_Gbps << std::endl;
+ std::cout << "Batch count: " << results.batch_count << std::endl;
+ std::cout << "Total transfer size [KB]: " << (results.total_transfer_size / BYTES_TO_KILOBYTES) << std::endl;
+ std::cout << "Total frames passed: " << results.total_frames_passed << std::endl;
+ std::cout << "Total time [s]: " << results.time_sec << std::endl;
+ std::cout << "Total FPS [1/s]: " << results.fps << std::endl;
+ std::cout << "BW [Gbps]: " << results.BW_Gbps << std::endl;
std::cout << "======================" << std::endl;
std::cout << " End of report" << std::endl;
signal(SIGINT, signit_handler);
std::chrono::milliseconds time_interval = DEFAULT_SCHEDULER_MON_INTERVAL + EPSILON_TIME;
- auto terminal_line_width_expected = get_terminal_line_width();
- CHECK_EXPECTED_AS_STATUS(terminal_line_width_expected);
- auto terminal_line_width = terminal_line_width_expected.release();
+ TRY(const auto terminal_line_width, get_terminal_line_width());
AlternativeTerminal alt_terminal;
while (keep_running) {
bool print_warning_msg = true; // Will change to false only if mon directory is valid and there are updated files in it.
-
- auto mon_dir_valid = Filesystem::is_directory(SCHEDULER_MON_TMP_DIR);
- CHECK_EXPECTED_AS_STATUS(mon_dir_valid);
+ TRY(const auto mon_dir_valid, Filesystem::is_directory(SCHEDULER_MON_TMP_DIR));
std::vector<ProtoMon> mon_messages;
- if (mon_dir_valid.value()) {
- auto scheduler_mon_files = Filesystem::get_latest_files_in_dir_flat(SCHEDULER_MON_TMP_DIR, time_interval);
- CHECK_EXPECTED_AS_STATUS(scheduler_mon_files);
- print_warning_msg = scheduler_mon_files->empty();
+ if (mon_dir_valid) {
+ TRY(auto scheduler_mon_files, Filesystem::get_latest_files_in_dir_flat(SCHEDULER_MON_TMP_DIR, time_interval));
+ print_warning_msg = scheduler_mon_files.empty();
- mon_messages.reserve(scheduler_mon_files->size());
- for (const auto &mon_file : scheduler_mon_files.value()) {
+ mon_messages.reserve(scheduler_mon_files.size());
+ for (const auto &mon_file : scheduler_mon_files) {
auto file = LockedFile::create(mon_file, "r");
if (HAILO_SUCCESS != file.status()) {
LOGGER__ERROR("Failed to open and lock file {}, with status: {}", mon_file, file.status());
hailo_status ParseHefCommand::execute()
{
- auto is_dir = Filesystem::is_directory(m_hef_path.c_str());
- CHECK_EXPECTED_AS_STATUS(is_dir, "Failed checking if path is directory");
-
- if (is_dir.value()){
+ TRY(const auto is_dir, Filesystem::is_directory(m_hef_path.c_str()), "Failed checking if path is directory");
+ if (is_dir) {
return ParseHefCommand::parse_hefs_infos_dir(m_hef_path, m_parse_streams, m_parse_vstreams);
} else {
return ParseHefCommand::parse_hefs_info(m_hef_path, m_parse_streams, m_parse_vstreams);
hailo_status ParseHefCommand::parse_hefs_info(const std::string &hef_path, bool stream_infos, bool vstream_infos)
{
- auto hef_exp = Hef::create(hef_path);
- CHECK_EXPECTED_AS_STATUS(hef_exp, "Failed to parse HEF");
- auto hef = hef_exp.release();
-
- auto hef_info = hef.get_description(stream_infos, vstream_infos);
- CHECK_EXPECTED_AS_STATUS(hef_info, "Failed to parse HEF");
- std::cout << hef_info.release();
+ TRY(const auto hef, Hef::create(hef_path));
+ TRY(const auto hef_info, hef.get_description(stream_infos, vstream_infos));
+ std::cout << hef_info;
return HAILO_SUCCESS;
}
{
bool contains_hef = false;
std::string hef_dir = hef_path;
- const auto files = Filesystem::get_files_in_dir_flat(hef_dir);
- CHECK_EXPECTED_AS_STATUS(files);
+ TRY(const auto files, Filesystem::get_files_in_dir_flat(hef_dir));
- for (const auto &full_path : files.value()) {
+ for (const auto &full_path : files) {
if (Filesystem::has_suffix(full_path, ".hef")) {
contains_hef = true;
std::cout << std::string(80, '*') << std::endl << "Parsing " << full_path << ":"<< std::endl;
{
const uint8_t const_byte = 0xAB;
- auto constant_buffer = Buffer::create_shared(frame_size, const_byte, BufferStorageParams::create_dma());
- CHECK_EXPECTED(constant_buffer);
+ TRY(auto constant_buffer,
+ Buffer::create_shared(frame_size, const_byte, BufferStorageParams::create_dma()));
- return std::vector<BufferPtr>{constant_buffer.release()};
+ return std::vector<BufferPtr>{ constant_buffer };
}
static Expected<std::vector<BufferPtr>> create_dataset_from_input_file(const std::string &file_path, size_t frame_size)
{
- auto buffer = read_binary_file(file_path);
- CHECK_EXPECTED(buffer);
- CHECK_AS_EXPECTED(0 == (buffer->size() % frame_size), HAILO_INVALID_ARGUMENT,
+ TRY(auto buffer, read_binary_file(file_path));
+ CHECK_AS_EXPECTED(0 == (buffer.size() % frame_size), HAILO_INVALID_ARGUMENT,
"Input file ({}) size {} must be a multiple of the frame size {}",
- file_path, buffer->size(), frame_size);
+ file_path, buffer.size(), frame_size);
std::vector<BufferPtr> dataset;
- const size_t frames_count = buffer->size() / frame_size;
+ const size_t frames_count = buffer.size() / frame_size;
dataset.reserve(frames_count);
for (size_t i = 0; i < frames_count; i++) {
const auto offset = frame_size * i;
- auto frame_buffer = Buffer::create_shared(buffer->data() + offset, frame_size, BufferStorageParams::create_dma());
- CHECK_EXPECTED(frame_buffer);
- dataset.emplace_back(frame_buffer.release());
+ TRY(auto frame_buffer,
+ Buffer::create_shared(buffer.data() + offset, frame_size, BufferStorageParams::create_dma()));
+ dataset.emplace_back(frame_buffer);
}
return dataset;
static Expected<std::vector<DmaMappedBuffer>> dma_map_dataset(const std::vector<BufferPtr> &dataset, VDevice &vdevice) {
std::vector<DmaMappedBuffer> dataset_mapped_buffers;
for (const auto &buffer : dataset) {
- auto mapped_buffer = DmaMappedBuffer::create(vdevice, buffer->data(), buffer->size(), HAILO_DMA_BUFFER_DIRECTION_H2D);
- CHECK_EXPECTED(mapped_buffer);
- dataset_mapped_buffers.emplace_back(mapped_buffer.release());
+ TRY(auto mapped_buffer,
+ DmaMappedBuffer::create(vdevice, buffer->data(), buffer->size(), HAILO_DMA_BUFFER_DIRECTION_H2D));
+ dataset_mapped_buffers.emplace_back(std::move(mapped_buffer));
}
return dataset_mapped_buffers;
}
CHECK_AS_EXPECTED(contains(m_tracks, NETWORK_STATS_LEVEL), HAILO_NOT_AVAILABLE);
for (size_t network_stats_track_index = 0; network_stats_track_index < m_tracks[NETWORK_STATS_LEVEL].size(); network_stats_track_index++) {
- auto expected_fps = m_tracks[NETWORK_STATS_LEVEL][network_stats_track_index]->get_last_measured_fps();
- CHECK_EXPECTED(expected_fps);
- last_measured_fpss.emplace_back(expected_fps.release());
+ TRY(auto fps,
+ m_tracks[NETWORK_STATS_LEVEL][network_stats_track_index]->get_last_measured_fps());
+ last_measured_fpss.emplace_back(fps);
}
return last_measured_fpss;
{
std::shared_ptr<PowerMeasurement> power_measurement = nullptr;
if (measure_power) {
- auto power_measurement_exp = PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__POWER);
- CHECK_EXPECTED(power_measurement_exp);
- power_measurement = power_measurement_exp.release();
+ TRY(power_measurement,
+ PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__POWER));
}
std::shared_ptr<PowerMeasurement> current_measurement = nullptr;
if (measure_current) {
- auto current_measurement_exp = PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__CURRENT);
- CHECK_EXPECTED(current_measurement_exp);
- current_measurement = current_measurement_exp.release();
+ TRY(current_measurement,
+ PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__CURRENT));
}
std::shared_ptr<TemperatureMeasurement> temp_measurement = nullptr;
if (measure_temp) {
- auto temp_measurement_exp = TemperatureMeasurement::create_shared(device);
- CHECK_EXPECTED(temp_measurement_exp);
- temp_measurement = temp_measurement_exp.release();
+ TRY(temp_measurement, TemperatureMeasurement::create_shared(device));
}
auto ptr = make_shared_nothrow<MeasurementLiveTrack>(power_measurement, current_measurement, temp_measurement, device.get_dev_id());
Expected<std::shared_ptr<FullAsyncNetworkRunner>> FullAsyncNetworkRunner::create_shared(VDevice &vdevice,
NetworkParams params)
{
- auto infer_model = vdevice.create_infer_model(params.hef_path);
- CHECK_EXPECTED(infer_model);
- auto infer_model_ptr = infer_model.release();
-
- auto expected_net_group_name = get_network_group_name(params, infer_model_ptr->hef());
- CHECK_EXPECTED(expected_net_group_name);
+ TRY(auto infer_model_ptr, vdevice.create_infer_model(params.hef_path));
+ TRY(auto net_group_name, get_network_group_name(params, infer_model_ptr->hef()));
/* Configure Params */
infer_model_ptr->set_batch_size(params.batch_size);
});
auto input_params = (input_params_it == params.vstream_params.end()) ? VStreamParams() : *input_params_it;
- auto input_config = infer_model_ptr->input(input_name);
- CHECK_EXPECTED(input_config);
- input_config->set_format_order(input_params.params.user_buffer_format.order);
- input_config->set_format_type(input_params.params.user_buffer_format.type);
+ TRY(auto input_config, infer_model_ptr->input(input_name));
+ input_config.set_format_order(input_params.params.user_buffer_format.order);
+ input_config.set_format_type(input_params.params.user_buffer_format.type);
}
for (const auto &output_name : infer_model_ptr->get_output_names()) {
auto output_params_it = std::find_if(params.vstream_params.begin(), params.vstream_params.end(),
});
auto output_params = (output_params_it == params.vstream_params.end()) ? VStreamParams() : *output_params_it;
- auto output_config = infer_model_ptr->output(output_name);
- CHECK_EXPECTED(output_config);
- output_config->set_format_order(output_params.params.user_buffer_format.order);
- output_config->set_format_type(output_params.params.user_buffer_format.type);
+ TRY(auto output_config, infer_model_ptr->output(output_name));
+ output_config.set_format_order(output_params.params.user_buffer_format.order);
+ output_config.set_format_type(output_params.params.user_buffer_format.type);
}
- auto configured_model = infer_model_ptr->configure();
- CHECK_EXPECTED(configured_model);
- auto configured_infer_model_ptr = make_shared_nothrow<ConfiguredInferModel>(configured_model.release());
+ TRY(auto configured_model, infer_model_ptr->configure());
+ auto configured_infer_model_ptr = make_shared_nothrow<ConfiguredInferModel>(std::move(configured_model));
CHECK_NOT_NULL_AS_EXPECTED(configured_infer_model_ptr, HAILO_OUT_OF_HOST_MEMORY);
- auto res = make_shared_nothrow<FullAsyncNetworkRunner>(params, expected_net_group_name.value(), vdevice,
+ auto res = make_shared_nothrow<FullAsyncNetworkRunner>(params, net_group_name, vdevice,
infer_model_ptr, configured_infer_model_ptr);
CHECK_NOT_NULL_AS_EXPECTED(res, HAILO_OUT_OF_HOST_MEMORY);
std::shared_ptr<NetworkRunner> net_runner_ptr = nullptr;
if (InferenceMode::FULL_ASYNC == final_net_params.mode) {
- auto runner_exp = FullAsyncNetworkRunner::create_shared(vdevice, final_net_params);
- CHECK_EXPECTED(runner_exp);
- net_runner_ptr = runner_exp.release();
+ TRY(net_runner_ptr, FullAsyncNetworkRunner::create_shared(vdevice, final_net_params));
} else {
- auto hef = Hef::create(final_net_params.hef_path);
- CHECK_EXPECTED(hef);
-
- auto expected_net_group_name = get_network_group_name(final_net_params, hef.value());
- CHECK_EXPECTED(expected_net_group_name);
-
- auto cfg_params = vdevice.create_configure_params(hef.value(), expected_net_group_name.value());
- CHECK_EXPECTED(cfg_params);
- cfg_params->batch_size = final_net_params.batch_size;
+ TRY(auto hef, Hef::create(final_net_params.hef_path));
+ TRY(auto net_group_name, get_network_group_name(final_net_params, hef));
+ TRY(auto cfg_params, vdevice.create_configure_params(hef, net_group_name));
+ cfg_params.batch_size = final_net_params.batch_size;
if (final_net_params.batch_size == HAILO_DEFAULT_BATCH_SIZE) {
// Changing batch_size to 1 (after configuring the vdevice) - as we iterate over 'final_net_params.batch_size' in latency measurements scenarios
final_net_params.batch_size = 1;
}
if (final_net_params.measure_hw_latency) {
- cfg_params->latency |= HAILO_LATENCY_MEASURE;
+ cfg_params.latency |= HAILO_LATENCY_MEASURE;
}
if (final_net_params.is_async()) {
- for (auto &stream_name_params_pair : cfg_params->stream_params_by_name) {
+ for (auto &stream_name_params_pair : cfg_params.stream_params_by_name) {
stream_name_params_pair.second.flags = HAILO_STREAM_FLAGS_ASYNC;
}
}
- auto cfgr_net_groups = vdevice.configure(hef.value(), {{expected_net_group_name.value(), cfg_params.value()}});
- CHECK_EXPECTED(cfgr_net_groups);
- assert(1 == cfgr_net_groups->size());
- auto cfgr_net_group = cfgr_net_groups.value()[0];
+ TRY(auto cfgr_net_groups, vdevice.configure(hef, {{ net_group_name, cfg_params }}));
+ assert(1 == cfgr_net_groups.size());
+ auto cfgr_net_group = cfgr_net_groups[0];
if (HAILO_SCHEDULING_ALGORITHM_NONE != final_net_params.scheduling_algorithm) {
CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_threshold(final_net_params.scheduler_threshold));
for (auto &vstream_params : final_net_params.vstream_params) {
vstreams_params.emplace(vstream_params.name, vstream_params.params);
}
- auto vstreams = create_vstreams(*cfgr_net_group, vstreams_params);
- CHECK_EXPECTED(vstreams);
+ TRY(auto vstreams, create_vstreams(*cfgr_net_group, vstreams_params));
- auto net_runner = make_shared_nothrow<FullSyncNetworkRunner>(final_net_params, expected_net_group_name.value(), vdevice,
- std::move(vstreams->first), std::move(vstreams->second), cfgr_net_group);
+ auto net_runner = make_shared_nothrow<FullSyncNetworkRunner>(final_net_params, net_group_name, vdevice,
+ std::move(vstreams.first), std::move(vstreams.second), cfgr_net_group);
CHECK_NOT_NULL_AS_EXPECTED(net_runner, HAILO_OUT_OF_HOST_MEMORY);
net_runner_ptr = std::static_pointer_cast<NetworkRunner>(net_runner);
break;
auto output_streams = cfgr_net_group->get_output_streams();
CHECK_AS_EXPECTED(output_streams.size() > 0, HAILO_INTERNAL_FAILURE);
- auto net_runner = make_shared_nothrow<RawNetworkRunner>(final_net_params, expected_net_group_name.value(), vdevice,
+ auto net_runner = make_shared_nothrow<RawNetworkRunner>(final_net_params, net_group_name, vdevice,
std::move(input_streams), std::move(output_streams), cfgr_net_group);
CHECK_NOT_NULL_AS_EXPECTED(net_runner, HAILO_OUT_OF_HOST_MEMORY);
net_runner_ptr = std::static_pointer_cast<NetworkRunner>(net_runner);
if (!ang_exp) {
activation_barrier.terminate();
}
- CHECK_EXPECTED_AS_STATUS(ang_exp);
+ CHECK_EXPECTED_AS_STATUS(ang_exp); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
ang = ang_exp.release();
}
}
if ((InferenceMode::RAW_ASYNC_SINGLE_THREAD == m_params.mode) || (InferenceMode::FULL_ASYNC == m_params.mode)) {
return run_single_thread_async_infer(shutdown_event, net_live_track);
} else {
- auto threads = start_inference_threads(shutdown_event, net_live_track);
- CHECK_EXPECTED_AS_STATUS(threads);
+ TRY(auto threads, start_inference_threads(shutdown_event, net_live_track));
CHECK_SUCCESS(shutdown_event->wait(HAILO_INFINITE_TIMEOUT));
stop();
- return wait_for_threads(threads.value());
+ return wait_for_threads(threads);
}
}
size_t match_count = 0;
std::map<std::string, hailo_vstream_params_t> input_vstreams_params;
- auto input_vstreams_info = net_group.get_input_vstream_infos();
- CHECK_EXPECTED(input_vstreams_info);
- for (auto &input_vstream_info : input_vstreams_info.value()) {
+ TRY(auto input_vstreams_info, net_group.get_input_vstream_infos());
+ for (auto &input_vstream_info : input_vstreams_info) {
if (params.end() != params.find(input_vstream_info.name)) {
match_count++;
input_vstreams_params.emplace(input_vstream_info.name, params.at(input_vstream_info.name));
}
std::map<std::string, hailo_vstream_params_t> output_vstreams_params;
- auto output_vstreams_info = net_group.get_output_vstream_infos();
- CHECK_EXPECTED(output_vstreams_info);
- for (auto &output_vstream_info : output_vstreams_info.value()) {
+ TRY(auto output_vstreams_info, net_group.get_output_vstream_infos());
+ for (auto &output_vstream_info : output_vstreams_info) {
if (params.end() != params.find(output_vstream_info.name)) {
match_count++;
output_vstreams_params.emplace(output_vstream_info.name, params.at(output_vstream_info.name));
CHECK(match_count == params.size(), make_unexpected(HAILO_INVALID_ARGUMENT), "One of the params has an invalid vStream name");
- auto input_vstreams = VStreamsBuilder::create_input_vstreams(net_group, input_vstreams_params);
- CHECK_EXPECTED(input_vstreams);
+ TRY(auto input_vstreams, VStreamsBuilder::create_input_vstreams(net_group, input_vstreams_params));
+ TRY(auto output_vstreams, VStreamsBuilder::create_output_vstreams(net_group, output_vstreams_params));
- auto output_vstreams = VStreamsBuilder::create_output_vstreams(net_group, output_vstreams_params);
- CHECK_EXPECTED(output_vstreams);
-
- return {{input_vstreams.release(), output_vstreams.release()}};//TODO: move? copy elision?
+ return std::make_pair(std::move(input_vstreams), std::move(output_vstreams));
}
const std::vector<hailo_status> NetworkRunner::ALLOWED_INFERENCE_RETURN_VALUES{
std::vector<AsyncThreadPtr<hailo_status>> threads;
for (auto &input_vstream : m_input_vstreams) {
const auto vstream_params = get_params(input_vstream.name());
- auto writer = WriterWrapper<InputVStream>::create(input_vstream, vstream_params, m_vdevice,
- m_overall_latency_meter, m_params.framerate, SYNC_API);
- CHECK_EXPECTED(writer);
+ TRY(auto writer, WriterWrapper<InputVStream>::create(input_vstream, vstream_params, m_vdevice,
+ m_overall_latency_meter, m_params.framerate, SYNC_API));
threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("WRITE",
- [this, writer = writer.release(), shutdown_event]() mutable {
+ [this, writer, shutdown_event]() mutable {
return run_write(writer, shutdown_event, m_latency_barrier);
}));
}
bool first = true; //TODO: check with multiple outputs
for (auto &output_vstream : m_output_vstreams) {
- auto reader = ReaderWrapper<OutputVStream>::create(output_vstream, m_vdevice,
- m_overall_latency_meter, first ? net_live_track : nullptr, SYNC_API);
- CHECK_EXPECTED(reader);
+ TRY(auto reader, ReaderWrapper<OutputVStream>::create(output_vstream, m_vdevice,
+ m_overall_latency_meter, first ? net_live_track : nullptr, SYNC_API));
threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("READ",
- [this, reader=reader.release(), shutdown_event]() mutable {
+ [this, reader, shutdown_event]() mutable {
return run_read(reader, shutdown_event, m_latency_barrier);
}));
first = false;
if (m_overall_latency_meter) {
m_overall_latency_meter->add_start_sample(std::chrono::steady_clock::now().time_since_epoch());
}
- auto job = m_configured_infer_model->run_async(bindings, [=, &inference_status] (const AsyncInferCompletionInfo &completion_info) {
+ TRY(auto job, m_configured_infer_model->run_async(bindings, [=, &inference_status] (const AsyncInferCompletionInfo &completion_info) {
if (HAILO_SUCCESS != completion_info.status) {
inference_status = completion_info.status;
if (HAILO_STREAM_ABORT != completion_info.status) {
so there's a circular dependency */
net_live_track->progress();
}
- });
- CHECK_EXPECTED(job);
- return job.release();
+ }));
+ return job;
}
hailo_status FullAsyncNetworkRunner::run_single_thread_async_infer(EventPtr shutdown_event,
status = m_configured_infer_model->set_scheduler_priority(m_params.scheduler_priority);
CHECK_SUCCESS(status);
} else {
- auto guard_exp = ConfiguredInferModelActivationGuard::create(m_configured_infer_model);
- CHECK_EXPECTED_AS_STATUS(guard_exp);
- guard = guard_exp.release();
+ TRY(guard, ConfiguredInferModelActivationGuard::create(m_configured_infer_model));
}
- auto bindings = m_configured_infer_model->create_bindings();
- CHECK_EXPECTED_AS_STATUS(bindings);
+ TRY(auto bindings, m_configured_infer_model->create_bindings());
std::unordered_map<std::string, Buffer> input_buffers; // Keys are inputs names
std::vector<Buffer> output_buffers;
const uint8_t const_byte = 0xAB;
for (const auto &name : get_input_names()) {
- auto input_config = m_infer_model->input(name);
- CHECK_EXPECTED_AS_STATUS(input_config);
+ TRY(auto input_config, m_infer_model->input(name));
auto params = get_params(name);
- auto buffer = params.input_file_path.empty() ?
- Buffer::create(input_config->get_frame_size(), const_byte, BufferStorageParams::create_dma()) :
- read_binary_file(params.input_file_path, BufferStorageParams::create_dma());
- CHECK_EXPECTED_AS_STATUS(buffer);
- CHECK(0 == (buffer->size() % input_config->get_frame_size()), HAILO_INVALID_ARGUMENT,
- "Size of data for input '{}' must be a multiple of the frame size {}. Received - {}", name, input_config->get_frame_size(), buffer->size());
- input_buffers.emplace(name, buffer.release());
-
- for (uint32_t i = 0; i < (input_buffers.at(name).size() % input_config->get_frame_size()); i++) {
- auto mapped_buffer = DmaMappedBuffer::create(m_vdevice, input_buffers.at(name).data() + (i * input_config->get_frame_size()),
- input_config->get_frame_size(), HAILO_DMA_BUFFER_DIRECTION_H2D);
- CHECK_EXPECTED_AS_STATUS(mapped_buffer);
- dma_mapped_buffers.emplace_back(mapped_buffer.release());
+ Buffer buffer {};
+ if (params.input_file_path.empty()) {
+ TRY(buffer, Buffer::create(input_config.get_frame_size(), const_byte, BufferStorageParams::create_dma()));
+ } else {
+ TRY(buffer, read_binary_file(params.input_file_path, BufferStorageParams::create_dma()));
+ }
+ CHECK(0 == (buffer.size() % input_config.get_frame_size()), HAILO_INVALID_ARGUMENT,
+ "Size of data for input '{}' must be a multiple of the frame size {}. Received - {}", name, input_config.get_frame_size(), buffer.size());
+ input_buffers.emplace(name, std::move(buffer));
+
+ for (uint32_t i = 0; i < (input_buffers.at(name).size() % input_config.get_frame_size()); i++) {
+ TRY(auto mapped_buffer, DmaMappedBuffer::create(m_vdevice, input_buffers.at(name).data() + (i * input_config.get_frame_size()),
+ input_config.get_frame_size(), HAILO_DMA_BUFFER_DIRECTION_H2D));
+ dma_mapped_buffers.emplace_back(std::move(mapped_buffer));
}
}
for (const auto &name : get_output_names()) {
- auto output_config = m_infer_model->output(name);
- CHECK_EXPECTED_AS_STATUS(output_config);
-
- auto buffer = Buffer::create(output_config->get_frame_size(), 0, BufferStorageParams::create_dma());
- CHECK_EXPECTED_AS_STATUS(buffer);
- output_buffers.emplace_back(buffer.release());
+ TRY(auto output_config, m_infer_model->output(name));
+ TRY(auto buffer, Buffer::create(output_config.get_frame_size(), 0, BufferStorageParams::create_dma()));
+ output_buffers.emplace_back(std::move(buffer));
- auto mapped_buffer = DmaMappedBuffer::create(m_vdevice, output_buffers.back().data(), output_buffers.back().size(),
- HAILO_DMA_BUFFER_DIRECTION_D2H);
- CHECK_EXPECTED_AS_STATUS(mapped_buffer);
- dma_mapped_buffers.emplace_back(mapped_buffer.release());
+ TRY(auto mapped_buffer, DmaMappedBuffer::create(m_vdevice, output_buffers.back().data(), output_buffers.back().size(),
+ HAILO_DMA_BUFFER_DIRECTION_D2H));
+ dma_mapped_buffers.emplace_back(std::move(mapped_buffer));
- CHECK_SUCCESS(bindings->output(name)->set_buffer(MemoryView(output_buffers.back())));
+ CHECK_SUCCESS(bindings.output(name)->set_buffer(MemoryView(output_buffers.back())));
}
FramerateThrottle frame_rate_throttle(m_params.framerate);
while (HAILO_TIMEOUT == shutdown_event->wait(std::chrono::milliseconds(0)) && (HAILO_SUCCESS == inference_status)) {
for (uint32_t frames_in_cycle = 0; frames_in_cycle < m_params.batch_size; frames_in_cycle++) {
for (const auto &name : get_input_names()) {
- auto input_config = m_infer_model->input(name);
- CHECK_EXPECTED_AS_STATUS(input_config);
- auto offset = (frame_id % (input_buffers.at(name).size() / input_config->get_frame_size())) * input_config->get_frame_size();
- CHECK_SUCCESS(bindings->input(name)->set_buffer(MemoryView(input_buffers.at(name).data() + offset,
- input_config->get_frame_size())));
+ TRY(auto input_config, m_infer_model->input(name));
+ auto offset = (frame_id % (input_buffers.at(name).size() / input_config.get_frame_size())) * input_config.get_frame_size();
+ CHECK_SUCCESS(bindings.input(name)->set_buffer(MemoryView(input_buffers.at(name).data() + offset,
+ input_config.get_frame_size())));
}
frame_id++;
if (HAILO_SUCCESS == m_configured_infer_model->wait_for_async_ready(DEFAULT_TRANSFER_TIMEOUT)) {
- auto job_exp = create_infer_job(*bindings, net_live_track, frame_rate_throttle, inference_status);
- CHECK_EXPECTED_AS_STATUS(job_exp);
- last_job = job_exp.release();
+ TRY(last_job, create_infer_job(bindings, net_live_track, frame_rate_throttle, inference_status));
last_job.detach();
}
}
std::vector<AsyncThreadPtr<hailo_status>> threads;
for (auto &input_stream : m_input_streams) {
const auto stream_params = get_params(input_stream.get().name());
- auto writer = WriterWrapper<InputStream>::create(input_stream.get(), stream_params, m_vdevice,
- m_overall_latency_meter, m_params.framerate, async_streams);
- CHECK_EXPECTED(writer);
+ TRY(auto writer, WriterWrapper<InputStream>::create(input_stream.get(), stream_params, m_vdevice,
+ m_overall_latency_meter, m_params.framerate, async_streams));
if (async_streams) {
threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("WRITE_ASYNC",
- [this, writer = writer.release(), shutdown_event]() mutable {
+ [this, writer, shutdown_event]() mutable {
return run_write_async(writer, shutdown_event, m_latency_barrier);
}));
} else {
threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("WRITE",
- [this, writer = writer.release(), shutdown_event]() mutable {
+ [this, writer, shutdown_event]() mutable {
return run_write(writer, shutdown_event, m_latency_barrier);
}));
}
bool first = true; //TODO: check with multiple outputs
for (auto &output_stream : m_output_streams) {
- auto reader = ReaderWrapper<OutputStream>::create(output_stream.get(), m_vdevice,
- m_overall_latency_meter, first ? net_live_track : nullptr, async_streams);
- CHECK_EXPECTED(reader);
+ TRY(auto reader, ReaderWrapper<OutputStream>::create(output_stream.get(), m_vdevice,
+ m_overall_latency_meter, first ? net_live_track : nullptr, async_streams));
if (async_streams) {
threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("READ_ASYNC",
- [this, reader=reader.release(), shutdown_event]() mutable {
+ [this, reader, shutdown_event]() mutable {
return run_read_async(reader, shutdown_event, m_latency_barrier);
}));
} else {
threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("READ",
- [this, reader=reader.release(), shutdown_event]() mutable {
+ [this, reader, shutdown_event]() mutable {
return run_read(reader, shutdown_event, m_latency_barrier);
}));
}
std::vector<SemaphorePtr> output_semaphores;
bool is_first_output = true;
for (auto &output_stream : m_output_streams) {
- auto reader_wrapper = ReaderWrapper<OutputStream>::create(output_stream.get(), m_vdevice,
- m_overall_latency_meter, is_first_output ? net_live_track : nullptr, ASYNC_API);
- CHECK_EXPECTED_AS_STATUS(reader_wrapper);
+ TRY(auto reader_wrapper, ReaderWrapper<OutputStream>::create(output_stream.get(), m_vdevice,
+ m_overall_latency_meter, is_first_output ? net_live_track : nullptr, ASYNC_API));
is_first_output = false;
- auto max_queue_size = reader_wrapper.value()->get().get_async_max_queue_size();
- CHECK_EXPECTED_AS_STATUS(max_queue_size);
+ TRY(auto max_queue_size, reader_wrapper->get().get_async_max_queue_size());
+ TRY(auto semaphore, Semaphore::create_shared(static_cast<uint32_t>(max_queue_size)));
- auto semaphore = Semaphore::create_shared(static_cast<uint32_t>(*max_queue_size));
- CHECK_EXPECTED_AS_STATUS(semaphore);
-
- output_semaphores.emplace_back(semaphore.release());
- reader_wrappers.emplace_back(reader_wrapper.release());
+ output_semaphores.emplace_back(semaphore);
+ reader_wrappers.emplace_back(reader_wrapper);
}
// Build input wrappers
std::vector<WriterWrapperPtr<InputStream>> writer_wrappers;
std::vector<SemaphorePtr> input_semaphores;
for (auto &input_stream : m_input_streams) {
- auto writer_wrapper = WriterWrapper<InputStream>::create(input_stream.get(),
- get_params(input_stream.get().name()), m_vdevice, m_overall_latency_meter, m_params.framerate, ASYNC_API);
- CHECK_EXPECTED_AS_STATUS(writer_wrapper);
-
- auto max_queue_size = writer_wrapper.value()->get().get_async_max_queue_size();
- CHECK_EXPECTED_AS_STATUS(max_queue_size);
+ TRY(auto writer_wrapper, WriterWrapper<InputStream>::create(input_stream.get(),
+ get_params(input_stream.get().name()), m_vdevice, m_overall_latency_meter, m_params.framerate, ASYNC_API));
- auto semaphore = Semaphore::create_shared(static_cast<uint32_t>(*max_queue_size));
- CHECK_EXPECTED_AS_STATUS(semaphore);
+ TRY(auto max_queue_size, writer_wrapper->get().get_async_max_queue_size());
+ TRY(auto semaphore, Semaphore::create_shared(static_cast<uint32_t>(max_queue_size)));
- input_semaphores.emplace_back(semaphore.release());
- writer_wrappers.emplace_back(writer_wrapper.release());
+ input_semaphores.emplace_back(semaphore);
+ writer_wrappers.emplace_back(writer_wrapper);
}
// Build waitables list with reference to previous input/output semaphores.
// Inference
while (true) {
- auto wait_index = wait_group.wait_any(HAILORTCLI_DEFAULT_TIMEOUT);
- CHECK_EXPECTED_AS_STATUS(wait_index);
+ TRY(auto wait_index, wait_group.wait_any(HAILORTCLI_DEFAULT_TIMEOUT));
- if (*wait_index == shutdown_index) {
+ if (wait_index == shutdown_index) {
// Stopping the network so we won't get timeout on the flush. The async operations may still be active
// (until network deactivation).
stop();
break;
- } else if ((*wait_index >= output_index_start) && (*wait_index < input_index_start)) {
+ } else if ((wait_index >= output_index_start) && (wait_index < input_index_start)) {
// output is ready
- const size_t output_index = *wait_index - output_index_start;
+ const size_t output_index = wait_index - output_index_start;
auto status = reader_wrappers[output_index]->read_async(
[semaphore=output_semaphores[output_index]](const OutputStream::CompletionInfo &) {
(void)semaphore->signal();
CHECK_SUCCESS(status);
} else {
// input is ready
- const size_t input_index = *wait_index - input_index_start;
+ const size_t input_index = wait_index - input_index_start;
auto status = writer_wrappers[input_index]->write_async(
[semaphore=input_semaphores[input_index]](const InputStream::CompletionInfo &) {
(void)semaphore->signal();
// sync_event will be used to send one frame at a time
EventPtr sync_event = nullptr;
if (m_params.measure_hw_latency || m_params.measure_overall_latency) {
- auto sync_event_exp = Event::create_shared(Event::State::not_signalled);
- CHECK_EXPECTED_AS_STATUS(sync_event_exp);
- sync_event = sync_event_exp.release();
+ TRY(sync_event, Event::create_shared(Event::State::not_signalled));
}
while (true) {
// sync_event will be used to send one frame at a time
EventPtr sync_event = nullptr;
if (m_params.measure_hw_latency || m_params.measure_overall_latency) {
- auto sync_event_exp = Event::create_shared(Event::State::not_signalled);
- CHECK_EXPECTED_AS_STATUS(sync_event_exp);
- sync_event = sync_event_exp.release();
+ TRY(sync_event, Event::create_shared(Event::State::not_signalled));
}
while (true) {
Expected<std::reference_wrapper<Device>> get_single_physical_device(VDevice &vdevice)
{
- auto expected_physical_devices = vdevice.get_physical_devices();
- CHECK_EXPECTED(expected_physical_devices);
- CHECK_AS_EXPECTED(1 == expected_physical_devices->size(), HAILO_INVALID_OPERATION, "Operation not allowed for multi-device");
- auto &res = expected_physical_devices->at(0);
+ TRY(auto physical_devices, vdevice.get_physical_devices());
+ CHECK_AS_EXPECTED(1 == physical_devices.size(), HAILO_INVALID_OPERATION,
+ "Operation not allowed for multi-device");
+ auto &res = physical_devices.at(0);
return std::move(res);
}
Expected<std::vector<std::shared_ptr<NetworkRunner>>> Run2::init_and_run_net_runners(VDevice *vdevice)
{
std::vector<std::shared_ptr<NetworkRunner>> net_runners;
-
- auto shutdown_event_exp = Event::create_shared(Event::State::not_signalled);
- CHECK_EXPECTED(shutdown_event_exp);
- auto shutdown_event = shutdown_event_exp.release();
+ TRY(auto shutdown_event, Event::create_shared(Event::State::not_signalled));
// create network runners
for (auto &net_params : get_network_params()) {
- auto expected_net_runner = NetworkRunner::create_shared(*vdevice, net_params);
- CHECK_EXPECTED(expected_net_runner);
- auto net_runner = expected_net_runner.release();
+ TRY(auto net_runner, NetworkRunner::create_shared(*vdevice, net_params));
net_runners.emplace_back(net_runner);
}
activation_barrier.arrive_and_wait();
if (get_measure_power() || get_measure_current() || get_measure_temp()) {
- auto physical_devices = vdevice->get_physical_devices();
- CHECK_EXPECTED(physical_devices);
-
- for (auto &device : physical_devices.value()) {
- auto measurement_live_track = MeasurementLiveTrack::create_shared(device.get(), get_measure_power(),
- get_measure_current(), get_measure_temp());
- CHECK_EXPECTED(measurement_live_track);
-
- live_stats->add(measurement_live_track.release(), 2);
+ TRY(auto physical_devices, vdevice->get_physical_devices());
+
+ for (auto &device : physical_devices) {
+ TRY(const auto identity, device.get().identify());
+ CHECK_AS_EXPECTED(HailoRTCommon::is_power_measurement_supported(identity.device_architecture) || !(get_measure_power()),
+ HAILO_INVALID_OPERATION, "HW arch {} does not support power measurement. Disable the power-measure option",
+ HailoRTCommon::get_device_arch_str(identity.device_architecture));
+ CHECK_AS_EXPECTED(HailoRTCommon::is_current_measurement_supported(identity.device_architecture) || !(get_measure_current()),
+ HAILO_INVALID_OPERATION, "HW arch {} does not support current measurement. Disable the current-measure option",
+ HailoRTCommon::get_device_arch_str(identity.device_architecture));
+ CHECK_AS_EXPECTED(HailoRTCommon::is_temp_measurement_supported(identity.device_architecture) || !(get_measure_temp()),
+ HAILO_INVALID_OPERATION, "HW arch {} does not support temperature measurement. Disable the temp-measure option",
+ HailoRTCommon::get_device_arch_str(identity.device_architecture));
+
+ TRY(auto measurement_live_track, MeasurementLiveTrack::create_shared(device.get(),
+ get_measure_power(), get_measure_current(), get_measure_temp()));
+
+ live_stats->add(measurement_live_track, 2);
}
}
if (!get_output_json_path().empty()){
live_stats->dump_stats(get_output_json_path(), get_str_infer_mode(get_mode()));
}
- auto expected_fps_per_network = live_stats->get_last_measured_fps_per_network_group();
- CHECK_EXPECTED(expected_fps_per_network);
- auto fps_per_network = expected_fps_per_network.release();
+ TRY(auto fps_per_network, live_stats->get_last_measured_fps_per_network_group());
for (size_t network_runner_index = 0; network_runner_index < fps_per_network.size(); network_runner_index++) {
net_runners[network_runner_index]->set_last_measured_fps(fps_per_network[network_runner_index]);
}
LOGGER__WARNING("\"hailortcli run2\" is not optimized for single model usage. It is recommended to use \"hailortcli run\" command for a single model");
}
- auto expected_vdevice = app->create_vdevice();
- CHECK_EXPECTED_AS_STATUS(expected_vdevice);
- auto vdevice = expected_vdevice.release();
-
+ TRY(auto vdevice, app->create_vdevice());
std::vector<uint16_t> batch_sizes_to_run = { app->get_network_params()[0].batch_size };
if(app->get_measure_fw_actions() && app->get_network_params()[0].batch_size == HAILO_DEFAULT_BATCH_SIZE) {
// In case measure-fw-actions is enabled and no batch size was provided - we want to run with batch sizes 1,2,4,8,16
ordered_json action_list_json;
if (app->get_measure_fw_actions()) {
- auto device = get_single_physical_device(*vdevice);
- CHECK_EXPECTED_AS_STATUS(device);
-
- auto expected_action_list_json = DownloadActionListCommand::init_json_object(device.release(), app->get_network_params()[0].hef_path);
- CHECK_EXPECTED_AS_STATUS(expected_action_list_json);
- action_list_json = expected_action_list_json.release();
+ TRY(auto device, get_single_physical_device(*vdevice));
+ TRY(action_list_json,
+ DownloadActionListCommand::init_json_object(device, app->get_network_params()[0].hef_path));
runtime_data_output_path = format_measure_fw_actions_output_path(
app->get_measure_fw_actions_output_path(), app->get_network_params()[0].hef_path);
}
for (auto batch_size : batch_sizes_to_run) {
if(app->get_measure_fw_actions()) {
app->set_batch_size(batch_size);
-
- auto device = get_single_physical_device(*vdevice);
- CHECK_EXPECTED_AS_STATUS(device);
-
- auto status = DownloadActionListCommand::set_batch_to_measure(device.release(), RUNTIME_DATA_BATCH_INDEX_TO_MEASURE_DEFAULT);
+ TRY(auto device, get_single_physical_device(*vdevice));
+ auto status = DownloadActionListCommand::set_batch_to_measure(device, RUNTIME_DATA_BATCH_INDEX_TO_MEASURE_DEFAULT);
CHECK_SUCCESS(status);
}
- auto expected_net_runners = app->init_and_run_net_runners(vdevice.get());
- CHECK_EXPECTED_AS_STATUS(expected_net_runners);
- auto net_runners = expected_net_runners.release();
-
+ TRY(auto net_runners, app->init_and_run_net_runners(vdevice.get()));
if(app->get_measure_fw_actions()) { // Collecting runtime data
- auto device = get_single_physical_device(*vdevice);
- CHECK_EXPECTED_AS_STATUS(device);
-
- auto status = DownloadActionListCommand::execute(device.release(), net_runners[0]->get_configured_network_group(), batch_size, action_list_json, net_runners[0]->get_last_measured_fps(), network_group_index);
+ TRY(auto device, get_single_physical_device(*vdevice));
+ auto status = DownloadActionListCommand::execute(device, net_runners[0]->get_configured_network_group(),
+ batch_size, action_list_json, net_runners[0]->get_last_measured_fps(), network_group_index);
CHECK_SUCCESS(status);
network_group_index++;
constexpr uint32_t DEFAULT_TIME_TO_RUN_SECONDS = 5;
#ifndef HAILO_EMULATOR
-constexpr std::chrono::milliseconds TIME_TO_WAIT_FOR_CONFIG(300);
#define HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS (HAILO_DEFAULT_VSTREAM_TIMEOUT_MS)
#else /* ifndef HAILO_EMULATOR */
-constexpr std::chrono::milliseconds TIME_TO_WAIT_FOR_CONFIG(30000);
#define HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS (HAILO_DEFAULT_VSTREAM_TIMEOUT_MS * 100)
#endif /* ifndef HAILO_EMULATOR */
static const char *RUNTIME_DATA_OUTPUT_PATH_HEF_PLACE_HOLDER = "<hef>";
const inference_runner_params ¶ms)
{
std::map<std::string, std::vector<InputVStream>> res;
- auto network_infos = configured_net_group.get_network_infos();
- CHECK_EXPECTED(network_infos);
- for (auto &network_info : network_infos.value()) {
+ TRY(const auto network_infos, configured_net_group.get_network_infos());
+ for (const auto &network_info : network_infos) {
auto quantized = (params.transform.format_type != HAILO_FORMAT_TYPE_FLOAT32);
- auto input_vstreams_params = configured_net_group.make_input_vstream_params(quantized,
- params.transform.format_type, HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, network_info.name);
- CHECK_EXPECTED(input_vstreams_params);
+ TRY(auto input_vstreams_params, configured_net_group.make_input_vstream_params(quantized,
+ params.transform.format_type, HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, network_info.name));
- for (auto &vstream_params : input_vstreams_params.value()) {
+ for (auto &vstream_params : input_vstreams_params) {
vstream_params.second.pipeline_elements_stats_flags = inference_runner_params_to_pipeline_elem_stats_flags(params.pipeline_stats);
vstream_params.second.vstream_stats_flags = inference_runner_params_to_vstream_stats_flags(params.pipeline_stats);
}
- auto input_vstreams = VStreamsBuilder::create_input_vstreams(configured_net_group, input_vstreams_params.value());
- CHECK_EXPECTED(input_vstreams);
- res.emplace(network_info.name, input_vstreams.release());
+ TRY(auto input_vstreams, VStreamsBuilder::create_input_vstreams(configured_net_group, input_vstreams_params));
+ res.emplace(network_info.name, std::move(input_vstreams));
}
return res;
}
const inference_runner_params ¶ms)
{
std::map<std::string, std::vector<OutputVStream>> res;
- auto network_infos = configured_net_group.get_network_infos();
- CHECK_EXPECTED(network_infos);
- for (auto &network_info : network_infos.value()) {
+ TRY(const auto network_infos, configured_net_group.get_network_infos());
+ for (const auto &network_info : network_infos) {
// Data is not quantized if format_type is explicitly float32, or if an output is NMS (which also enforces float32 output)
// We don't cover a case of multiple outputs where only some of them are NMS (no such model currently), and anyway it is handled in run2
- auto vstream_infos = configured_net_group.get_output_vstream_infos();
- CHECK_EXPECTED(vstream_infos);
- auto nms_output = std::any_of(vstream_infos->begin(), vstream_infos->end(), [] (const hailo_vstream_info_t &output_info) {
+ TRY(const auto vstream_infos, configured_net_group.get_output_vstream_infos());
+ auto nms_output = std::any_of(vstream_infos.begin(), vstream_infos.end(), [] (const hailo_vstream_info_t &output_info) {
return HailoRTCommon::is_nms(output_info);
});
auto quantized = ((params.transform.format_type != HAILO_FORMAT_TYPE_FLOAT32) && !nms_output);
- auto output_vstreams_params = configured_net_group.make_output_vstream_params(quantized,
- params.transform.format_type, HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, network_info.name);
- CHECK_EXPECTED(output_vstreams_params);
+ TRY(auto output_vstreams_params, configured_net_group.make_output_vstream_params(quantized,
+ params.transform.format_type, HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, network_info.name));
- for (auto &vstream_params : output_vstreams_params.value()) {
+ for (auto &vstream_params : output_vstreams_params) {
vstream_params.second.pipeline_elements_stats_flags = inference_runner_params_to_pipeline_elem_stats_flags(params.pipeline_stats);
vstream_params.second.vstream_stats_flags = inference_runner_params_to_vstream_stats_flags(params.pipeline_stats);
}
- auto output_vstreams = VStreamsBuilder::create_output_vstreams(configured_net_group, output_vstreams_params.value());
- CHECK_EXPECTED(output_vstreams);
- res.emplace(network_info.name, output_vstreams.release());
+ TRY(auto output_vstreams, VStreamsBuilder::create_output_vstreams(configured_net_group, output_vstreams_params));
+ res.emplace(network_info.name, std::move(output_vstreams));
}
return res;
}
Expected<std::map<std::string, std::vector<std::reference_wrapper<InputStream>>>> create_input_streams(ConfiguredNetworkGroup &configured_net_group)
{
std::map<std::string, std::vector<std::reference_wrapper<InputStream>>> res;
- auto network_infos = configured_net_group.get_network_infos();
- CHECK_EXPECTED(network_infos);
- for (auto &network_info : network_infos.value()) {
- auto input_streams = configured_net_group.get_input_streams_by_network(network_info.name);
- CHECK_EXPECTED(input_streams);
- res.emplace(network_info.name, input_streams.release());
+ TRY(const auto network_infos, configured_net_group.get_network_infos());
+ for (const auto &network_info : network_infos) {
+ TRY(auto input_streams, configured_net_group.get_input_streams_by_network(network_info.name));
+ res.emplace(network_info.name, std::move(input_streams));
}
return res;
}
Expected<std::map<std::string, std::vector<std::reference_wrapper<OutputStream>>>> create_output_streams(ConfiguredNetworkGroup &configured_net_group)
{
std::map<std::string, std::vector<std::reference_wrapper<OutputStream>>> res;
- auto network_infos = configured_net_group.get_network_infos();
- CHECK_EXPECTED(network_infos);
- for (auto &network_info : network_infos.value()) {
- auto output_streams = configured_net_group.get_output_streams_by_network(network_info.name);
- CHECK_EXPECTED(output_streams);
- res.emplace(network_info.name, output_streams.release());
+ TRY(const auto network_infos, configured_net_group.get_network_infos());
+ for (const auto &network_info : network_infos) {
+ TRY(auto output_streams, configured_net_group.get_output_streams_by_network(network_info.name));
+ res.emplace(network_info.name, std::move(output_streams));
}
return res;
}
std::map<std::string, BufferPtr> dst_data;
for (auto &recv_objects : recv_objects_per_network) {
for (auto &recv_object : recv_objects.second) {
- auto buffer = Buffer::create_shared(recv_object.get().get_frame_size());
- CHECK_EXPECTED(buffer);
- dst_data[recv_object.get().name()] = buffer.release();
+ TRY(dst_data[recv_object.get().name()],
+ Buffer::create_shared(recv_object.get().get_frame_size()));
}
}
std::vector<AsyncThreadPtr<hailo_status>> results;
- auto network_progress_bar_exp = progress_bar.create_network_progress_bar(configured_net_group, network_name);
- CHECK_EXPECTED_AS_STATUS(network_progress_bar_exp);
- auto network_progress_bar = network_progress_bar_exp.release();
+ TRY(auto network_progress_bar,
+ progress_bar.create_network_progress_bar(configured_net_group, network_name));
const auto start = std::chrono::high_resolution_clock::now();
// Launch async read/writes
// TODO: HRT-7798
if (!params.vdevice_params.multi_process_service) {
- auto network_input_streams = configured_net_group->get_input_streams_by_network(network_name);
- CHECK_EXPECTED_AS_STATUS(network_input_streams);
- inference_result.m_total_send_frame_size = total_send_frame_size(network_input_streams.value());
- auto network_output_streams = configured_net_group->get_output_streams_by_network(network_name);
- CHECK_EXPECTED_AS_STATUS(network_output_streams);
- inference_result.m_total_recv_frame_size = total_recv_frame_size(network_output_streams.value());
+ TRY(auto network_input_streams, configured_net_group->get_input_streams_by_network(network_name));
+ inference_result.m_total_send_frame_size = total_send_frame_size(network_input_streams);
+ TRY(auto network_output_streams, configured_net_group->get_output_streams_by_network(network_name));
+ inference_result.m_total_recv_frame_size = total_recv_frame_size(network_output_streams);
}
if (params.measure_latency) {
}
if (params.measure_overall_latency) {
- auto overall_latency = overall_latency_meter.get_latency(true);
- CHECK_EXPECTED_AS_STATUS(overall_latency);
- inference_result.m_overall_latency = std::make_unique<std::chrono::nanoseconds>(*overall_latency);
+ TRY(auto overall_latency, overall_latency_meter.get_latency(true));
+ inference_result.m_overall_latency = std::make_unique<std::chrono::nanoseconds>(std::move(overall_latency));
}
return HAILO_SUCCESS;
std::vector<std::map<std::string, NetworkInferResult>> networks_results; // Map of networks results for each network group
networks_results.reserve(configured_net_groups.size());
- auto progress_bar_exp = InferProgress::create(params, std::chrono::seconds(1));
- CHECK_EXPECTED(progress_bar_exp);
- auto progress_bar = progress_bar_exp.release();
-
+ TRY(auto progress_bar, InferProgress::create(params, std::chrono::seconds(1)));
for (size_t network_group_index = 0; network_group_index < configured_net_groups.size(); network_group_index++) {
networks_threads_status.emplace_back();
networks_results.emplace_back();
for (size_t network_group_index = 0; network_group_index < configured_net_groups.size(); network_group_index++) {
input_vstreams.emplace_back();
output_vstreams.emplace_back();
- auto in_vstreams = create_input_vstreams(*configured_net_groups[network_group_index], params);
- CHECK_EXPECTED(in_vstreams);
- auto in_vstreams_ptr = make_shared_nothrow<std::map<std::string, std::vector<InputVStream>>>(in_vstreams.release());
+ TRY(auto in_vstreams, create_input_vstreams(*configured_net_groups[network_group_index], params));
+ auto in_vstreams_ptr = make_shared_nothrow<std::map<std::string, std::vector<InputVStream>>>(std::move(in_vstreams));
CHECK_NOT_NULL_AS_EXPECTED(in_vstreams_ptr, HAILO_OUT_OF_HOST_MEMORY);
input_vstreams[network_group_index] = in_vstreams_ptr;
- auto out_vstreams = create_output_vstreams(*configured_net_groups[network_group_index], params);
- CHECK_EXPECTED(out_vstreams);
- auto out_vstreams_ptr = make_shared_nothrow<std::map<std::string, std::vector<OutputVStream>>>(out_vstreams.release());
+ TRY(auto out_vstreams, create_output_vstreams(*configured_net_groups[network_group_index], params));
+ auto out_vstreams_ptr = make_shared_nothrow<std::map<std::string, std::vector<OutputVStream>>>(std::move(out_vstreams));
CHECK_NOT_NULL_AS_EXPECTED(out_vstreams_ptr, HAILO_OUT_OF_HOST_MEMORY);
output_vstreams[network_group_index] = out_vstreams_ptr;
output_vstreams_refs[network_group_index].emplace(output_vstreams_per_network.first, output_refs);
}
- auto network_group_output_buffers = create_output_buffers(output_vstreams_refs[network_group_index]);
- CHECK_EXPECTED(network_group_output_buffers);
- output_buffers[network_group_index] = network_group_output_buffers.release();
+ TRY(output_buffers[network_group_index], create_output_buffers(output_vstreams_refs[network_group_index]));
}
auto res = run_streaming<InputVStream, OutputVStream>(configured_net_groups, input_datasets,
input_vstreams.clear();
output_vstreams.clear();
- CHECK_EXPECTED(res);
+ CHECK_EXPECTED(res); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
return res;
}
case InferMode::HW_ONLY:
std::map<std::string, hailort::BufferPtr>());
for (size_t network_group_index = 0; network_group_index < configured_net_groups.size(); network_group_index++) {
- auto input_streams = create_input_streams(*configured_net_groups[network_group_index]);
- CHECK_EXPECTED(input_streams);
- input_streams_refs[network_group_index] = input_streams.release();
- auto output_streams = create_output_streams(*configured_net_groups[network_group_index]);
- output_streams_refs[network_group_index] = output_streams.release();
-
- auto network_group_output_buffers = create_output_buffers(output_streams_refs[network_group_index]);
- CHECK_EXPECTED(network_group_output_buffers);
- output_buffers[network_group_index] = network_group_output_buffers.release();
+ TRY(input_streams_refs[network_group_index], create_input_streams(*configured_net_groups[network_group_index]));
+ TRY(output_streams_refs[network_group_index], create_output_streams(*configured_net_groups[network_group_index]));
+ TRY(output_buffers[network_group_index], create_output_buffers(output_streams_refs[network_group_index]));
}
return run_streaming<InputStream, OutputStream>(configured_net_groups, input_datasets, output_buffers,
}
}
-static Expected<std::unique_ptr<ActivatedNetworkGroup>> activate_network_group(ConfiguredNetworkGroup &network_group)
-{
- hailo_activate_network_group_params_t network_group_params = {};
- auto activated_network_group = network_group.activate(network_group_params);
- CHECK_EXPECTED(activated_network_group, "Failed activating network group");
-
- // Wait for configuration
- // TODO: HRT-2492 wait for config in a normal way
- std::this_thread::sleep_for(TIME_TO_WAIT_FOR_CONFIG);
-
- return activated_network_group;
-}
-
static Expected<std::map<std::string, BufferPtr>> create_constant_dataset(
const std::pair<std::vector<hailo_stream_info_t>, std::vector<hailo_vstream_info_t>> &input_infos, const hailo_transform_params_t &trans_params,
InferMode mode)
const auto file_path_it = file_paths.find(stream_name);
CHECK_AS_EXPECTED(file_paths.end() != file_path_it, HAILO_INVALID_ARGUMENT, "Missing input file for input: {}", stream_name);
- auto host_buffer = read_binary_file(file_path_it->second);
- CHECK_EXPECTED(host_buffer, "Failed reading file {}", file_path_it->second);
- CHECK_AS_EXPECTED((host_buffer->size() % host_frame_size) == 0, HAILO_INVALID_ARGUMENT,
- "Input file ({}) size {} must be a multiple of the frame size {} ({})", file_path_it->second, host_buffer->size(), host_frame_size, stream_name);
+ TRY(auto host_buffer, read_binary_file(file_path_it->second));
+ CHECK_AS_EXPECTED((host_buffer.size() % host_frame_size) == 0, HAILO_INVALID_ARGUMENT,
+ "Input file ({}) size {} must be a multiple of the frame size {} ({})", file_path_it->second, host_buffer.size(), host_frame_size, stream_name);
if (InferMode::HW_ONLY == mode) {
auto matching_stream_info = std::find_if(input_infos.first.begin(), input_infos.first.end(), [&stream_name] (const auto &stream_info) {
return std::string(stream_info.name) == stream_name;
});
CHECK_AS_EXPECTED(matching_stream_info != input_infos.first.end(), HAILO_INVALID_OPERATION, "Failed to find raw-stream with name {}.", stream_name);
- const size_t frames_count = (host_buffer->size() / host_frame_size);
+ const size_t frames_count = (host_buffer.size() / host_frame_size);
const size_t hw_frame_size = matching_stream_info->hw_frame_size;
const size_t hw_buffer_size = frames_count * hw_frame_size;
- auto hw_buffer = Buffer::create_shared(hw_buffer_size);
- CHECK_EXPECTED(hw_buffer);
-
- auto transform_context = InputTransformContext::create(*matching_stream_info, trans_params);
- CHECK_EXPECTED(transform_context);
+ TRY(auto hw_buffer, Buffer::create_shared(hw_buffer_size));
+ TRY(auto transform_context, InputTransformContext::create(*matching_stream_info, trans_params));
for (size_t i = 0; i < frames_count; i++) {
- MemoryView host_data(static_cast<uint8_t*>(host_buffer->data() + (i*host_frame_size)), host_frame_size);
- MemoryView hw_data(static_cast<uint8_t*>(hw_buffer.value()->data() + (i*hw_frame_size)), hw_frame_size);
+ MemoryView host_data(static_cast<uint8_t*>(host_buffer.data() + (i*host_frame_size)), host_frame_size);
+ MemoryView hw_data(static_cast<uint8_t*>(hw_buffer->data() + (i*hw_frame_size)), hw_frame_size);
- auto status = transform_context.value()->transform(host_data, hw_data);
+ auto status = transform_context->transform(host_data, hw_data);
CHECK_SUCCESS_AS_EXPECTED(status);
}
- dataset[stream_name] = hw_buffer.release();
+ dataset[stream_name] = std::move(hw_buffer);
} else {
- auto host_buffer_shared = make_shared_nothrow<Buffer>(host_buffer.release());
+ auto host_buffer_shared = make_shared_nothrow<Buffer>(std::move(host_buffer));
CHECK_NOT_NULL_AS_EXPECTED(host_buffer_shared, HAILO_OUT_OF_HOST_MEMORY);
dataset[stream_name] = host_buffer_shared;
}
// Vector of len(ng.conut), each element is pair of all input_stream_infos, and all input_vstream_infos
std::vector<std::pair<std::vector<hailo_stream_info_t>, std::vector<hailo_vstream_info_t>>> input_infos;
for (auto &network_group : network_groups) {
- auto expected_all_streams_infos = network_group->get_all_stream_infos();
- CHECK_EXPECTED(expected_all_streams_infos);
- auto &all_stream_infos = expected_all_streams_infos.value();
+ TRY(const auto all_streams_infos, network_group->get_all_stream_infos());
std::vector<hailo_stream_info_t> group_input_stream_infos;
- std::copy_if(all_stream_infos.begin(), all_stream_infos.end(), std::back_inserter(group_input_stream_infos), [](const auto &info) {
+ std::copy_if(all_streams_infos.begin(), all_streams_infos.end(), std::back_inserter(group_input_stream_infos), [](const auto &info) {
return info.direction == HAILO_H2D_STREAM;
});
- auto expected_input_vstreams_infos = network_group->get_input_vstream_infos();
- CHECK_EXPECTED(expected_input_vstreams_infos);
- input_infos.push_back({group_input_stream_infos, expected_input_vstreams_infos.release()});
+ TRY(const auto input_vstreams_infos, network_group->get_input_vstream_infos());
+ input_infos.push_back({group_input_stream_infos, std::move(input_vstreams_infos)});
}
if (!params.inputs_name_and_file_path.empty()) {
for (auto &group_input_infos : input_infos) {
- auto network_group_dataset = create_dataset_from_files(group_input_infos, params.inputs_name_and_file_path,
- trans_params, params.mode);
- CHECK_EXPECTED(network_group_dataset);
- results.emplace_back(network_group_dataset.release());
+ TRY(auto network_group_dataset, create_dataset_from_files(group_input_infos, params.inputs_name_and_file_path,
+ trans_params, params.mode));
+ results.emplace_back(std::move(network_group_dataset));
}
} else {
for (auto &group_input_infos : input_infos) {
- auto network_group_dataset = create_constant_dataset(group_input_infos, trans_params, params.mode);
- CHECK_EXPECTED(network_group_dataset);
- results.emplace_back(network_group_dataset.release());
+ TRY(auto network_group_dataset, create_constant_dataset(group_input_infos, trans_params, params.mode));
+ results.emplace_back(std::move(network_group_dataset));
}
}
return results;
const inference_runner_params ¶ms)
{
CHECK_AS_EXPECTED(1 == network_groups.size(), HAILO_INVALID_OPERATION, "Inference is not supported on HEFs with multiple network groups");
- auto activated_net_group = activate_network_group(*network_groups[0]);
- CHECK_EXPECTED(activated_net_group, "Failed activate network_group");
-
- auto input_dataset = create_dataset(network_groups, params);
- CHECK_EXPECTED(input_dataset, "Failed creating input dataset");
+ TRY(auto activated_net_group, network_groups[0]->activate(), "Failed activate network_group");
+ TRY(auto input_dataset, create_dataset(network_groups, params));
hailo_power_measurement_types_t measurement_type = HAILO_POWER_MEASUREMENT_TYPES__MAX_ENUM;
bool should_measure_power = false;
should_measure_power = true;
}
- std::shared_ptr<LongPowerMeasurement> long_power_measurement = nullptr;
+ std::shared_ptr<LongPowerMeasurement> long_power_measurement_ptr = nullptr;
if (should_measure_power) {
- auto long_power_measurement_exp = PowerMeasurementSubcommand::start_power_measurement(device,
- HAILO_DVM_OPTIONS_AUTO,
- measurement_type, params.power_measurement.sampling_period, params.power_measurement.averaging_factor);
- CHECK_EXPECTED(long_power_measurement_exp);
- long_power_measurement = make_shared_nothrow<LongPowerMeasurement>(long_power_measurement_exp.release());
- CHECK_NOT_NULL_AS_EXPECTED(long_power_measurement, HAILO_OUT_OF_HOST_MEMORY);
+ TRY(auto long_power_measurement, PowerMeasurementSubcommand::start_power_measurement(device,
+ HAILO_DVM_OPTIONS_AUTO, measurement_type, params.power_measurement.sampling_period, params.power_measurement.averaging_factor));
+ long_power_measurement_ptr = make_shared_nothrow<LongPowerMeasurement>(std::move(long_power_measurement));
+ CHECK_NOT_NULL_AS_EXPECTED(long_power_measurement_ptr, HAILO_OUT_OF_HOST_MEMORY);
}
bool should_measure_temp = params.measure_temp;
- auto temp_measure = TemperatureMeasurement::create_shared(device);
- CHECK_EXPECTED(temp_measure);
+ TRY(auto temp_measure, TemperatureMeasurement::create_shared(device));
if (should_measure_temp) {
- auto status = temp_measure.value()->start_measurement();
+ auto status = temp_measure->start_measurement();
CHECK_SUCCESS_AS_EXPECTED(status, "Failed to get chip's temperature");
}
- auto infer_result = run_inference(network_groups, input_dataset.value(), params);
- CHECK_EXPECTED(infer_result, "Error failed running inference");
+ TRY(auto inference_result, run_inference(network_groups, input_dataset, params), "Error while running inference");
- InferResult inference_result(infer_result.release());
std::vector<std::reference_wrapper<Device>> device_refs;
device_refs.push_back(device);
inference_result.initialize_measurements(device_refs);
if (should_measure_power) {
- auto status = long_power_measurement->stop();
+ auto status = long_power_measurement_ptr->stop();
CHECK_SUCCESS_AS_EXPECTED(status);
if (params.power_measurement.measure_current) {
- status = inference_result.set_current_measurement(device.get_dev_id(), std::move(long_power_measurement));
+ status = inference_result.set_current_measurement(device.get_dev_id(), std::move(long_power_measurement_ptr));
CHECK_SUCCESS_AS_EXPECTED(status);
} else {
- status = inference_result.set_power_measurement(device.get_dev_id(), std::move(long_power_measurement));
+ status = inference_result.set_power_measurement(device.get_dev_id(), std::move(long_power_measurement_ptr));
CHECK_SUCCESS_AS_EXPECTED(status);
}
}
if (should_measure_temp) {
- temp_measure.value()->stop_measurement();
- auto temp_measure_p = make_shared_nothrow<AccumulatorResults>(temp_measure.value()->get_data());
+ temp_measure->stop_measurement();
+ auto temp_measure_p = make_shared_nothrow<AccumulatorResults>(temp_measure->get_data());
CHECK_NOT_NULL_AS_EXPECTED(temp_measure_p, HAILO_OUT_OF_HOST_MEMORY);
auto status = inference_result.set_temp_measurement(device.get_dev_id(), std::move(temp_measure_p));
CHECK_SUCCESS_AS_EXPECTED(status);
size_t min_frames_count = UINT32_MAX;
for (auto &network_group_results : inference_result.network_group_results()) {
for (const auto &network_results_pair : network_group_results.results_per_network()) {
- auto frames_count = network_group_results.frames_count(network_results_pair.first);
- CHECK_EXPECTED(frames_count);
- min_frames_count = std::min(frames_count.value(), min_frames_count);
+ TRY(const auto frames_count, network_group_results.frames_count(network_results_pair.first));
+ min_frames_count = std::min(frames_count, min_frames_count);
}
}
return min_frames_count;
Expected<InferResult> run_command_hef_single_device(const inference_runner_params ¶ms)
{
- auto devices = create_devices(params.vdevice_params.device_params);
- CHECK_EXPECTED(devices, "Failed creating device");
+ TRY(auto devices, create_devices(params.vdevice_params.device_params), "Failed creating device");
/* This function supports controls for multiple devices.
We validate there is only 1 device generated as we are on a single device flow */
- CHECK_AS_EXPECTED(1 == devices->size(), HAILO_INTERNAL_FAILURE);
- auto &device = devices.value()[0];
-
- auto hef = Hef::create(params.hef_path.c_str());
- CHECK_EXPECTED(hef, "Failed reading hef file {}", params.hef_path);
+ CHECK_AS_EXPECTED(1 == devices.size(), HAILO_INTERNAL_FAILURE);
+ auto &device = devices[0];
- auto interface = device->get_default_streams_interface();
- CHECK_EXPECTED(interface, "Failed to get default streams interface");
-
- auto configure_params = get_configure_params(params, hef.value(), interface.value());
- CHECK_EXPECTED(configure_params);
-
- auto network_group_list = device->configure(hef.value(), configure_params.value());
- CHECK_EXPECTED(network_group_list, "Failed configure device from hef");
+ TRY(auto hef, Hef::create(params.hef_path.c_str()), "Failed reading hef file {}", params.hef_path);
+ TRY(const auto interface, device->get_default_streams_interface(), "Failed to get default streams interface");
+ TRY(auto configure_params, get_configure_params(params, hef, interface));
+ TRY(auto network_group_list, device->configure(hef, configure_params), "Failed configure device from hef");
if (use_batch_to_measure_opt(params)) {
auto status = DownloadActionListCommand::set_batch_to_measure(*device, params.runtime_data.batch_to_measure);
CHECK_SUCCESS_AS_EXPECTED(status);
}
- auto inference_result = activate_and_run_single_device(*device, network_group_list.value(), params);
+ auto inference_result = activate_and_run_single_device(*device, network_group_list, params);
if (use_batch_to_measure_opt(params) && (0 == params.frames_count) && inference_result) {
- auto min_frames_count = get_min_inferred_frames_count(inference_result.value());
- CHECK_EXPECTED(min_frames_count);
- if (min_frames_count.value() < params.runtime_data.batch_to_measure) {
+ TRY(auto min_frames_count, get_min_inferred_frames_count(inference_result.value()));
+ if (min_frames_count < params.runtime_data.batch_to_measure) {
LOGGER__WARNING("Number of frames sent ({}) is smaller than --batch-to-measure ({}), "
- "hence timestamps will not be updated in runtime data", min_frames_count.value() ,
+ "hence timestamps will not be updated in runtime data", min_frames_count,
params.runtime_data.batch_to_measure);
}
}
if (params.runtime_data.collect_runtime_data) {
const auto runtime_data_output_path = format_runtime_data_output_path(
params.runtime_data.runtime_data_output_path, params.hef_path);
- DownloadActionListCommand::execute(*device, runtime_data_output_path, network_group_list.value(),
+ DownloadActionListCommand::execute(*device, runtime_data_output_path, network_group_list,
params.hef_path);
}
- CHECK_EXPECTED(inference_result);
+ CHECK_EXPECTED(inference_result); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
return inference_result;
}
{
std::unique_ptr<ActivatedNetworkGroup> activated_network_group;
if (!scheduler_is_used) {
- auto activated_net_group_exp = activate_network_group(*network_groups[0]);
- CHECK_EXPECTED(activated_net_group_exp, "Failed activate network_group");
- activated_network_group = activated_net_group_exp.release();
+ TRY(activated_network_group, network_groups[0]->activate());
}
- auto input_dataset = create_dataset(network_groups, params);
- CHECK_EXPECTED(input_dataset, "Failed creating input dataset");
+ TRY(const auto input_dataset, create_dataset(network_groups, params), "Failed creating input dataset");
hailo_power_measurement_types_t measurement_type = HAILO_POWER_MEASUREMENT_TYPES__MAX_ENUM;
bool should_measure_power = false;
std::map<std::string, std::shared_ptr<LongPowerMeasurement>> power_measurements;
if (should_measure_power) {
for (auto &device : physical_devices) {
- auto long_power_measurement_exp = PowerMeasurementSubcommand::start_power_measurement(device,
- HAILO_DVM_OPTIONS_AUTO,
- measurement_type, params.power_measurement.sampling_period, params.power_measurement.averaging_factor);
- CHECK_EXPECTED(long_power_measurement_exp, "Failed starting power measurement on device {}", device.get().get_dev_id());
- auto long_power_measurement_p = make_shared_nothrow<LongPowerMeasurement>(long_power_measurement_exp.release());
+ TRY(auto long_power_measurement, PowerMeasurementSubcommand::start_power_measurement(device,
+ HAILO_DVM_OPTIONS_AUTO, measurement_type, params.power_measurement.sampling_period, params.power_measurement.averaging_factor),
+ "Failed starting power measurement on device {}", device.get().get_dev_id());
+ auto long_power_measurement_p = make_shared_nothrow<LongPowerMeasurement>(std::move(long_power_measurement));
CHECK_NOT_NULL_AS_EXPECTED(long_power_measurement_p, HAILO_OUT_OF_HOST_MEMORY);
power_measurements.emplace(device.get().get_dev_id(), std::move(long_power_measurement_p));
}
std::map<std::string, std::shared_ptr<TemperatureMeasurement>> temp_measurements;
if (params.measure_temp) {
for (auto &device : physical_devices) {
- auto temp_measure = TemperatureMeasurement::create_shared(device);
- CHECK_EXPECTED(temp_measure);
- auto status = temp_measure.value()->start_measurement();
+ TRY(auto temp_measure, TemperatureMeasurement::create_shared(device));
+ auto status = temp_measure->start_measurement();
CHECK_SUCCESS_AS_EXPECTED(status, "Failed starting temperature measurement on device {}", device.get().get_dev_id());
- temp_measurements.emplace(device.get().get_dev_id(), temp_measure.release());
+ temp_measurements.emplace(device.get().get_dev_id(), temp_measure);
}
}
- auto infer_result = run_inference(network_groups, input_dataset.value(), params);
- CHECK_EXPECTED(infer_result, "Error failed running inference");
-
- InferResult inference_result(infer_result.release());
+ TRY(auto inference_result, run_inference(network_groups, input_dataset, params), "Error failed running inference");
inference_result.initialize_measurements(physical_devices);
if (should_measure_power) {
Expected<InferResult> run_command_hef_vdevice(const inference_runner_params ¶ms)
{
- auto hef = Hef::create(params.hef_path.c_str());
- CHECK_EXPECTED(hef, "Failed reading hef file {}", params.hef_path);
-
- auto network_groups_infos = hef->get_network_groups_infos();
- CHECK_EXPECTED(network_groups_infos);
- bool scheduler_is_used = (1 < network_groups_infos->size()) || params.vdevice_params.multi_process_service;
+ TRY(auto hef, Hef::create(params.hef_path.c_str()), "Failed reading hef file {}", params.hef_path);
+ TRY(auto network_groups_infos, hef.get_network_groups_infos());
+ bool scheduler_is_used = (1 < network_groups_infos.size()) || params.vdevice_params.multi_process_service;
hailo_vdevice_params_t vdevice_params = {};
auto status = hailo_init_vdevice_params(&vdevice_params);
}
std::vector<hailo_device_id_t> dev_ids;
if (!params.vdevice_params.device_params.device_ids.empty()) {
- auto dev_ids_exp = get_device_ids(params.vdevice_params.device_params);
- CHECK_EXPECTED(dev_ids_exp);
-
- auto dev_ids_struct_exp = HailoRTCommon::to_device_ids_vector(dev_ids_exp.value());
- CHECK_EXPECTED(dev_ids_struct_exp);
- dev_ids = dev_ids_struct_exp.release();
+ TRY(auto dev_ids_strs, get_device_ids(params.vdevice_params.device_params));
+ TRY(dev_ids, HailoRTCommon::to_device_ids_vector(dev_ids_strs));
vdevice_params.device_ids = dev_ids.data();
vdevice_params.device_count = static_cast<uint32_t>(dev_ids.size());
vdevice_params.scheduling_algorithm = (scheduler_is_used) ? HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN : HAILO_SCHEDULING_ALGORITHM_NONE;
vdevice_params.group_id = params.vdevice_params.group_id.c_str();
vdevice_params.multi_process_service = params.vdevice_params.multi_process_service;
- auto vdevice = VDevice::create(vdevice_params);
- CHECK_EXPECTED(vdevice, "Failed creating vdevice");
+ TRY(auto vdevice, VDevice::create(vdevice_params), "Failed creating vdevice");
std::vector<std::reference_wrapper<Device>> physical_devices;
if (!params.vdevice_params.multi_process_service) {
- auto expected_physical_devices = vdevice.value()->get_physical_devices();
- CHECK_EXPECTED(expected_physical_devices);
- physical_devices = expected_physical_devices.value();
+ TRY(physical_devices, vdevice->get_physical_devices());
}
- auto interface = vdevice.value()->get_default_streams_interface();
- CHECK_EXPECTED(interface, "Failed to get default streams interface");
-
- auto configure_params = get_configure_params(params, hef.value(), *interface);
- CHECK_EXPECTED(configure_params);
-
- auto network_group_list = vdevice.value()->configure(hef.value(), configure_params.value());
- CHECK_EXPECTED(network_group_list, "Failed configure vdevice from hef");
+ TRY(const auto interface, vdevice->get_default_streams_interface(), "Failed to get default streams interface");
+ TRY(auto configure_params, get_configure_params(params, hef, interface));
+ TRY(auto network_group_list, vdevice->configure(hef, configure_params), "Failed configure vdevice from hef");
for (auto &device : physical_devices) {
+ TRY(const auto identity, device.get().identify());
+ CHECK_AS_EXPECTED((HailoRTCommon::is_power_measurement_supported(identity.device_architecture) ||
+ !(params.power_measurement.measure_power)), HAILO_INVALID_OPERATION,
+ "HW arch {} does not support power measurement. Disable the power-measure option",
+ HailoRTCommon::get_device_arch_str(identity.device_architecture));
+ CHECK_AS_EXPECTED((HailoRTCommon::is_current_measurement_supported(identity.device_architecture) ||
+ !(params.power_measurement.measure_current)), HAILO_INVALID_OPERATION,
+ "HW arch {} does not support current measurement. Disable the current-measure option",
+ HailoRTCommon::get_device_arch_str(identity.device_architecture));
+ CHECK_AS_EXPECTED((HailoRTCommon::is_temp_measurement_supported(identity.device_architecture) ||
+ !(params.measure_temp)), HAILO_INVALID_OPERATION,
+ "HW arch {} does not support temperature measurement. Disable the temp-measure option",
+ HailoRTCommon::get_device_arch_str(identity.device_architecture));
+
if (use_batch_to_measure_opt(params)) {
status = DownloadActionListCommand::set_batch_to_measure(device.get(), params.runtime_data.batch_to_measure);
CHECK_SUCCESS_AS_EXPECTED(status);
}
}
- auto infer_result = activate_and_run_vdevice(physical_devices, scheduler_is_used, network_group_list.value(), params);
- CHECK_EXPECTED(infer_result, "Error failed running inference");
+ TRY(auto infer_result, activate_and_run_vdevice(physical_devices, scheduler_is_used, network_group_list, params),
+ "Error while running inference");
for (auto &device : physical_devices) {
- if (use_batch_to_measure_opt(params) && (0 == params.frames_count) && infer_result) {
- auto min_frames_count = get_min_inferred_frames_count(infer_result.value());
- CHECK_EXPECTED(min_frames_count);
- if (min_frames_count.value() < params.runtime_data.batch_to_measure) {
+ if (use_batch_to_measure_opt(params) && (0 == params.frames_count)) {
+ TRY(const auto min_frames_count, get_min_inferred_frames_count(infer_result));
+ if (min_frames_count < params.runtime_data.batch_to_measure) {
LOGGER__WARNING("Number of frames sent ({}) is smaller than --batch-to-measure ({}), "
- "hence timestamps will not be updated in runtime data", min_frames_count.value() ,
+ "hence timestamps will not be updated in runtime data", min_frames_count,
params.runtime_data.batch_to_measure);
}
}
auto output_path = (1 == physical_devices.size()) ? params.runtime_data.runtime_data_output_path :
(std::string(device.get().get_dev_id()) + "_" + params.runtime_data.runtime_data_output_path);
const auto runtime_data_output_path = format_runtime_data_output_path(output_path, params.hef_path);
- DownloadActionListCommand::execute(device.get(), runtime_data_output_path, network_group_list.value(),
+ DownloadActionListCommand::execute(device.get(), runtime_data_output_path, network_group_list,
params.hef_path);
}
}
return true;
}
- auto device_type = Device::get_device_type(params.device_params.device_ids[0]);
- CHECK_EXPECTED(device_type);
+ TRY(const auto device_type, Device::get_device_type(params.device_params.device_ids[0]));
- return device_type.value() != Device::Type::ETH;
+ return device_type != Device::Type::ETH;
}
Expected<InferResult> run_command_hef(const inference_runner_params ¶ms)
{
- auto use_vdevice_expected = use_vdevice(params.vdevice_params);
- CHECK_EXPECTED(use_vdevice_expected);
+ TRY(auto use_vdevice, use_vdevice(params.vdevice_params));
- if (use_vdevice_expected.value()) {
+ if (use_vdevice) {
return run_command_hef_vdevice(params);
}
else {
std::string hef_dir = params.hef_path;
inference_runner_params curr_params = params;
- const auto files = Filesystem::get_files_in_dir_flat(hef_dir);
- CHECK_EXPECTED_AS_STATUS(files);
+ TRY(const auto files, Filesystem::get_files_in_dir_flat(hef_dir));
- for (const auto &full_path : files.value()) {
+ for (const auto &full_path : files) {
if (Filesystem::has_suffix(full_path, ".hef")) {
contains_hef = true;
curr_params.hef_path = full_path;
std::cout << std::string(80, '*') << std::endl << "Inferring " << full_path << ":"<< std::endl;
- auto hef = Hef::create(full_path);
- CHECK_EXPECTED_AS_STATUS(hef);
- auto network_groups_names = hef->get_network_groups_names();
+ TRY(auto hef, Hef::create(full_path));
+ auto network_groups_names = hef.get_network_groups_names();
auto infer_stats = run_command_hef(curr_params);
printer.print(network_groups_names , infer_stats);
hailo_status run_command(const inference_runner_params ¶ms)
{
- auto printer = InferStatsPrinter::create(params);
- CHECK_EXPECTED_AS_STATUS(printer, "Failed to initialize infer stats printer");
+ TRY(auto printer, InferStatsPrinter::create(params), "Failed to initialize infer stats printer");
if (!params.csv_output.empty()) {
- printer->print_csv_header();
+ printer.print_csv_header();
}
- auto is_dir = Filesystem::is_directory(params.hef_path.c_str());
- CHECK_EXPECTED_AS_STATUS(is_dir, "Failed checking if path is directory");
+ TRY(const auto is_dir, Filesystem::is_directory(params.hef_path.c_str()), "Failed checking if path is directory");
- if (is_dir.value()){
- return run_command_hefs_dir(params, printer.value());
+ if (is_dir){
+ return run_command_hefs_dir(params, printer);
} else {
auto infer_stats = run_command_hef(params);
- auto hef = Hef::create(params.hef_path.c_str());
- CHECK_EXPECTED_AS_STATUS(hef);
- auto network_groups_names = hef->get_network_groups_names();
- printer->print(network_groups_names, infer_stats);
+ TRY(auto hef, Hef::create(params.hef_path.c_str()));
+ auto network_groups_names = hef.get_network_groups_names();
+ printer.print(network_groups_names, infer_stats);
return infer_stats.status();
}
}
return scan();
}
else {
- auto res = scan_ethernet(m_interface_ip_addr, m_interface_name);
- CHECK_EXPECTED_AS_STATUS(res);
+ TRY(const auto res, scan_ethernet(m_interface_ip_addr, m_interface_name));
return HAILO_SUCCESS;
}
}
hailo_status ScanSubcommand::scan()
{
- auto device_ids = Device::scan();
- CHECK_EXPECTED_AS_STATUS(device_ids);
-
- if (device_ids->size() == 0) {
+ TRY(const auto device_ids, Device::scan());
+ if (device_ids.size() == 0) {
std::cout << "Hailo devices not found" << std::endl;
}
else {
std::cout << "Hailo Devices:" << std::endl;
- for (const auto& device_id : device_ids.value()) {
+ for (const auto &device_id : device_ids) {
std::cout << "[-] Device: " << device_id << std::endl;
}
}
hailo_status SensorSectionsInfoSubcommand::execute_on_device(Device &device)
{
- auto sections_info = device.sensor_get_sections_info();
- CHECK_EXPECTED_AS_STATUS(sections_info);
- return print_sections_info((SENSOR_CONFIG__section_info_t*)sections_info->data());
+ TRY(auto sections_info, device.sensor_get_sections_info());
+ return print_sections_info((SENSOR_CONFIG__section_info_t*)sections_info.data());
}
hailo_status SensorSectionsInfoSubcommand::print_sections_info(SENSOR_CONFIG__section_info_t *operation_cfg)
}
else {
std::string reset_config = section_info->no_reset_offset ? "not valid" : "valid";
- auto sensor_type_expected = convert_sensor_type_to_string(section_info->sensor_type);
- CHECK_EXPECTED_AS_STATUS(sensor_type_expected, "Failed convert sensor type to string");
+ TRY( const auto sensor_type, convert_sensor_type_to_string(section_info->sensor_type),
+ "Failed convert sensor type to string");
std::cout << "Configuration Name: " << section_info->config_name << "\n";
- std::cout << "Sensor Type: " << sensor_type_expected.value() << "\n";
+ std::cout << "Sensor Type: " << sensor_type << "\n";
std::cout << "Configuration lines number: " << (section_info->config_size / sizeof(SENSOR_CONFIG__operation_cfg_t)) << "\n";
std::cout << "Configuration size in bytes: " << section_info->config_size << "\n";
std::cout << "Reset configuration: " << reset_config << "\n";
hailo_status UdpRateLimiterCommand::autoset_commnad(const std::vector<uint16_t> &board_ports)
{
- const auto rates_from_hef = calc_rate_from_hef(m_hef_path, m_network_group_name, m_fps);
- CHECK_EXPECTED_AS_STATUS(rates_from_hef);
+ TRY(const auto rates_from_hef, calc_rate_from_hef(m_hef_path, m_network_group_name, m_fps));
// On auto set, we use min rate for all input ports
- auto min_rate_pair = *std::min_element(rates_from_hef.value().begin(), rates_from_hef.value().end(),
+ auto min_rate_pair = *std::min_element(rates_from_hef.begin(), rates_from_hef.end(),
[](const auto& lhs, const auto& rhs) { return lhs.second < rhs.second; });
return set_command(board_ports, static_cast<uint32_t>(min_rate_pair.second));
Expected<std::map<std::string, uint32_t>> UdpRateLimiterCommand::calc_rate_from_hef(const std::string &hef_path,
const std::string &network_group_name, uint32_t fps)
{
- auto hef = Hef::create(hef_path.c_str());
- CHECK_EXPECTED(hef, "Failed reading hef file {}", hef_path.c_str());
+ TRY(auto hef, Hef::create(hef_path.c_str()), "Failed reading hef file {}", hef_path.c_str());
+ TRY(auto rate_calc, NetworkUdpRateCalculator::create(&(hef), network_group_name));
+ TRY(auto calculated_rates, rate_calc.calculate_inputs_bandwith(fps));
- auto rate_calc = NetworkUdpRateCalculator::create(&(hef.value()), network_group_name);
- CHECK_EXPECTED(rate_calc);
-
- auto calculated_rates = rate_calc->calculate_inputs_bandwith(fps);
- CHECK_EXPECTED(calculated_rates);
-
- return calculated_rates.release();
+ return calculated_rates;
}
std::vector<uint16_t> UdpRateLimiterCommand::get_dports()
--- /dev/null
+cmake_minimum_required(VERSION 3.0.0)
+
+set(HRPC_IMPL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/os")
+if(WIN32)
+ set(HRPC_OS_DIR "${HRPC_IMPL_DIR}/windows")
+elseif(UNIX)
+ set(HRPC_OS_DIR "${HRPC_IMPL_DIR}/posix")
+else()
+ message(FATAL_ERROR "Unexpeced host, stopping build")
+endif()
+
+set(SRC_FILES
+ ${CMAKE_CURRENT_SOURCE_DIR}/rpc_connection.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/raw_connection.cpp
+ ${HRPC_IMPL_DIR}/pcie/raw_connection_internal.cpp
+ ${HRPC_OS_DIR}/raw_connection_internal.cpp
+
+ ${CMAKE_CURRENT_SOURCE_DIR}/client.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/server.cpp
+)
+
+set(HRPC_CPP_SOURCES ${SRC_FILES} PARENT_SCOPE)
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file client.hpp
+ * @brief RPC Client
+ **/
+
+#include "client.hpp"
+
+using namespace hrpc;
+
+Expected<std::shared_ptr<ResultEvent>> ResultEvent::create_shared()
+{
+ TRY(auto event, hailort::Event::create_shared(hailort::Event::State::not_signalled));
+ auto ptr = make_shared_nothrow<ResultEvent>(event);
+ CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY);
+ return ptr;
+}
+
+ResultEvent::ResultEvent(EventPtr event) :
+ m_event(event)
+{
+}
+
+Buffer &&ResultEvent::release()
+{
+ return std::move(m_value);
+}
+
+hailo_status ResultEvent::signal(Buffer &&value)
+{
+ m_value = std::move(value);
+ return m_event->signal();
+}
+
+hailo_status ResultEvent::wait(std::chrono::milliseconds timeout)
+{
+ return m_event->wait(timeout);
+}
+
+Client::~Client()
+{
+ is_running = false;
+ (void)m_connection.close();
+ if (m_thread.joinable()) {
+ m_thread.join();
+ }
+}
+
+hailo_status Client::connect()
+{
+ TRY(m_conn_context, ConnectionContext::create_shared(false));
+ TRY(auto conn, RawConnection::create_shared(m_conn_context));
+ auto status = conn->connect();
+ CHECK_SUCCESS(status);
+
+ m_connection = RpcConnection(conn);
+ m_thread = std::thread([this] {
+ auto status = message_loop();
+ if ((status != HAILO_SUCCESS) && (status != HAILO_COMMUNICATION_CLOSED)) { // TODO: Use this to prevent future requests
+ LOGGER__ERROR("Error in message loop - {}", status);
+ }
+ });
+ return HAILO_SUCCESS;
+}
+
+hailo_status Client::message_loop()
+{
+ while (is_running) {
+ rpc_message_header_t header;
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_COMMUNICATION_CLOSED, auto message, m_connection.read_message(header));
+
+ assert(header.action_id < static_cast<uint32_t>(HailoRpcActionID::MAX_VALUE));
+ auto action_id_enum = static_cast<HailoRpcActionID>(header.action_id);
+ if (m_custom_callbacks.find(action_id_enum) != m_custom_callbacks.end()) {
+ auto status = m_custom_callbacks[action_id_enum](MemoryView(message), m_connection);
+ CHECK_SUCCESS(status);
+ continue;
+ }
+
+ std::unique_lock<std::mutex> lock(m_message_mutex);
+ auto event = m_events[header.message_id];
+ lock.unlock();
+ auto status = event->signal(std::move(message));
+ CHECK_SUCCESS(status);
+ }
+
+ return HAILO_SUCCESS;
+}
+
+Expected<Buffer> Client::execute_request(HailoRpcActionID action_id, const MemoryView &request,
+ std::function<hailo_status(RpcConnection)> write_buffers_callback)
+{
+ std::unique_lock<std::mutex> lock(m_message_mutex);
+ rpc_message_header_t header;
+ header.size = static_cast<uint32_t>(request.size());
+ header.message_id = m_messages_sent++;
+ header.action_id = static_cast<uint32_t>(action_id);
+
+ auto status = m_connection.write_message(header, request);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+ if (write_buffers_callback) {
+ status = write_buffers_callback(m_connection);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+ }
+
+ TRY(auto event, ResultEvent::create_shared());
+ m_events[header.message_id] = event;
+
+ lock.unlock();
+ status = event->wait(REQUEST_TIMEOUT);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ m_events.erase(header.message_id);
+ return event->release();
+}
+
+
+void Client::register_custom_reply(HailoRpcActionID action_id,
+ std::function<hailo_status(const MemoryView&, RpcConnection connection)> callback)
+{
+ m_custom_callbacks[action_id] = callback;
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file client.hpp
+ * @brief RPC Client Header
+ **/
+
+#ifndef _CLIENT_HPP_
+#define _CLIENT_HPP_
+
+#include <hailo/event.hpp>
+#include <fcntl.h>
+#include <functional>
+#include <thread>
+
+#include "rpc_connection.hpp"
+#include "hrpc_protocol/serializer.hpp"
+
+
+namespace hrpc
+{
+
+#define REQUEST_TIMEOUT std::chrono::milliseconds(10000)
+
+class ResultEvent
+{
+public:
+ static Expected<std::shared_ptr<ResultEvent>> create_shared();
+ ResultEvent(EventPtr event);
+
+ Buffer &&release();
+ hailo_status signal(Buffer &&value);
+ hailo_status wait(std::chrono::milliseconds timeout);
+
+private:
+ Buffer m_value;
+ EventPtr m_event;
+};
+
+class Client
+{
+public:
+ Client() = default;
+ ~Client();
+
+ hailo_status connect();
+ Expected<Buffer> execute_request(HailoRpcActionID action_id, const MemoryView &request,
+ std::function<hailo_status(RpcConnection)> write_buffers_callback = nullptr);
+ void register_custom_reply(HailoRpcActionID action_id, std::function<hailo_status(const MemoryView&, RpcConnection connection)> callback);
+
+protected:
+ hailo_status message_loop();
+
+ bool is_running = true;
+ std::shared_ptr<ConnectionContext> m_conn_context;
+ RpcConnection m_connection;
+ std::thread m_thread;
+ std::unordered_map<uint32_t, std::shared_ptr<ResultEvent>> m_events;
+ std::unordered_map<HailoRpcActionID, std::function<hailo_status(const MemoryView&, RpcConnection)>> m_custom_callbacks;
+ uint32_t m_messages_sent = 0;
+ std::mutex m_message_mutex;
+};
+
+} // namespace hrpc
+
+#endif // _CLIENT_HPP_
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file raw_connection_internal.cpp
+ * @brief PCIE Raw Connection
+ **/
+
+#include "hrpc/os/pcie/raw_connection_internal.hpp"
+#include "common/logger_macros.hpp"
+#include "common/utils.hpp"
+#include "hailo/hailort.h"
+#include "vdma/driver/hailort_driver.hpp"
+
+// TODO: Remove this after we can choose ports in the driver
+#define PCIE_PORT (1213355091)
+
+using namespace hrpc;
+
+Expected<std::shared_ptr<ConnectionContext>> PcieConnectionContext::create_shared(bool is_accepting)
+{
+ const auto max_size = PcieSession::max_transfer_size();
+ TRY(auto write_buffer, Buffer::create(static_cast<size_t>(max_size), BufferStorageParams::create_dma()));
+ TRY(auto read_buffer, Buffer::create(static_cast<size_t>(max_size), BufferStorageParams::create_dma()));
+
+ std::shared_ptr<PcieConnectionContext> ptr = nullptr;
+ if (is_accepting) {
+ // Server side
+ TRY(auto driver, HailoRTDriver::create_pcie_ep());
+ ptr = make_shared_nothrow<PcieConnectionContext>(std::move(driver), is_accepting,
+ std::move(write_buffer), std::move(read_buffer));
+ CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY);
+ return std::dynamic_pointer_cast<ConnectionContext>(ptr);
+ } else {
+ // Client side
+ TRY(auto device_infos, HailoRTDriver::scan_devices());
+ CHECK(device_infos.size() > 0, HAILO_NOT_FOUND, "No devices found");
+ for (auto &device_info : device_infos) {
+ if (HailoRTDriver::AcceleratorType::SOC_ACCELERATOR == device_info.accelerator_type) {
+ TRY(auto driver, HailoRTDriver::create(device_info.device_id, device_info.dev_path));
+ ptr = make_shared_nothrow<PcieConnectionContext>(std::move(driver), is_accepting,
+ std::move(write_buffer), std::move(read_buffer));
+ CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY);
+ return std::dynamic_pointer_cast<ConnectionContext>(ptr);
+ }
+ }
+ }
+ LOGGER__ERROR("No suitable device found");
+ return make_unexpected(HAILO_NOT_FOUND);
+}
+
+hailo_status PcieConnectionContext::wait_for_available_connection()
+{
+ std::unique_lock<std::mutex> lock(m_mutex);
+ bool was_successful = m_cv.wait_for(lock, std::chrono::milliseconds(HAILO_INFINITE), [this] () -> bool {
+ return (m_conn_count == 0);
+ });
+ CHECK(was_successful, HAILO_TIMEOUT, "Got timeout in accept");
+
+ m_conn_count++;
+ return HAILO_SUCCESS;
+}
+
+void PcieConnectionContext::mark_connection_closed()
+{
+ {
+ std::unique_lock<std::mutex> lock(m_mutex);
+ m_conn_count--;
+ }
+ m_cv.notify_one();
+}
+
+Expected<std::shared_ptr<RawConnection>> PcieRawConnection::create_shared(std::shared_ptr<PcieConnectionContext> context)
+{
+ auto ptr = make_shared_nothrow<PcieRawConnection>(context);
+ CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+ return std::dynamic_pointer_cast<RawConnection>(ptr);
+}
+
+Expected<std::shared_ptr<RawConnection>> PcieRawConnection::accept()
+{
+ auto status = m_context->wait_for_available_connection();
+ CHECK_SUCCESS(status);
+
+ auto new_conn = make_shared_nothrow<PcieRawConnection>(m_context);
+ CHECK_NOT_NULL_AS_EXPECTED(new_conn, HAILO_OUT_OF_HOST_MEMORY);
+
+ TRY(auto session, PcieSession::accept(m_context->driver(), PCIE_PORT));
+ status = new_conn->set_session(std::move(session));
+ CHECK_SUCCESS(status);
+
+ return std::dynamic_pointer_cast<RawConnection>(new_conn);
+}
+
+hailo_status PcieRawConnection::set_session(PcieSession &&session)
+{
+ m_session = make_shared_nothrow<PcieSession>(std::move(session));
+ CHECK_NOT_NULL(m_session, HAILO_OUT_OF_HOST_MEMORY);
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status PcieRawConnection::connect()
+{
+ TRY(auto session, PcieSession::connect(m_context->driver(), PCIE_PORT));
+ auto status = set_session(std::move(session));
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status PcieRawConnection::write(const uint8_t *buffer, size_t size)
+{
+ if (0 == size) {
+ return HAILO_SUCCESS;
+ }
+
+ const auto alignment = OsUtils::get_dma_able_alignment();
+ const auto max_size = PcieSession::max_transfer_size();
+ bool is_aligned = ((reinterpret_cast<uintptr_t>(buffer) % alignment )== 0);
+
+ size_t bytes_written = 0;
+ while (bytes_written < size) {
+ size_t amount_to_write = 0;
+ auto size_left = size - bytes_written;
+ if (is_aligned) {
+ amount_to_write = std::min(static_cast<size_t>(size_left), static_cast<size_t>(max_size));
+ auto status = m_session->write(buffer + bytes_written, amount_to_write, m_timeout);
+ if (HAILO_STREAM_ABORT == status) {
+ return HAILO_COMMUNICATION_CLOSED;
+ }
+ CHECK_SUCCESS(status);
+ } else {
+ amount_to_write = std::min(static_cast<size_t>(size_left), m_context->write_buffer().size());
+ memcpy(m_context->write_buffer().data(), buffer + bytes_written, amount_to_write);
+ auto status = m_session->write(m_context->write_buffer().data(), amount_to_write, m_timeout);
+ if (HAILO_STREAM_ABORT == status) {
+ return HAILO_COMMUNICATION_CLOSED;
+ }
+ CHECK_SUCCESS(status);
+ }
+
+ bytes_written += amount_to_write;
+ }
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status PcieRawConnection::read(uint8_t *buffer, size_t size)
+{
+ if (0 == size) {
+ return HAILO_SUCCESS;
+ }
+
+ const auto alignment = OsUtils::get_dma_able_alignment();
+ const auto max_size = PcieSession::max_transfer_size();
+ bool is_aligned = ((reinterpret_cast<uintptr_t>(buffer) % alignment) == 0);
+
+ size_t bytes_read = 0;
+ while (bytes_read < size) {
+ size_t amount_to_read = 0;
+ auto size_left = size - bytes_read;
+ if (is_aligned) {
+ amount_to_read = std::min(static_cast<size_t>(size_left), static_cast<size_t>(max_size));
+ auto status = m_session->read(buffer + bytes_read, amount_to_read, m_timeout);
+ if (HAILO_STREAM_ABORT == status) {
+ return HAILO_COMMUNICATION_CLOSED;
+ }
+ CHECK_SUCCESS(status);
+ } else {
+ amount_to_read = std::min(static_cast<size_t>(size_left), m_context->read_buffer().size());
+ auto status = m_session->read(m_context->read_buffer().data(), amount_to_read, m_timeout);
+ if (HAILO_STREAM_ABORT == status) {
+ return HAILO_COMMUNICATION_CLOSED;
+ }
+ CHECK_SUCCESS(status);
+
+ memcpy(buffer + bytes_read, m_context->read_buffer().data(), amount_to_read);
+ }
+
+ bytes_read += amount_to_read;
+ }
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status PcieRawConnection::close()
+{
+ auto status = m_session->close();
+ CHECK_SUCCESS(status);
+
+ m_context->mark_connection_closed();
+
+ return HAILO_SUCCESS;
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file raw_connection_internal.hpp
+ * @brief Raw Connection Header for pcie based comunication
+ **/
+
+#ifndef _PCIE_RAW_CONNECTION_INTERNAL_HPP_
+#define _PCIE_RAW_CONNECTION_INTERNAL_HPP_
+
+#include "hailo/expected.hpp"
+#include "vdma/pcie_session.hpp"
+#include "hrpc/raw_connection.hpp"
+
+#include <memory>
+#include <condition_variable>
+
+using namespace hailort;
+
+namespace hrpc
+{
+
+class PcieConnectionContext : public ConnectionContext
+{
+public:
+ static Expected<std::shared_ptr<ConnectionContext>> create_shared(bool is_accepting);
+
+ PcieConnectionContext(std::shared_ptr<HailoRTDriver> &&driver, bool is_accepting,
+ Buffer &&write_buffer, Buffer &&read_buffer)
+ : ConnectionContext(is_accepting), m_driver(std::move(driver)),
+ m_write_buffer(std::move(write_buffer)), m_read_buffer(std::move(read_buffer)),
+ m_conn_count(0) {}
+
+ virtual ~PcieConnectionContext() = default;
+
+ std::shared_ptr<HailoRTDriver> driver() { return m_driver; }
+ Buffer &write_buffer() { return m_write_buffer; }
+ Buffer &read_buffer() { return m_read_buffer; }
+
+ hailo_status wait_for_available_connection();
+ void mark_connection_closed();
+
+private:
+ std::shared_ptr<HailoRTDriver> m_driver;
+ Buffer m_write_buffer;
+ Buffer m_read_buffer;
+ uint32_t m_conn_count;
+ std::mutex m_mutex;
+ std::condition_variable m_cv;
+};
+
+class PcieRawConnection : public RawConnection
+{
+public:
+ static Expected<std::shared_ptr<RawConnection>> create_shared(std::shared_ptr<PcieConnectionContext> context);
+
+ PcieRawConnection() = default;
+ virtual ~PcieRawConnection() = default;
+
+ virtual Expected<std::shared_ptr<RawConnection>> accept() override;
+ virtual hailo_status connect() override;
+ virtual hailo_status write(const uint8_t *buffer, size_t size) override;
+ virtual hailo_status read(uint8_t *buffer, size_t size) override;
+ virtual hailo_status close() override;
+
+ explicit PcieRawConnection(std::shared_ptr<PcieConnectionContext> context) : m_context(context) {}
+private:
+ hailo_status set_session(PcieSession &&session);
+
+ std::shared_ptr<PcieConnectionContext> m_context;
+ std::shared_ptr<PcieSession> m_session;
+};
+
+} // namespace hrpc
+
+#endif // _PCIE_RAW_CONNECTION_INTERNAL_HPP_
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file raw_connection_internal.cpp
+ * @brief Linux Sockets Raw Connection
+ **/
+
+#include "hrpc/os/posix/raw_connection_internal.hpp"
+
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <string>
+#include <unistd.h>
+#include <common/logger_macros.hpp>
+#include <common/utils.hpp>
+#include <hailo/hailort.h>
+
+using namespace hrpc;
+
+Expected<std::shared_ptr<ConnectionContext>> OsConnectionContext::create_shared(bool is_accepting)
+{
+ auto ptr = make_shared_nothrow<OsConnectionContext>(is_accepting);
+ CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return std::dynamic_pointer_cast<ConnectionContext>(ptr);
+}
+
+Expected<std::shared_ptr<RawConnection>> OsRawConnection::create_shared(std::shared_ptr<OsConnectionContext> context)
+{
+ std::shared_ptr<RawConnection> ptr;
+ if (context->is_accepting()) {
+ int fd = ::socket(AF_UNIX, SOCK_STREAM, 0);
+ CHECK_AS_EXPECTED(fd >= 0, HAILO_OPEN_FILE_FAILURE, "Socket creation error, errno = {}", errno);
+
+ struct sockaddr_un server_addr;
+ memset(&server_addr, 0, sizeof(server_addr));
+ server_addr.sun_family = AF_UNIX;
+ std::string addr = "/tmp/unix_socket";
+ strncpy(server_addr.sun_path, addr.c_str(), addr.size());
+
+ unlink(addr.c_str());
+ int result = ::bind(fd, (struct sockaddr*)&server_addr, sizeof(server_addr));
+ CHECK_AS_EXPECTED(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Bind error, errno = {}", errno);
+
+ result = ::listen(fd, 5);
+ CHECK_AS_EXPECTED(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Listen error, errno = {}", errno);
+
+ ptr = make_shared_nothrow<OsRawConnection>(fd, context);
+ } else {
+
+ int fd = ::socket(AF_UNIX, SOCK_STREAM, 0);
+ CHECK_AS_EXPECTED(fd >= 0, HAILO_OPEN_FILE_FAILURE, "Socket creation error, errno = {}", errno);
+ ptr = make_shared_nothrow<OsRawConnection>(fd, context);
+ }
+
+ CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+ return ptr;
+}
+
+Expected<std::shared_ptr<RawConnection>> OsRawConnection::accept()
+{
+ int fd = ::accept(m_fd, nullptr, nullptr);
+ CHECK_AS_EXPECTED(fd >= 0, HAILO_FILE_OPERATION_FAILURE, "Accept error, errno = {}", errno);
+
+ std::shared_ptr<RawConnection> ptr = make_shared_nothrow<OsRawConnection>(fd, m_context);
+ CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return ptr;
+}
+
+hailo_status OsRawConnection::connect()
+{
+ struct sockaddr_un server_addr;
+ std::string addr = "/tmp/unix_socket";
+
+ memset(&server_addr, 0, sizeof(server_addr));
+ server_addr.sun_family = AF_UNIX;
+ strncpy(server_addr.sun_path, addr.c_str(), addr.size());
+
+ int result = ::connect(m_fd, (struct sockaddr*)&server_addr, sizeof(server_addr));
+ CHECK(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Connect error, errno = {}", errno);
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status OsRawConnection::write(const uint8_t *buffer, size_t size)
+{
+ size_t bytes_written = 0;
+ while (bytes_written < size) {
+ ssize_t result = ::send(m_fd, buffer + bytes_written, size - bytes_written, MSG_NOSIGNAL);
+ CHECK(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Write error, errno = {}", errno);
+ bytes_written += result;
+ }
+ return HAILO_SUCCESS;
+}
+
+hailo_status OsRawConnection::read(uint8_t *buffer, size_t size)
+{
+ size_t bytes_read = 0;
+ while (bytes_read < size) {
+ ssize_t result = ::read(m_fd, buffer + bytes_read, size - bytes_read);
+ if (0 == result) {
+ return HAILO_COMMUNICATION_CLOSED; // 0 means the communication is closed
+ }
+ CHECK(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Read error, errno = {}", errno);
+ bytes_read += result;
+ }
+ return HAILO_SUCCESS;
+}
+
+hailo_status OsRawConnection::close()
+{
+ int result = ::shutdown(m_fd, SHUT_RDWR);
+ CHECK(0 == result, HAILO_CLOSE_FAILURE, "Socket shutdown failed, errno = {}", errno);
+
+ result = ::close(m_fd);
+ CHECK(0 == result, HAILO_CLOSE_FAILURE, "Socket close failed, errno = {}", errno);
+
+ return HAILO_SUCCESS;
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file raw_connection_internal.hpp
+ * @brief Raw Connection Header for sockets based comunication
+ **/
+
+#ifndef _POSIX_RAW_CONNECTION_INTERNAL_HPP_
+#define _POSIX_RAW_CONNECTION_INTERNAL_HPP_
+
+#include "hailo/expected.hpp"
+#include "hrpc/raw_connection.hpp"
+
+#include <memory>
+
+using namespace hailort;
+
+namespace hrpc
+{
+
+class OsConnectionContext : public ConnectionContext
+{
+public:
+ static Expected<std::shared_ptr<ConnectionContext>> create_shared(bool is_accepting);
+
+ OsConnectionContext(bool is_accepting) : ConnectionContext(is_accepting) {}
+
+ virtual ~OsConnectionContext() = default;
+};
+
+class OsRawConnection : public RawConnection
+{
+public:
+ static Expected<std::shared_ptr<RawConnection>> create_shared(std::shared_ptr<OsConnectionContext> context);
+
+ virtual ~OsRawConnection() = default;
+
+ virtual Expected<std::shared_ptr<RawConnection>> accept() override;
+ virtual hailo_status connect() override;
+ virtual hailo_status write(const uint8_t *buffer, size_t size) override;
+ virtual hailo_status read(uint8_t *buffer, size_t size) override;
+ virtual hailo_status close() override;
+
+ OsRawConnection(int fd, std::shared_ptr<OsConnectionContext> context) : m_fd(fd), m_context(context) {}
+private:
+ int m_fd;
+ std::shared_ptr<OsConnectionContext> m_context;
+};
+
+} // namespace hrpc
+
+#endif // _POSIX_RAW_CONNECTION_INTERNAL_HPP_
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file raw_connection_internal.cpp
+ * @brief Windows Sockets Raw Connection
+ **/
+
+#include "hrpc/os/windows/raw_connection_internal.hpp"
+
+#include "common/logger_macros.hpp"
+#include "common/utils.hpp"
+#include "hailo/hailort.h"
+
+using namespace hrpc;
+
+Expected<std::shared_ptr<ConnectionContext>> OsConnectionContext::create_shared(bool is_accepting)
+{
+ (void)is_accepting;
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+Expected<std::shared_ptr<RawConnection>> OsRawConnection::create_shared(std::shared_ptr<OsConnectionContext> context)
+{
+ (void)context;
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+Expected<std::shared_ptr<RawConnection>> OsRawConnection::accept()
+{
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+hailo_status OsRawConnection::connect()
+{
+ return HAILO_NOT_IMPLEMENTED;
+}
+
+hailo_status OsRawConnection::write(const uint8_t *buffer, size_t size)
+{
+ (void)buffer;
+ (void)size;
+ return HAILO_NOT_IMPLEMENTED;
+}
+
+hailo_status OsRawConnection::read(uint8_t *buffer, size_t size)
+{
+ (void)buffer;
+ (void)size;
+ return HAILO_NOT_IMPLEMENTED;
+}
+
+hailo_status OsRawConnection::close()
+{
+ return HAILO_NOT_IMPLEMENTED;
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file raw_connection_internal.hpp
+ * @brief Raw Connection Header for sockets based comunication
+ **/
+
+#ifndef _WINDOWS_RAW_CONNECTION_INTERNAL_HPP_
+#define _WINDOWS_RAW_CONNECTION_INTERNAL_HPP_
+
+#include "hailo/expected.hpp"
+#include "hrpc/raw_connection.hpp"
+
+#include <memory>
+
+using namespace hailort;
+
+namespace hrpc
+{
+
+class OsConnectionContext : public ConnectionContext
+{
+public:
+ static Expected<std::shared_ptr<ConnectionContext>> create_shared(bool is_accepting);
+};
+
+class OsRawConnection : public RawConnection
+{
+public:
+ static Expected<std::shared_ptr<RawConnection>> create_shared(std::shared_ptr<OsConnectionContext> context);
+
+ OsRawConnection() = default;
+ virtual ~OsRawConnection() = default;
+
+ virtual Expected<std::shared_ptr<RawConnection>> accept() override;
+ virtual hailo_status connect() override;
+ virtual hailo_status write(const uint8_t *buffer, size_t size) override;
+ virtual hailo_status read(uint8_t *buffer, size_t size) override;
+ virtual hailo_status close() override;
+
+ explicit OsRawConnection(std::shared_ptr<OsConnectionContext> /*context*/) {}
+};
+
+} // namespace hrpc
+
+#endif // _WINDOWS_RAW_CONNECTION_INTERNAL_HPP_
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file raw_connection.cpp
+ * @brief Raw Connection
+ **/
+
+#include "hailo/vdevice.hpp"
+#include "hrpc/raw_connection.hpp"
+#include "hrpc/os/pcie/raw_connection_internal.hpp"
+
+#ifdef _WIN32
+#include "hrpc/os/windows/raw_connection_internal.hpp"
+#else
+#include "hrpc/os/posix/raw_connection_internal.hpp"
+#endif
+
+#define HAILO_FORCE_SOCKET_COM_ENV_VAR "HAILO_FORCE_SOCKET_COM"
+
+using namespace hrpc;
+
+
+Expected<std::shared_ptr<ConnectionContext>> ConnectionContext::create_shared(bool is_accepting)
+{
+ // The env var HAILO_FORCE_HRPC_CLIENT_ENV_VAR is supported for debug purposes
+ char *socket_com = std::getenv(HAILO_FORCE_SOCKET_COM_ENV_VAR); // TODO: Remove duplication
+ auto force_socket_com = (nullptr != socket_com) && ("1" == std::string(socket_com));
+
+ if (force_socket_com || VDevice::force_hrpc_client()) {// If forcing hrpc service, its because we work without EP driver -> use sockets
+ return OsConnectionContext::create_shared(is_accepting);
+ } else {
+ return PcieConnectionContext::create_shared(is_accepting);
+ }
+}
+
+Expected<std::shared_ptr<RawConnection>> RawConnection::create_shared(std::shared_ptr<ConnectionContext> context)
+{
+ // Create according to ConnectionContext type
+ auto os_connection_context = std::dynamic_pointer_cast<OsConnectionContext>(context);
+ if (os_connection_context != nullptr) {
+ return OsRawConnection::create_shared(os_connection_context);
+ } else {
+ return PcieRawConnection::create_shared(std::dynamic_pointer_cast<PcieConnectionContext>(context));
+ }
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file raw_connection.hpp
+ * @brief Raw Connection Header
+ **/
+
+#ifndef _RAW_CONNECTION_HPP_
+#define _RAW_CONNECTION_HPP_
+
+#include "hailo/expected.hpp"
+#include "vdma/pcie_session.hpp"
+
+#include <memory>
+
+using namespace hailort;
+
+namespace hrpc
+{
+
+class ConnectionContext
+{
+public:
+ static Expected<std::shared_ptr<ConnectionContext>> create_shared(bool is_accepting);
+
+ bool is_accepting() const { return m_is_accepting; }
+
+ ConnectionContext(bool is_accepting) : m_is_accepting(is_accepting) {}
+ virtual ~ConnectionContext() = default;
+
+protected:
+ bool m_is_accepting;
+};
+
+
+class RawConnection
+{
+public:
+ static Expected<std::shared_ptr<RawConnection>> create_shared(std::shared_ptr<ConnectionContext> context);
+
+ RawConnection() = default;
+ virtual ~RawConnection() = default;
+
+ virtual Expected<std::shared_ptr<RawConnection>> accept() = 0;
+ virtual hailo_status connect() = 0;
+ virtual hailo_status write(const uint8_t *buffer, size_t size) = 0;
+ virtual hailo_status read(uint8_t *buffer, size_t size) = 0;
+ virtual hailo_status close() = 0;
+
+protected:
+ std::chrono::milliseconds m_timeout = std::chrono::milliseconds(HAILO_INFINITE);
+};
+
+} // namespace hrpc
+
+#endif // _RAW_CONNECTION_HPP_
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file rpc_connection.cpp
+ * @brief RPC connection implementation
+ **/
+
+#include "rpc_connection.hpp"
+
+namespace hrpc
+{
+
+hailo_status RpcConnection::write_message(const rpc_message_header_t &header, const MemoryView &buffer) {
+ auto header_with_magic = header;
+ header_with_magic.magic = RPC_MESSAGE_MAGIC;
+ auto status = m_raw->write(reinterpret_cast<const uint8_t*>(&header_with_magic), sizeof(header_with_magic));
+ if (HAILO_COMMUNICATION_CLOSED == status) {
+ return make_unexpected(status);
+ }
+ CHECK_SUCCESS(status);
+
+ status = m_raw->write(buffer.data(), header.size);
+ if (HAILO_COMMUNICATION_CLOSED == status) {
+ return make_unexpected(status);
+ }
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
+Expected<Buffer> RpcConnection::read_message(rpc_message_header_t &header) {
+ auto status = m_raw->read(reinterpret_cast<uint8_t*>(&header), sizeof(header));
+ if (HAILO_COMMUNICATION_CLOSED == status) {
+ return make_unexpected(status);
+ }
+ CHECK_SUCCESS_AS_EXPECTED(status);
+ CHECK_AS_EXPECTED(RPC_MESSAGE_MAGIC == header.magic, HAILO_INTERNAL_FAILURE, "Invalid magic! {} != {}",
+ header.magic, RPC_MESSAGE_MAGIC);
+
+ TRY(auto buffer, Buffer::create(header.size, BufferStorageParams::create_dma()));
+ status = m_raw->read(buffer.data(), header.size);
+ if (HAILO_COMMUNICATION_CLOSED == status) {
+ return make_unexpected(status);
+ }
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ return buffer;
+}
+
+hailo_status RpcConnection::write_buffer(const MemoryView &buffer)
+{
+ auto status = m_raw->write(buffer.data(), buffer.size());
+ if (HAILO_COMMUNICATION_CLOSED == status) {
+ return make_unexpected(status);
+ }
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status RpcConnection::read_buffer(MemoryView buffer)
+{
+ auto status = m_raw->read(buffer.data(), buffer.size());
+ if (HAILO_COMMUNICATION_CLOSED == status) {
+ return make_unexpected(status);
+ }
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status RpcConnection::close()
+{
+ if (m_raw) {
+ return m_raw->close();
+ }
+ return HAILO_SUCCESS;
+}
+
+} // namespace hrpc
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file rpc_connection.hpp
+ * @brief RPC Connection Header
+ **/
+
+#ifndef _RPC_CONNECTION_HPP_
+#define _RPC_CONNECTION_HPP_
+
+#include "raw_connection.hpp"
+
+#include "hailo/buffer.hpp"
+#include "common/utils.hpp"
+
+#define RPC_MESSAGE_MAGIC (0x8A554432)
+
+
+namespace hrpc
+{
+
+#pragma pack(push, 1)
+struct rpc_message_header_t
+{
+ uint32_t magic; // TODO: consider removing. check if hurts performance
+ uint32_t size;
+ uint32_t message_id;
+ uint32_t action_id;
+};
+#pragma pack(pop)
+
+class RpcConnection
+{
+public:
+ RpcConnection() = default;
+ explicit RpcConnection(std::shared_ptr<RawConnection> raw) : m_raw(raw) {}
+
+ hailo_status write_message(const rpc_message_header_t &header, const MemoryView &buffer);
+ Expected<Buffer> read_message(rpc_message_header_t &header);
+
+ hailo_status write_buffer(const MemoryView &buffer);
+ hailo_status read_buffer(MemoryView buffer);
+
+ hailo_status close();
+
+private:
+ std::shared_ptr<RawConnection> m_raw;
+};
+
+} // namespace hrpc
+
+#endif // _RPC_CONNECTION_HPP_
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file server.cpp
+ * @brief RPC Server
+ **/
+
+#include "server.hpp"
+
+namespace hrpc
+{
+
+ServerContext::ServerContext(Server &server, RpcConnection connection) :
+ m_server(server), m_connection(connection) {}
+
+hailo_status ServerContext::trigger_callback(uint32_t callback_id, hailo_status callback_status, std::function<hailo_status(RpcConnection)> write_buffers_callback)
+{
+ return m_server.trigger_callback(callback_id, m_connection, callback_status, write_buffers_callback);
+}
+
+RpcConnection &ServerContext::connection()
+{
+ return m_connection;
+}
+
+void Dispatcher::register_action(HailoRpcActionID action_id,
+ std::function<Expected<Buffer>(const MemoryView&, ServerContextPtr)> action)
+{
+ m_actions[action_id] = action;
+}
+
+Expected<Buffer> Dispatcher::call_action(HailoRpcActionID action_id, const MemoryView &request, ServerContextPtr server_context)
+{
+ if (m_actions.find(action_id) != m_actions.end()) {
+ return m_actions[action_id](request, server_context);
+ }
+ LOGGER__ERROR("Failed to find RPC action {}", action_id);
+ return make_unexpected(HAILO_RPC_FAILED);
+}
+
+hailo_status Server::serve()
+{
+ while (true) {
+ TRY(auto client_connection, create_client_connection());
+ auto th = std::thread([this, client_connection]() { serve_client(client_connection); });
+ th.detach();
+ }
+ return HAILO_SUCCESS;
+}
+
+void Server::set_dispatcher(Dispatcher dispatcher)
+{
+ m_dispatcher = dispatcher;
+}
+
+Expected<RpcConnection> Server::create_client_connection()
+{
+ TRY(auto server_connection, RawConnection::create_shared(m_connection_context));
+ TRY(auto conn, server_connection->accept());
+
+ return RpcConnection(conn);
+}
+
+hailo_status Server::serve_client(RpcConnection client_connection)
+{
+ auto server_context = make_shared_nothrow<ServerContext>(*this, client_connection);
+ CHECK_NOT_NULL(server_context, HAILO_OUT_OF_HOST_MEMORY);
+ while (true) {
+ rpc_message_header_t header;
+ auto request = client_connection.read_message(header);
+ if (HAILO_COMMUNICATION_CLOSED == request.status()) {
+ cleanup_client_resources(client_connection);
+ break; // Client EP is disconnected, exit this loop
+ }
+ CHECK_EXPECTED_AS_STATUS(request);
+
+ assert(header.action_id < static_cast<uint32_t>(HailoRpcActionID::MAX_VALUE));
+ TRY(auto reply, m_dispatcher.call_action(static_cast<HailoRpcActionID>(header.action_id), MemoryView(*request), server_context));
+ {
+ std::unique_lock<std::mutex> lock(m_write_mutex);
+ header.size = static_cast<uint32_t>(reply.size());
+
+ auto status = client_connection.write_message(header, MemoryView(reply));
+ if ((HAILO_COMMUNICATION_CLOSED == status) || (HAILO_FILE_OPERATION_FAILURE == status)) {
+ lock.unlock(); // We need to acquire this lock when releasing the client resources (trigger cb)
+ cleanup_client_resources(client_connection);
+ break; // Client EP is disconnected, exit this loop
+ }
+ CHECK_SUCCESS(status);
+ }
+ }
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status Server::trigger_callback(uint32_t callback_id, RpcConnection connection, hailo_status callback_status,
+ std::function<hailo_status(RpcConnection)> write_buffers_callback)
+{
+ TRY(auto reply, CallbackCalledSerializer::serialize_reply(callback_status, callback_id));
+
+ std::unique_lock<std::mutex> lock(m_write_mutex);
+ rpc_message_header_t header;
+ header.action_id = static_cast<uint32_t>(HailoRpcActionID::CALLBACK_CALLED);
+ header.message_id = callback_id;
+ header.size = static_cast<uint32_t>(reply.size());
+
+ auto status = connection.write_message(header, MemoryView(reply));
+ if ((HAILO_COMMUNICATION_CLOSED == status) || (HAILO_FILE_OPERATION_FAILURE == status)) {
+ return status;
+ }
+ CHECK_SUCCESS(status);
+
+ if (write_buffers_callback) {
+ status = write_buffers_callback(connection);
+ CHECK_SUCCESS(status);
+ }
+
+ return HAILO_SUCCESS;
+}
+
+} // namespace hrpc
--- /dev/null
+#ifndef _SERVER_HPP_
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file server.hpp
+ * @brief RPC Server Header
+ **/
+
+#define _SERVER_HPP_
+
+#include <functional>
+#include <thread>
+
+#include "rpc_connection.hpp"
+#include "hailort_service/service_resource_manager.hpp"
+#include "hrpc_protocol/serializer.hpp"
+
+
+namespace hrpc
+{
+
+class Server;
+class ServerContext
+{
+public:
+ ServerContext(Server &server, RpcConnection connection);
+ hailo_status trigger_callback(uint32_t callback_id, hailo_status callback_status,
+ std::function<hailo_status(RpcConnection)> write_buffers_callback = nullptr);
+ RpcConnection &connection();
+
+private:
+ Server &m_server;
+ RpcConnection m_connection;
+};
+using ServerContextPtr = std::shared_ptr<ServerContext>;
+
+class Dispatcher
+{
+public:
+ Dispatcher() = default;
+
+ void register_action(HailoRpcActionID action_id,
+ std::function<Expected<Buffer>(const MemoryView&, ServerContextPtr)> action);
+ Expected<Buffer> call_action(HailoRpcActionID action_id, const MemoryView &request, ServerContextPtr server_context);
+
+private:
+ std::unordered_map<HailoRpcActionID, std::function<Expected<Buffer>(const MemoryView&, ServerContextPtr)>> m_actions;
+};
+
+class Server
+{
+public:
+ Server(std::shared_ptr<ConnectionContext> connection_context) : m_connection_context(connection_context) {};
+ virtual ~Server() = default;
+
+ hailo_status serve();
+
+ void set_dispatcher(Dispatcher dispatcher);
+
+ friend class ServerContext;
+protected:
+ std::shared_ptr<ConnectionContext> m_connection_context;
+private:
+ Expected<RpcConnection> create_client_connection();
+ hailo_status serve_client(RpcConnection client_connection);
+ hailo_status trigger_callback(uint32_t callback_id, RpcConnection connection, hailo_status callback_status,
+ std::function<hailo_status(RpcConnection)> write_buffers_callback = nullptr);
+ virtual hailo_status cleanup_client_resources(RpcConnection client_connection) = 0;
+
+ Dispatcher m_dispatcher;
+ std::mutex m_write_mutex;
+};
+
+} // namespace hrpc
+
+#endif // _SERVER_HPP_
\ No newline at end of file
--- /dev/null
+cmake_minimum_required(VERSION 3.0.0)
+
+protobuf_generate_cpp(PROTO_RPC_SRC PROTO_RPC_HEADER rpc.proto)
+get_filename_component(PROTO_HEADER_DIRECTORY ${PROTO_RPC_HEADER} DIRECTORY)
+
+add_library(rpc_proto STATIC EXCLUDE_FROM_ALL ${PROTO_RPC_SRC} ${PROTO_RPC_HEADER})
+target_link_libraries(rpc_proto libprotobuf-lite)
+set_target_properties(rpc_proto PROPERTIES CXX_STANDARD 14 GENERATED TRUE POSITION_INDEPENDENT_CODE ON)
+if(CMAKE_HOST_WIN32)
+ # https://github.com/protocolbuffers/protobuf/tree/master/cmake#notes-on-compiler-warnings
+ target_compile_options(rpc_proto PRIVATE /wd4244)
+endif()
+get_filename_component(PROTO_HEADER_DIRECTORY ${PROTO_RPC_HEADER} DIRECTORY)
+target_include_directories(rpc_proto
+ PUBLIC
+ $<BUILD_INTERFACE: ${PROTO_HEADER_DIRECTORY}>
+ $<BUILD_INTERFACE: ${Protobuf_INCLUDE_DIRS}>
+)
+
+set(SRC_FILES
+ ${CMAKE_CURRENT_SOURCE_DIR}/serializer.cpp
+)
+
+set(HRPC_PROTOCOL_CPP_SOURCES ${SRC_FILES} PARENT_SCOPE)
\ No newline at end of file
--- /dev/null
+syntax = "proto3";
+
+option optimize_for = LITE_RUNTIME;
+
+message RpcRequest {
+ oneof request {
+ VDevice_Create_Request create_vdevice_request = 1;
+ VDevice_Destroy_Request destroy_vdevice_request = 2;
+ VDevice_CreateInferModel_Request create_infer_model_request = 3;
+
+ InferModel_Destroy_Request destroy_infer_model_request = 4;
+ InferModel_CreateConfiguredInferModel_Request create_configured_infer_model_request = 5;
+
+ ConfiguredInferModel_Destroy_Request destroy_configured_infer_model_request = 6;
+ ConfiguredInferModel_SetSchedulerTimeout_Request set_scheduler_timeout_request = 7;
+ ConfiguredInferModel_SetSchedulerThreshold_Request set_scheduler_threshold_request = 8;
+ ConfiguredInferModel_SetSchedulerPriority_Request set_scheduler_priority_request = 9;
+ ConfiguredInferModel_GetHwLatencyMeasurement_Request get_hw_latency_measurement_request = 10;
+ ConfiguredInferModel_Activate_Request activate_request = 11;
+ ConfiguredInferModel_Deactivate_Request deactivate_request = 12;
+ ConfiguredInferModel_Shutdown_Request shutdown_request = 13;
+ ConfiguredInferModel_AsyncInfer_Request async_infer_request = 14;
+ }
+}
+
+message RpcReply {
+ oneof reply {
+ VDevice_Create_Reply create_vdevice_reply = 1;
+ VDevice_Destroy_Reply destroy_vdevice_reply = 2;
+ VDevice_CreateInferModel_Reply create_infer_model_reply = 3;
+
+ InferModel_Destroy_Reply destroy_infer_model_reply = 4;
+ InferModel_CreateConfiguredInferModel_Reply create_configured_infer_model_reply = 5;
+
+ ConfiguredInferModel_Destroy_Reply destroy_configured_infer_model_reply = 6;
+ ConfiguredInferModel_SetSchedulerTimeout_Reply set_scheduler_timeout_reply = 7;
+ ConfiguredInferModel_SetSchedulerThreshold_Reply set_scheduler_threshold_reply = 8;
+ ConfiguredInferModel_SetSchedulerPriority_Reply set_scheduler_priority_reply = 9;
+ ConfiguredInferModel_GetHwLatencyMeasurement_Reply get_hw_latency_measurement_reply = 10;
+ ConfiguredInferModel_Activate_Reply activate_reply = 11;
+ ConfiguredInferModel_Deactivate_Reply deactivate_reply = 12;
+ ConfiguredInferModel_Shutdown_Reply shutdown_reply = 13;
+ ConfiguredInferModel_AsyncInfer_Reply async_infer_reply = 14;
+
+ CallbackCalled_Reply callback_called_reply = 15;
+ }
+}
+
+message HailoObjectHandle {
+ uint32 id = 1;
+}
+
+message HailoCallbackHandle {
+ uint32 id = 1;
+}
+
+message VDeviceParamsProto {
+ uint32 scheduling_algorithm = 1;
+ string group_id = 2;
+}
+
+message VDevice_Create_Request {
+ VDeviceParamsProto params = 1;
+}
+
+message VDevice_Create_Reply {
+ uint32 status = 1;
+ HailoObjectHandle vdevice_handle = 2;
+}
+
+message VDevice_Destroy_Request {
+ HailoObjectHandle vdevice_handle = 1;
+}
+
+message VDevice_Destroy_Reply {
+ uint32 status = 1;
+}
+
+message VDevice_CreateInferModel_Request {
+ HailoObjectHandle vdevice_handle = 1;
+ uint64 hef_size = 2;
+ // Protocol note: After this message, server expects to get HEF data (buffer of size 'hef_size')
+}
+
+message VDevice_CreateInferModel_Reply {
+ uint32 status = 1;
+ HailoObjectHandle infer_model_handle = 2;
+}
+
+message InferModel_Destroy_Request {
+ HailoObjectHandle infer_model_handle = 1;
+}
+
+message InferModel_Destroy_Reply {
+ uint32 status = 1;
+}
+
+message InferStreamParamsProto {
+ string name = 1;
+ uint32 format_order = 2;
+ uint32 format_type = 3;
+ float nms_score_threshold = 4;
+ float nms_iou_threshold = 5;
+ uint32 nms_max_proposals_per_class = 6;
+ uint32 nms_max_accumulated_mask_size = 7;
+};
+
+message InferModel_CreateConfiguredInferModel_Request {
+ HailoObjectHandle infer_model_handle = 1;
+ HailoObjectHandle vdevice_handle = 2;
+ repeated InferStreamParamsProto input_infer_streams = 3;
+ repeated InferStreamParamsProto output_infer_streams = 4;
+ uint32 batch_size = 5;
+ uint32 power_mode = 6;
+ uint32 latency_flag = 7;
+}
+
+message InferModel_CreateConfiguredInferModel_Reply {
+ uint32 status = 1;
+ HailoObjectHandle configured_infer_model_handle = 2;
+ uint32 async_queue_size = 3;
+}
+
+message ConfiguredInferModel_Destroy_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+}
+
+message ConfiguredInferModel_Destroy_Reply {
+ uint32 status = 1;
+}
+
+message ConfiguredInferModel_SetSchedulerTimeout_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+ uint32 timeout = 2;
+}
+
+message ConfiguredInferModel_SetSchedulerTimeout_Reply {
+ uint32 status = 1;
+}
+
+message ConfiguredInferModel_SetSchedulerThreshold_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+ uint32 threshold = 2;
+}
+
+message ConfiguredInferModel_SetSchedulerThreshold_Reply {
+ uint32 status = 1;
+}
+
+message ConfiguredInferModel_SetSchedulerPriority_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+ uint32 priority = 2;
+}
+
+message ConfiguredInferModel_SetSchedulerPriority_Reply {
+ uint32 status = 1;
+}
+
+message ConfiguredInferModel_GetHwLatencyMeasurement_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+}
+
+message ConfiguredInferModel_GetHwLatencyMeasurement_Reply {
+ uint32 status = 1;
+ uint32 avg_hw_latency = 2;
+}
+
+message ConfiguredInferModel_Activate_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+}
+
+message ConfiguredInferModel_Activate_Reply {
+ uint32 status = 1;
+}
+
+message ConfiguredInferModel_Deactivate_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+}
+
+message ConfiguredInferModel_Deactivate_Reply {
+ uint32 status = 1;
+}
+
+message ConfiguredInferModel_Shutdown_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+}
+
+message ConfiguredInferModel_Shutdown_Reply {
+ uint32 status = 1;
+}
+
+message ConfiguredInferModel_AsyncInfer_Request {
+ HailoObjectHandle configured_infer_model_handle = 1;
+ HailoObjectHandle infer_model_handle = 2;
+ HailoCallbackHandle callback_handle = 3;
+ // Protocol note: After this messgae, server expects to get the input buffers, one after the other, in order
+}
+
+message ConfiguredInferModel_AsyncInfer_Reply {
+ uint32 status = 1;
+}
+
+message CallbackCalled_Reply {
+ uint32 status = 1;
+ HailoCallbackHandle callback_handle = 2;
+ // Protocol note: After this messgae, and only if status is HAILO_SUCCESS, server expects to get the output buffers, one after the other, in order
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file serializer.cpp
+ * @brief HRPC Serialization implementation
+ **/
+
+#include "serializer.hpp"
+#include "hailo/hailort.h"
+#include "hailo/hailort_common.hpp"
+#include "hailo/hailort_defaults.hpp"
+#include "common/utils.hpp"
+
+// https://github.com/protocolbuffers/protobuf/tree/master/cmake#notes-on-compiler-warnings
+#if defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4244 4267 4127)
+#else
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wconversion"
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+#include "rpc.pb.h"
+#if defined(_MSC_VER)
+#pragma warning(pop)
+#else
+#pragma GCC diagnostic pop
+#endif
+
+namespace hailort
+{
+
+Expected<Buffer> CreateVDeviceSerializer::serialize_request(const hailo_vdevice_params_t ¶ms)
+{
+ VDevice_Create_Request request;
+
+ auto proto_params = request.mutable_params();
+ proto_params->set_scheduling_algorithm(params.scheduling_algorithm);
+ proto_params->set_group_id(params.group_id == nullptr ? "" : std::string(params.group_id));
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'CreateVDevice'");
+
+ return serialized_request;
+}
+
+Expected<hailo_vdevice_params_t> CreateVDeviceSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ VDevice_Create_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'CreateVDevice'");
+
+ bool multi_process_service_flag = false;
+ hailo_vdevice_params_t res = {
+ 1,
+ nullptr,
+ static_cast<hailo_scheduling_algorithm_e>(request.params().scheduling_algorithm()),
+ request.params().group_id().c_str(),
+ multi_process_service_flag
+ };
+
+ return res;
+}
+
+Expected<Buffer> CreateVDeviceSerializer::serialize_reply(hailo_status status, rpc_object_handle_t vdevice_handle)
+{
+ VDevice_Create_Reply reply;
+
+ reply.set_status(status);
+ auto proto_vdevice_handle = reply.mutable_vdevice_handle();
+ proto_vdevice_handle->set_id(vdevice_handle);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'CreateVDevice'");
+
+ return serialized_reply;
+}
+
+Expected<std::tuple<hailo_status, rpc_object_handle_t>> CreateVDeviceSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ VDevice_Create_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'CreateVDevice'");
+
+ return std::make_tuple(static_cast<hailo_status>(reply.status()), reply.vdevice_handle().id());
+}
+
+Expected<Buffer> DestroyVDeviceSerializer::serialize_request(rpc_object_handle_t vdevice_handle)
+{
+ VDevice_Destroy_Request request;
+
+ auto proto_vdevice_handle= request.mutable_vdevice_handle();
+ proto_vdevice_handle->set_id(vdevice_handle);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'DestroyVDevice'");
+
+ return serialized_request;
+}
+
+Expected<rpc_object_handle_t> DestroyVDeviceSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ VDevice_Destroy_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyVDevice'");
+
+ return request.vdevice_handle().id();
+}
+
+Expected<Buffer> DestroyVDeviceSerializer::serialize_reply(hailo_status status)
+{
+ VDevice_Destroy_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'DestroyVDevice'");
+
+ return serialized_reply;
+}
+
+hailo_status DestroyVDeviceSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ VDevice_Destroy_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyVDevice'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> CreateInferModelSerializer::serialize_request(rpc_object_handle_t vdevice_handle, uint64_t hef_size)
+{
+ VDevice_CreateInferModel_Request request;
+
+ auto proto_vdevice_handle = request.mutable_vdevice_handle();
+ proto_vdevice_handle->set_id(vdevice_handle);
+
+ request.set_hef_size(hef_size);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'CreateVInferModel'");
+
+ return serialized_request;
+}
+
+Expected<std::tuple<rpc_object_handle_t, uint64_t>> CreateInferModelSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ VDevice_CreateInferModel_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'CreateVInferModel'");
+
+ return std::make_tuple(request.vdevice_handle().id(), request.hef_size());
+}
+
+Expected<Buffer> CreateInferModelSerializer::serialize_reply(hailo_status status, rpc_object_handle_t infer_model_handle)
+{
+ VDevice_CreateInferModel_Reply reply;
+
+ reply.set_status(status);
+ auto proto_infer_model_handle = reply.mutable_infer_model_handle();
+ proto_infer_model_handle->set_id(infer_model_handle);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'CreateVInferModel'");
+
+ return serialized_reply;
+}
+
+Expected<std::tuple<hailo_status, rpc_object_handle_t>> CreateInferModelSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ VDevice_CreateInferModel_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'CreateVInferModel'");
+
+ return std::make_tuple(static_cast<hailo_status>(reply.status()), reply.infer_model_handle().id());
+}
+
+Expected<Buffer> DestroyInferModelSerializer::serialize_request(rpc_object_handle_t infer_model_handle)
+{
+ InferModel_Destroy_Request request;
+
+ auto proto_infer_model_handle = request.mutable_infer_model_handle();
+ proto_infer_model_handle->set_id(infer_model_handle);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'DestroyInferModel'");
+
+ return serialized_request;
+}
+
+Expected<rpc_object_handle_t> DestroyInferModelSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ InferModel_Destroy_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyInferModel'");
+
+ return request.infer_model_handle().id();
+}
+
+Expected<Buffer> DestroyInferModelSerializer::serialize_reply(hailo_status status)
+{
+ InferModel_Destroy_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'DestroyInferModel'");
+
+ return serialized_reply;
+}
+
+hailo_status DestroyInferModelSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ InferModel_Destroy_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyInferModel'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> CreateConfiguredInferModelSerializer::serialize_request(rpc_create_configured_infer_model_request_params_t params)
+{
+ InferModel_CreateConfiguredInferModel_Request request;
+
+ auto proto_infer_model_handle = request.mutable_infer_model_handle();
+ proto_infer_model_handle->set_id(params.infer_model_handle);
+
+ auto proto_vdevide_handle = request.mutable_vdevice_handle();
+ proto_vdevide_handle->set_id(params.vdevice_handle);
+
+ for (auto &input_stream_params : params.input_streams_params) {
+ auto proto_input_stream = request.add_input_infer_streams();
+ proto_input_stream->set_name(input_stream_params.first);
+ proto_input_stream->set_format_order(input_stream_params.second.format_order);
+ proto_input_stream->set_format_type(input_stream_params.second.format_type);
+ proto_input_stream->set_nms_score_threshold(input_stream_params.second.nms_score_threshold);
+ proto_input_stream->set_nms_iou_threshold(input_stream_params.second.nms_iou_threshold);
+ proto_input_stream->set_nms_max_proposals_per_class(input_stream_params.second.nms_max_proposals_per_class);
+ proto_input_stream->set_nms_max_accumulated_mask_size(input_stream_params.second.nms_max_accumulated_mask_size);
+ }
+
+ for (auto &output_stream_params : params.output_streams_params) {
+ auto proto_output_stream = request.add_output_infer_streams();
+ proto_output_stream->set_name(output_stream_params.first);
+ proto_output_stream->set_format_order(output_stream_params.second.format_order);
+ proto_output_stream->set_format_type(output_stream_params.second.format_type);
+ proto_output_stream->set_nms_score_threshold(output_stream_params.second.nms_score_threshold);
+ proto_output_stream->set_nms_iou_threshold(output_stream_params.second.nms_iou_threshold);
+ proto_output_stream->set_nms_max_proposals_per_class(output_stream_params.second.nms_max_proposals_per_class);
+ proto_output_stream->set_nms_max_accumulated_mask_size(output_stream_params.second.nms_max_accumulated_mask_size);
+ }
+
+ request.set_batch_size(static_cast<uint32_t>(params.batch_size));
+ request.set_power_mode(static_cast<uint32_t>(params.power_mode));
+ request.set_latency_flag(static_cast<uint32_t>(params.latency_flag));
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'CreateConfiguredInferModel'");
+
+ return serialized_request;
+}
+
+Expected<rpc_create_configured_infer_model_request_params_t> CreateConfiguredInferModelSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ rpc_create_configured_infer_model_request_params_t request_params;
+ InferModel_CreateConfiguredInferModel_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'CreateConfiguredInferModel'");
+
+ request_params.infer_model_handle = request.infer_model_handle().id();
+ request_params.vdevice_handle = request.vdevice_handle().id();
+
+ for (auto input_stream: request.input_infer_streams()) {
+ rpc_stream_params_t current_stream_params;
+ current_stream_params.format_order = input_stream.format_order();
+ current_stream_params.format_type = input_stream.format_type();
+ current_stream_params.nms_score_threshold = input_stream.nms_score_threshold();
+ current_stream_params.nms_iou_threshold = input_stream.nms_iou_threshold();
+ current_stream_params.nms_max_proposals_per_class = input_stream.nms_max_proposals_per_class();
+ current_stream_params.nms_max_accumulated_mask_size = input_stream.nms_max_accumulated_mask_size();
+ request_params.input_streams_params.emplace(input_stream.name(), current_stream_params);
+ }
+
+ for (auto output_stream: request.output_infer_streams()) {
+ rpc_stream_params_t current_stream_params;
+ current_stream_params.format_order = output_stream.format_order();
+ current_stream_params.format_type = output_stream.format_type();
+ current_stream_params.nms_score_threshold = output_stream.nms_score_threshold();
+ current_stream_params.nms_iou_threshold = output_stream.nms_iou_threshold();
+ current_stream_params.nms_max_proposals_per_class = output_stream.nms_max_proposals_per_class();
+ current_stream_params.nms_max_accumulated_mask_size = output_stream.nms_max_accumulated_mask_size();
+ request_params.output_streams_params.emplace(output_stream.name(), current_stream_params);
+ }
+
+ request_params.batch_size = static_cast<uint16_t>(request.batch_size());
+ request_params.power_mode = static_cast<hailo_power_mode_t>(request.power_mode());
+ request_params.latency_flag = static_cast<hailo_latency_measurement_flags_t>(request.latency_flag());
+
+ return request_params;
+}
+
+Expected<Buffer> CreateConfiguredInferModelSerializer::serialize_reply(hailo_status status, rpc_object_handle_t configured_infer_handle,
+ uint32_t async_queue_size)
+{
+ InferModel_CreateConfiguredInferModel_Reply reply;
+
+ reply.set_status(status);
+ auto proto_configured_infer_model_handle = reply.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_handle);
+ reply.set_async_queue_size(async_queue_size);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'CreateConfiguredInferModel'");
+
+ return serialized_reply;
+}
+
+Expected<std::tuple<hailo_status, rpc_object_handle_t, uint32_t>> CreateConfiguredInferModelSerializer::deserialize_reply(
+ const MemoryView &serialized_reply)
+{
+ InferModel_CreateConfiguredInferModel_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'CreateConfiguredInferModel'");
+
+ return std::make_tuple(static_cast<hailo_status>(reply.status()), reply.configured_infer_model_handle().id(), reply.async_queue_size());
+}
+
+Expected<Buffer> DestroyConfiguredInferModelSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle)
+{
+ ConfiguredInferModel_Destroy_Request request;
+
+ auto proto_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_infer_model_handle->set_id(configured_infer_model_handle);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'DestroyConfiguredInferModel'");
+
+ return serialized_request;
+}
+
+Expected<rpc_object_handle_t> DestroyConfiguredInferModelSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_Destroy_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyConfiguredInferModel'");
+
+ return request.configured_infer_model_handle().id();
+}
+
+Expected<Buffer> DestroyConfiguredInferModelSerializer::serialize_reply(hailo_status status)
+{
+ ConfiguredInferModel_Destroy_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'DestroyConfiguredInferModel'");
+
+ return serialized_reply;
+}
+
+hailo_status DestroyConfiguredInferModelSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_Destroy_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'CreateConfiguredInferModel'");
+ CHECK_SUCCESS(static_cast<hailo_status>(reply.status()));
+
+ return HAILO_SUCCESS;
+}
+
+Expected<Buffer> SetSchedulerTimeoutSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle, const std::chrono::milliseconds &timeout)
+{
+ ConfiguredInferModel_SetSchedulerTimeout_Request request;
+
+ auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_model_handle);
+ request.set_timeout(static_cast<uint32_t>(timeout.count()));
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerTimeout'");
+
+ return serialized_request;
+}
+
+Expected<std::tuple<rpc_object_handle_t, std::chrono::milliseconds>> SetSchedulerTimeoutSerializer::deserialize_request(
+ const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_SetSchedulerTimeout_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerTimeout'");
+
+ return std::make_tuple(request.configured_infer_model_handle().id(), std::chrono::milliseconds(request.timeout()));
+}
+
+Expected<Buffer> SetSchedulerTimeoutSerializer::serialize_reply(hailo_status status)
+{
+ ConfiguredInferModel_SetSchedulerTimeout_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerTimeout'");
+
+ return serialized_reply;
+}
+
+hailo_status SetSchedulerTimeoutSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_SetSchedulerTimeout_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerTimeout'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> SetSchedulerThresholdSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle, uint32_t threshold)
+{
+ ConfiguredInferModel_SetSchedulerThreshold_Request request;
+
+ auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_model_handle);
+ request.set_threshold(threshold);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerThreshold'");
+
+ return serialized_request;
+}
+
+Expected<std::tuple<rpc_object_handle_t, uint32_t>> SetSchedulerThresholdSerializer::deserialize_request(
+ const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_SetSchedulerThreshold_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerThreshold'");
+
+ return std::make_tuple(request.configured_infer_model_handle().id(), request.threshold());
+}
+
+Expected<Buffer> SetSchedulerThresholdSerializer::serialize_reply(hailo_status status)
+{
+ ConfiguredInferModel_SetSchedulerThreshold_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerThreshold'");
+
+ return serialized_reply;
+}
+
+hailo_status SetSchedulerThresholdSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_SetSchedulerThreshold_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerThreshold'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> SetSchedulerPrioritySerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle, uint32_t priority)
+{
+ ConfiguredInferModel_SetSchedulerPriority_Request request;
+
+ auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_model_handle);
+ request.set_priority(priority);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerPriority'");
+
+ return serialized_request;
+}
+
+Expected<std::tuple<rpc_object_handle_t, uint32_t>> SetSchedulerPrioritySerializer::deserialize_request(
+ const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_SetSchedulerPriority_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerPriority'");
+
+ return std::make_tuple(request.configured_infer_model_handle().id(), request.priority());
+}
+
+Expected<Buffer> SetSchedulerPrioritySerializer::serialize_reply(hailo_status status)
+{
+ ConfiguredInferModel_SetSchedulerPriority_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerPriority'");
+
+ return serialized_reply;
+}
+
+hailo_status SetSchedulerPrioritySerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_SetSchedulerPriority_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerPriority'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> GetHwLatencyMeasurementSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle)
+{
+ ConfiguredInferModel_GetHwLatencyMeasurement_Request request;
+
+ auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_model_handle);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'GetHwLatencyMeasurement'");
+
+ return serialized_request;
+}
+
+Expected<rpc_object_handle_t> GetHwLatencyMeasurementSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_GetHwLatencyMeasurement_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'GetHwLatencyMeasurement'");
+
+ return request.configured_infer_model_handle().id();
+}
+
+Expected<Buffer> GetHwLatencyMeasurementSerializer::serialize_reply(hailo_status status, uint32_t avg_hw_latency)
+{
+ ConfiguredInferModel_GetHwLatencyMeasurement_Reply reply;
+ reply.set_status(status);
+ reply.set_avg_hw_latency(avg_hw_latency);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'GetHwLatencyMeasurement'");
+
+ return serialized_reply;
+}
+
+Expected<std::tuple<hailo_status, std::chrono::nanoseconds>> GetHwLatencyMeasurementSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_GetHwLatencyMeasurement_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'GetHwLatencyMeasurement'");
+
+ return std::make_tuple(static_cast<hailo_status>(reply.status()), std::chrono::nanoseconds(reply.avg_hw_latency()));
+}
+
+Expected<Buffer> ActivateSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle)
+{
+ ConfiguredInferModel_Activate_Request request;
+
+ auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_model_handle);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'Activate'");
+
+ return serialized_request;
+}
+
+Expected<rpc_object_handle_t> ActivateSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_Activate_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'Activate'");
+
+ return request.configured_infer_model_handle().id();
+}
+
+Expected<Buffer> ActivateSerializer::serialize_reply(hailo_status status)
+{
+ ConfiguredInferModel_Activate_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'Activate'");
+
+ return serialized_reply;
+}
+
+hailo_status ActivateSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_Activate_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'Activate'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> DeactivateSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle)
+{
+ ConfiguredInferModel_Deactivate_Request request;
+
+ auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_model_handle);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'Deactivate'");
+
+ return serialized_request;
+}
+
+Expected<rpc_object_handle_t> DeactivateSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_Deactivate_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'Deactivate'");
+
+ return request.configured_infer_model_handle().id();
+}
+
+Expected<Buffer> DeactivateSerializer::serialize_reply(hailo_status status)
+{
+ ConfiguredInferModel_Deactivate_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'Deactivate'");
+
+ return serialized_reply;
+}
+
+hailo_status DeactivateSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_Deactivate_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'Deactivate'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> ShutdownSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle)
+{
+ ConfiguredInferModel_Shutdown_Request request;
+
+ auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_model_handle);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'Shutdown'");
+
+ return serialized_request;
+}
+
+Expected<rpc_object_handle_t> ShutdownSerializer::deserialize_request(const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_Shutdown_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'Shutdown'");
+
+ return request.configured_infer_model_handle().id();
+}
+
+Expected<Buffer> ShutdownSerializer::serialize_reply(hailo_status status)
+{
+ ConfiguredInferModel_Shutdown_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'Shutdown'");
+
+ return serialized_reply;
+}
+
+hailo_status ShutdownSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_Shutdown_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'Shutdown'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> RunAsyncSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle, rpc_object_handle_t infer_model_handle,
+ rpc_object_handle_t callback_handle)
+{
+ ConfiguredInferModel_AsyncInfer_Request request;
+
+ auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle();
+ proto_configured_infer_model_handle->set_id(configured_infer_model_handle);
+
+ auto proto_infer_model_handle = request.mutable_infer_model_handle();
+ proto_infer_model_handle->set_id(infer_model_handle);
+
+ auto proto_cb_handle = request.mutable_callback_handle();
+ proto_cb_handle->set_id(callback_handle);
+
+ // TODO (HRT-13983) - check if we can use GetCachedSize
+ TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma()));
+ CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'RunAsync'");
+
+ return serialized_request;
+}
+
+Expected<std::tuple<rpc_object_handle_t, rpc_object_handle_t, rpc_object_handle_t>> RunAsyncSerializer::deserialize_request(
+ const MemoryView &serialized_request)
+{
+ ConfiguredInferModel_AsyncInfer_Request request;
+
+ CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast<int>(serialized_request.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'RunAsync'");
+
+ return std::make_tuple(request.configured_infer_model_handle().id(), request.infer_model_handle().id(),
+ request.callback_handle().id());
+}
+
+Expected<Buffer> RunAsyncSerializer::serialize_reply(hailo_status status)
+{
+ ConfiguredInferModel_AsyncInfer_Reply reply;
+ reply.set_status(status);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'RunAsync'");
+
+ return serialized_reply;
+}
+
+hailo_status RunAsyncSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ ConfiguredInferModel_AsyncInfer_Reply reply;
+
+ CHECK(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'RunAsync'");
+
+ return static_cast<hailo_status>(reply.status());
+}
+
+Expected<Buffer> CallbackCalledSerializer::serialize_reply(hailo_status status, rpc_object_handle_t callback_handle)
+{
+ CallbackCalled_Reply reply;
+
+ reply.set_status(status);
+ auto proto_callback_handle = reply.mutable_callback_handle();
+ proto_callback_handle->set_id(callback_handle);
+
+ TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma()));
+
+ CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to serialize 'CallbackCalled'");
+
+ return serialized_reply;
+}
+
+Expected<std::tuple<hailo_status, rpc_object_handle_t>> CallbackCalledSerializer::deserialize_reply(const MemoryView &serialized_reply)
+{
+ CallbackCalled_Reply reply;
+
+ CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast<int>(serialized_reply.size())),
+ HAILO_RPC_FAILED, "Failed to de-serialize 'CallbackCalled'");
+
+ return std::make_tuple(static_cast<hailo_status>(reply.status()), reply.callback_handle().id());
+}
+
+} /* namespace hailort */
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file serializer.hpp
+ * @brief HRPC protocol serialization
+ **/
+
+#ifndef _HAILO_SERIALIZER_HPP_
+#define _HAILO_SERIALIZER_HPP_
+
+#include "hailo/hailort.h"
+#include "hailo/buffer.hpp"
+#include "hailo/expected.hpp"
+
+#include <chrono>
+#include <unordered_map>
+
+namespace hailort
+{
+
+#define INVALID_HANDLE_ID (UINT32_MAX)
+#define INVALID_LATENCY_MEASUREMENT (UINT32_MAX)
+
+enum class HailoRpcActionID {
+ VDEVICE__CREATE,
+ VDEVICE__DESTROY,
+ VDEVICE__CREATE_INFER_MODEL,
+
+ INFER_MODEL__DESTROY,
+ INFER_MODEL__CREATE_CONFIGURED_INFER_MODEL,
+
+ CONFIGURED_INFER_MODEL__DESTROY,
+ CONFIGURED_INFER_MODEL__SET_SCHEDULER_TIMEOUT,
+ CONFIGURED_INFER_MODEL__SET_SCHEDULER_THRESHOLD,
+ CONFIGURED_INFER_MODEL__SET_SCHEDULER_PRIORITY,
+ CONFIGURED_INFER_MODEL__GET_HW_LATENCY_MEASUREMENT,
+ CONFIGURED_INFER_MODEL__ACTIVATE,
+ CONFIGURED_INFER_MODEL__DEACTIVATE,
+ CONFIGURED_INFER_MODEL__SHUTDOWN,
+ CONFIGURED_INFER_MODEL__RUN_ASYNC,
+
+ CALLBACK_CALLED,
+
+ MAX_VALUE,
+};
+
+using rpc_object_handle_t = uint32_t;
+struct rpc_stream_params_t
+{
+ uint32_t format_order;
+ uint32_t format_type;
+ float32_t nms_score_threshold;
+ float32_t nms_iou_threshold;
+ uint32_t nms_max_proposals_per_class;
+ uint32_t nms_max_accumulated_mask_size;
+};
+using rpc_stream_params_map_t = std::unordered_map<std::string, rpc_stream_params_t>;
+struct rpc_create_configured_infer_model_request_params_t
+{
+ rpc_object_handle_t infer_model_handle;
+ rpc_object_handle_t vdevice_handle;
+ rpc_stream_params_map_t input_streams_params;
+ rpc_stream_params_map_t output_streams_params;
+ uint16_t batch_size;
+ hailo_power_mode_t power_mode;
+ hailo_latency_measurement_flags_t latency_flag;
+};
+
+class CreateVDeviceSerializer
+{
+public:
+ CreateVDeviceSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(const hailo_vdevice_params_t ¶ms);
+ static Expected<hailo_vdevice_params_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status, rpc_object_handle_t vdevice_handle = INVALID_HANDLE_ID);
+ static Expected<std::tuple<hailo_status, rpc_object_handle_t>> deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class DestroyVDeviceSerializer
+{
+public:
+ DestroyVDeviceSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t vdevice_handle);
+ static Expected<rpc_object_handle_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class CreateInferModelSerializer
+{
+public:
+ CreateInferModelSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t vdevice_handle, uint64_t hef_size);
+ static Expected<std::tuple<rpc_object_handle_t, uint64_t>> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status, rpc_object_handle_t infer_model_handle = INVALID_HANDLE_ID);
+ static Expected<std::tuple<hailo_status, rpc_object_handle_t>> deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class DestroyInferModelSerializer
+{
+public:
+ DestroyInferModelSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t infer_model_handle);
+ static Expected<rpc_object_handle_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class CreateConfiguredInferModelSerializer
+{
+public:
+ CreateConfiguredInferModelSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_create_configured_infer_model_request_params_t params);
+ static Expected<rpc_create_configured_infer_model_request_params_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status, rpc_object_handle_t configured_infer_handle = INVALID_HANDLE_ID,
+ uint32_t async_queue_size = 0);
+ static Expected<std::tuple<hailo_status, rpc_object_handle_t, uint32_t>> deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class DestroyConfiguredInferModelSerializer
+{
+public:
+ DestroyConfiguredInferModelSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle);
+ static Expected<rpc_object_handle_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class SetSchedulerTimeoutSerializer
+{
+public:
+ SetSchedulerTimeoutSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle, const std::chrono::milliseconds &timeout);
+ static Expected<std::tuple<rpc_object_handle_t, std::chrono::milliseconds>> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class SetSchedulerThresholdSerializer
+{
+public:
+ SetSchedulerThresholdSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle, uint32_t threshold);
+ static Expected<std::tuple<rpc_object_handle_t, uint32_t>> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class SetSchedulerPrioritySerializer
+{
+public:
+ SetSchedulerPrioritySerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle, uint32_t priority);
+ static Expected<std::tuple<rpc_object_handle_t, uint32_t>> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class GetHwLatencyMeasurementSerializer
+{
+public:
+ GetHwLatencyMeasurementSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle);
+ static Expected<rpc_object_handle_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status, uint32_t avg_hw_latency = INVALID_LATENCY_MEASUREMENT);
+ static Expected<std::tuple<hailo_status, std::chrono::nanoseconds>> deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class ActivateSerializer
+{
+public:
+ ActivateSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle);
+ static Expected<rpc_object_handle_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class DeactivateSerializer
+{
+public:
+ DeactivateSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle);
+ static Expected<rpc_object_handle_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class ShutdownSerializer
+{
+public:
+ ShutdownSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle);
+ static Expected<rpc_object_handle_t> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class RunAsyncSerializer
+{
+public:
+ RunAsyncSerializer() = delete;
+
+ static Expected<Buffer> serialize_request(rpc_object_handle_t configured_infer_model_handle, rpc_object_handle_t infer_model_handle,
+ rpc_object_handle_t callback_handle);
+ static Expected<std::tuple<rpc_object_handle_t, rpc_object_handle_t, rpc_object_handle_t>> deserialize_request(const MemoryView &serialized_request);
+
+ static Expected<Buffer> serialize_reply(hailo_status status);
+ static hailo_status deserialize_reply(const MemoryView &serialized_reply);
+};
+
+class CallbackCalledSerializer
+{
+public:
+ CallbackCalledSerializer() = delete;
+
+ static Expected<Buffer> serialize_reply(hailo_status status, rpc_object_handle_t callback_handle = INVALID_HANDLE_ID);
+ static Expected<std::tuple<hailo_status, rpc_object_handle_t>> deserialize_reply(const MemoryView &serialized_reply);
+};
+
+
+} /* namespace hailort */
+
+#endif /* _HAILO_SERIALIZER_HPP_ */
# set(CMAKE_C_CLANG_TIDY "clang-tidy;-checks=*")
set(HAILORT_MAJOR_VERSION 4)
-set(HAILORT_MINOR_VERSION 17)
-set(HAILORT_REVISION_VERSION 1)
+set(HAILORT_MINOR_VERSION 18)
+set(HAILORT_REVISION_VERSION 0)
# Add the cmake folder so the modules there are found
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
# Generate hef-proto files using host protobuf
protobuf_generate_cpp(PROTO_HEF_SRC PROTO_HEF_HEADER hef.proto)
protobuf_generate_python(PROTO_HEF_PY hef.proto) # TODO (HRT-12504): Copy hef_pb2.py to tools directory
+protobuf_generate_python(PROTO_HEF_PY tracer_profiler.proto)
add_library(hef_proto ${PROTO_HEF_SRC} ${PROTO_HEF_HEADER} ${PROTO_HEF_PY})
target_link_libraries(hef_proto libprotobuf-lite)
add_subdirectory(tests)
endif()
add_subdirectory(bindings)
-add_subdirectory(doc)
+if(HAILO_BUILD_DOC)
+ add_subdirectory(doc)
+endif()
message(FATAL_ERROR "Only unix hosts are supported, stopping build")
endif()
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
# GST_PLUGIN_DEFINE needs PACKAGE to be defined
set(GST_HAILO_PACKAGE_NAME "hailo")
gst-hailo/sync_gst_hailosend.cpp
gst-hailo/sync_gst_hailorecv.cpp
gst-hailo/gsthailonet.cpp
+ gst-hailo/gsthailo_allocator.cpp
+ gst-hailo/gsthailo_dmabuf_allocator.cpp
gst-hailo/gsthailodevicestats.cpp
gst-hailo/common.cpp
gst-hailo/network_group_handle.cpp
using namespace hailort;
+#define ERROR(msg, ...) g_print("HailoNet Error: " msg, ##__VA_ARGS__)
#define PLUGIN_AUTHOR "Hailo Technologies Ltd. (\"Hailo\")"
+#define MAX_STRING_SIZE (PATH_MAX)
+
#define MAX_QUEUED_BUFFERS_IN_INPUT (16)
#define MAX_QUEUED_BUFFERS_IN_OUTPUT (16)
#define MAX_QUEUED_BUFFERS_IN_CORE (16)
} while(0)
#define CHECK_EXPECTED(obj, ...) _CHECK_EXPECTED(obj, "" __VA_ARGS__)
+#define __HAILO_CONCAT(x, y) x ## y
+#define _HAILO_CONCAT(x, y) __HAILO_CONCAT(x, y)
+
+#define _TRY(expected_var_name, var_decl, expr, ...) \
+ auto expected_var_name = (expr); \
+ CHECK_EXPECTED(expected_var_name, __VA_ARGS__); \
+ var_decl = expected_var_name.release()
+
+/**
+ * The TRY macro is used to allow easier validation and access for variables returned as Expected<T>.
+ * If the expression returns an Expected<T> with status HAILO_SUCCESS, the macro will release the expected and assign
+ * the var_decl.
+ * Otherwise, the macro will cause current function to return the failed status.
+ *
+ * Usage example:
+ *
+ * Expected<int> func() {
+ * TRY(auto var, return_5());
+ * // Now var is int with value 5
+ *
+ * // func will return Unexpected with status HAILO_INTERNAL_FAILURE
+ * TRY(auto var2, return_error(HAILO_INTERNAL_FAILURE), "Failed doing stuff {}", 5);
+ */
+#define TRY(var_decl, expr, ...) _TRY(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__)
+
#define RGB_FEATURES_SIZE (3)
#define RGBA_FEATURES_SIZE (4)
#define GRAY8_FEATURES_SIZE (1)
return *this;
}
- const T &get()
+ const T &get() const
{
return m_value;
}
bool m_was_changed;
};
+class HailoElemStringProperty final
+{
+public:
+ HailoElemStringProperty(const std::string &default_val) : m_was_changed(false) {
+ memset(m_string, 0, sizeof(m_string));
+ strncpy(m_string, default_val.c_str(), sizeof(m_string) - 1);
+ }
+
+ ~HailoElemStringProperty() {}
+
+ HailoElemStringProperty &operator=(const std::string &value)
+ {
+ m_was_changed = true;
+ strncpy(m_string, value.c_str(), sizeof(m_string) - 1);
+ return *this;
+ }
+
+ const std::string get() const
+ {
+ return m_string;
+ }
+
+ bool was_changed()
+ {
+ return m_was_changed;
+ }
+
+private:
+ char m_string[MAX_STRING_SIZE];
+ bool m_was_changed;
+};
+
template<>
HailoElemProperty<gchar*>::~HailoElemProperty();
--- /dev/null
+/*
+ * Copyright (c) 2021-2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#include "gsthailo_allocator.hpp"
+
+
+G_DEFINE_TYPE (GstHailoAllocator, gst_hailo_allocator, GST_TYPE_ALLOCATOR);
+
+
+static GstMemory *gst_hailo_allocator_alloc(GstAllocator* allocator, gsize size, GstAllocationParams* /*params*/) {
+ GstHailoAllocator *hailo_allocator = GST_HAILO_ALLOCATOR(allocator);
+ auto buffer = Buffer::create(size, BufferStorageParams::create_dma());
+ if (!buffer) {
+ ERROR("Creating buffer for allocator has failed, status = %d\n", buffer.status());
+ return nullptr;
+ }
+
+ GstMemory *memory = gst_memory_new_wrapped(static_cast<GstMemoryFlags>(0), buffer->data(),
+ buffer->size(), 0, buffer->size(), nullptr, nullptr);
+ if (nullptr == memory) {
+ ERROR("Creating new GstMemory for allocator has failed!\n");
+ return nullptr;
+ }
+
+ hailo_allocator->buffers[memory] = std::move(buffer.release());
+ return memory;
+}
+
+static void gst_hailo_allocator_free(GstAllocator* allocator, GstMemory *mem) {
+ GstHailoAllocator *hailo_allocator = GST_HAILO_ALLOCATOR(allocator);
+ hailo_allocator->buffers.erase(mem);
+}
+
+static void gst_hailo_allocator_class_init(GstHailoAllocatorClass* klass) {
+ GstAllocatorClass* allocator_class = GST_ALLOCATOR_CLASS(klass);
+
+ allocator_class->alloc = gst_hailo_allocator_alloc;
+ allocator_class->free = gst_hailo_allocator_free;
+}
+
+static void gst_hailo_allocator_init(GstHailoAllocator* allocator) {
+ allocator->buffers = std::unordered_map<GstMemory*, Buffer>();
+}
--- /dev/null
+/*
+ * Copyright (c) 2021-2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#ifndef _GST_HAILO_ALLOCATOR_HPP_
+#define _GST_HAILO_ALLOCATOR_HPP_
+
+#include "common.hpp"
+
+using namespace hailort;
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_HAILO_ALLOCATOR (gst_hailo_allocator_get_type())
+#define GST_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocator))
+#define GST_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocatorClass))
+#define GST_IS_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_HAILO_ALLOCATOR))
+#define GST_IS_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_HAILO_ALLOCATOR))
+
+struct GstHailoAllocator
+{
+ GstAllocator parent;
+ std::unordered_map<GstMemory*, Buffer> buffers;
+};
+
+struct GstHailoAllocatorClass
+{
+ GstAllocatorClass parent;
+};
+
+GType gst_hailo_allocator_get_type(void);
+
+G_END_DECLS
+
+#endif /* _GST_HAILO_ALLOCATOR_HPP_ */
\ No newline at end of file
--- /dev/null
+/*
+ * Copyright (c) 2021-2023 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include "gsthailo_dmabuf_allocator.hpp"
+
+#include <sys/ioctl.h>
+#include <fcntl.h>
+
+// TODO: HRT-13107
+#define DEVPATH "/dev/dma_heap/linux,cma"
+
+G_DEFINE_TYPE (GstHailoDmabufAllocator, gst_hailo_dmabuf_allocator, GST_TYPE_DMABUF_ALLOCATOR);
+
+
+bool GstHailoDmaHeapControl::dma_heap_fd_open = false;
+int GstHailoDmaHeapControl::dma_heap_fd = -1;
+
+
+static GstMemory *gst_hailo_dmabuf_allocator_alloc(GstAllocator* allocator, gsize size, GstAllocationParams* /*params*/) {
+ GstHailoDmabufAllocator *hailo_allocator = GST_HAILO_DMABUF_ALLOCATOR(allocator);
+
+ if (!GstHailoDmaHeapControl::dma_heap_fd_open) {
+ GstHailoDmaHeapControl::dma_heap_fd = open(DEVPATH, O_RDWR | O_CLOEXEC);
+ if (GstHailoDmaHeapControl::dma_heap_fd < 0) {
+ ERROR("open fd failed!\n");
+ return nullptr;
+ }
+ GstHailoDmaHeapControl::dma_heap_fd_open = true;
+ }
+
+ dma_heap_allocation_data heap_data;
+ heap_data = {
+ .len = size,
+ .fd = 0,
+ .fd_flags = O_RDWR | O_CLOEXEC,
+ .heap_flags = 0,
+ };
+
+ int ret = ioctl(GstHailoDmaHeapControl::dma_heap_fd, DMA_HEAP_IOCTL_ALLOC, &heap_data);
+ if (ret < 0) {
+ ERROR("ioctl DMA_HEAP_IOCTL_ALLOC failed! ret = %d\n", ret);
+ return nullptr;
+ }
+
+ if (GST_IS_DMABUF_ALLOCATOR(hailo_allocator) == false) {
+ ERROR("hailo_allocator is not dmabuf!\n");
+ return nullptr;
+ }
+
+ GstMemory *memory = gst_dmabuf_allocator_alloc(allocator, heap_data.fd, size);
+ if (nullptr == memory) {
+ ERROR("Creating new GstMemory for allocator has failed!\n");
+ return nullptr;
+ }
+
+ hailo_allocator->dma_buffers[memory] = heap_data;
+ return memory;
+}
+
+static void gst_hailo_dmabuf_allocator_free(GstAllocator* allocator, GstMemory *mem) {
+ GstHailoDmabufAllocator *hailo_allocator = GST_HAILO_DMABUF_ALLOCATOR(allocator);
+ close(hailo_allocator->dma_buffers[mem].fd);
+ hailo_allocator->dma_buffers.erase(mem);
+}
+
+static void gst_hailo_dmabuf_allocator_class_init(GstHailoDmabufAllocatorClass* klass) {
+ GstAllocatorClass* allocator_class = GST_ALLOCATOR_CLASS(klass);
+ allocator_class->alloc = gst_hailo_dmabuf_allocator_alloc;
+ allocator_class->free = gst_hailo_dmabuf_allocator_free;
+}
+
+static void gst_hailo_dmabuf_allocator_init(GstHailoDmabufAllocator* allocator) {
+ allocator->dma_buffers = std::unordered_map<GstMemory*, dma_heap_allocation_data>();
+}
--- /dev/null
+/*
+ * Copyright (c) 2021-2023 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#ifndef _GST_HAILO_DMABUF_ALLOCATOR_HPP_
+#define _GST_HAILO_DMABUF_ALLOCATOR_HPP_
+
+#include "common.hpp"
+#include "hailo/hailort_dma-heap.h"
+
+#include <gst/allocators/gstdmabuf.h>
+
+using namespace hailort;
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_HAILO_DMABUF_ALLOCATOR (gst_hailo_dmabuf_allocator_get_type())
+#define GST_HAILO_DMABUF_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_HAILO_DMABUF_ALLOCATOR, GstHailoDmabufAllocator))
+#define GST_HAILO_DMABUF_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_HAILO_DMABUF_ALLOCATOR, GstHailoDmabufAllocator))
+#define GST_IS_HAILO_DMABUF_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_HAILO_DMABUF_ALLOCATOR))
+#define GST_IS_HAILO_DMABUF_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_HAILO_DMABUF_ALLOCATOR))
+
+#define GST_HAILO_USE_DMA_BUFFER_ENV_VAR "GST_HAILO_USE_DMA_BUFFER"
+
+class GstHailoDmaHeapControl {
+public:
+ static bool dma_heap_fd_open;
+ static int dma_heap_fd;
+};
+
+struct GstHailoDmabufAllocator
+{
+ GstDmaBufAllocator parent;
+ std::unordered_map<GstMemory*, dma_heap_allocation_data> dma_buffers;
+};
+
+struct GstHailoDmabufAllocatorClass
+{
+ GstDmaBufAllocatorClass parent;
+};
+
+GType gst_hailo_dmabuf_allocator_get_type(void);
+
+G_END_DECLS
+
+#endif /* _GST_HAILO_DMABUF_ALLOCATOR_HPP_ */
\ No newline at end of file
#include "hailo/hailort_common.hpp"
#include "hailo/hailort_defaults.hpp"
-#include <gst/allocators/gstdmabuf.h>
#include <algorithm>
#include <unordered_map>
#define WAIT_FOR_ASYNC_READY_TIMEOUT (std::chrono::milliseconds(10000))
-#define ERROR(msg, ...) g_print(msg, ##__VA_ARGS__)
enum
{
static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY);
static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY);
-G_DEFINE_TYPE (GstHailoAllocator, gst_hailo_allocator, GST_TYPE_ALLOCATOR);
G_DEFINE_TYPE (GstHailoNet, gst_hailonet, GST_TYPE_ELEMENT);
static std::atomic_uint32_t hailonet_count(0);
return (nullptr != env) && (0 == g_strcmp0(env, "1"));
}
-static GstMemory *gst_hailo_allocator_alloc(GstAllocator* allocator, gsize size, GstAllocationParams* /*params*/) {
- GstHailoAllocator *hailo_allocator = GST_HAILO_ALLOCATOR(allocator);
- auto buffer = Buffer::create(size, BufferStorageParams::create_dma());
- if (!buffer) {
- ERROR("Creating buffer for allocator has failed, status = %d\n", buffer.status());
- return nullptr;
- }
-
- GstMemory *memory = gst_memory_new_wrapped(static_cast<GstMemoryFlags>(0), buffer->data(),
- buffer->size(), 0, buffer->size(), nullptr, nullptr);
- if (nullptr == memory) {
- ERROR("Creating new GstMemory for allocator has failed!\n");
- return nullptr;
- }
-
- hailo_allocator->buffers[memory] = std::move(buffer.release());
- return memory;
-}
-
-static void gst_hailo_allocator_free(GstAllocator* allocator, GstMemory *mem) {
- GstHailoAllocator *hailo_allocator = GST_HAILO_ALLOCATOR(allocator);
- hailo_allocator->buffers.erase(mem);
-}
-
-static void gst_hailo_allocator_class_init(GstHailoAllocatorClass* klass) {
- GstAllocatorClass* allocator_class = GST_ALLOCATOR_CLASS(klass);
-
- allocator_class->alloc = gst_hailo_allocator_alloc;
- allocator_class->free = gst_hailo_allocator_free;
-}
-
-static void gst_hailo_allocator_init(GstHailoAllocator* allocator) {
- allocator->buffers = std::unordered_map<GstMemory*, Buffer>();
-}
-
static hailo_status gst_hailonet_deconfigure(GstHailoNet *self)
{
// This will wakeup any blocking calls to deuque
return HAILO_SUCCESS;
}
+static void gst_hailonet_unref_input_caps(GstHailoNet *self)
+{
+ if (nullptr != self->input_caps) {
+ gst_caps_unref(self->input_caps);
+ self->input_caps = nullptr;
+ }
+}
+
static hailo_status gst_hailonet_free(GstHailoNet *self)
{
std::unique_lock<std::mutex> lock(self->infer_mutex);
gst_queue_array_free(self->thread_queue);
}
- if (nullptr != self->input_caps) {
- gst_caps_unref(self->input_caps);
+ while(!self->curr_event_queue.empty()) {
+ auto event = self->curr_event_queue.front();
+ gst_event_unref(event);
+ self->curr_event_queue.pop();
+ }
+
+ for (auto &buffer_events_queue_pair : self->events_queue_per_buffer) {
+ while(!buffer_events_queue_pair.second.empty()) {
+ auto event = buffer_events_queue_pair.second.front();
+ gst_event_unref(event);
+ buffer_events_queue_pair.second.pop();
+ }
+ }
+ self->events_queue_per_buffer.clear();
+
+ {
+ std::unique_lock<std::mutex> lock(self->input_caps_mutex);
+ gst_hailonet_unref_input_caps(self);
}
for (auto &name_pool_pair : self->output_buffer_pools) {
CHECK(result, HAILO_INTERNAL_FAILURE, "Could not release buffer pool");
gst_object_unref(name_pool_pair.second);
}
+ self->output_buffer_pools.clear();
+
if (gst_hailo_should_use_dma_buffers()) {
- gst_object_unref(self->dma_allocator);
- } else {
+ if (GstHailoDmaHeapControl::dma_heap_fd_open) {
+ close(GstHailoDmaHeapControl::dma_heap_fd);
+ GstHailoDmaHeapControl::dma_heap_fd_open = false;
+ }
+
+ if (nullptr != self->dmabuf_allocator) {
+ gst_object_unref(self->dmabuf_allocator);
+ }
+ } else if (nullptr != self->allocator) {
gst_object_unref(self->allocator);
}
- self->props.free_strings();
-
return HAILO_SUCCESS;
}
{
if (self->props.m_input_format_type.was_changed()) {
for (const auto &input_name : infer_model->get_input_names()) {
- auto input = infer_model->input(input_name);
- CHECK_EXPECTED_AS_STATUS(input);
-
- input->set_format_type(self->props.m_input_format_type.get());
+ TRY(auto input, infer_model->input(input_name));
+ input.set_format_type(self->props.m_input_format_type.get());
}
}
if (self->props.m_output_format_type.was_changed()) {
for (const auto &output_name : infer_model->get_output_names()) {
- auto output = infer_model->output(output_name);
- CHECK_EXPECTED_AS_STATUS(output);
+ TRY(auto output, infer_model->output(output_name));
- output->set_format_type(self->props.m_output_format_type.get());
+ output.set_format_type(self->props.m_output_format_type.get());
}
}
});
for (const auto &output_name : infer_model->get_output_names()) {
- auto output = infer_model->output(output_name);
- CHECK_EXPECTED_AS_STATUS(output);
+ TRY(auto output, infer_model->output(output_name));
if (self->props.m_nms_score_threshold.was_changed()) {
CHECK(has_nms_output, HAILO_INVALID_OPERATION, "NMS score threshold is set, but there is no NMS output in this model.");
- if (output->is_nms()) {
- output->set_nms_score_threshold(self->props.m_nms_score_threshold.get());
+ if (output.is_nms()) {
+ output.set_nms_score_threshold(self->props.m_nms_score_threshold.get());
}
}
if (self->props.m_nms_iou_threshold.was_changed()) {
CHECK(has_nms_output, HAILO_INVALID_OPERATION, "NMS IoU threshold is set, but there is no NMS output in this model.");
- if (output->is_nms()) {
- output->set_nms_iou_threshold(self->props.m_nms_iou_threshold.get());
+ if (output.is_nms()) {
+ output.set_nms_iou_threshold(self->props.m_nms_iou_threshold.get());
}
}
if (self->props.m_nms_max_proposals_per_class.was_changed()) {
CHECK(has_nms_output, HAILO_INVALID_OPERATION, "NMS max proposals per class is set, but there is no NMS output in this model.");
- if (output->is_nms()) {
- output->set_nms_max_proposals_per_class(self->props.m_nms_max_proposals_per_class.get());
+ if (output.is_nms()) {
+ output.set_nms_max_proposals_per_class(self->props.m_nms_max_proposals_per_class.get());
}
}
}
self->props.m_outputs_max_pool_size.get());
if (gst_hailo_should_use_dma_buffers()) {
- gst_buffer_pool_config_set_allocator(config, self->dma_allocator, nullptr);
+ gst_buffer_pool_config_set_allocator(config, GST_ALLOCATOR(self->dmabuf_allocator), nullptr);
} else {
gst_buffer_pool_config_set_allocator(config, GST_ALLOCATOR(self->allocator), nullptr);
}
return pool;
}
+static void gst_hailonet_push_event_to_queue(GstHailoNet *self, GstEvent *event)
+{
+ std::unique_lock<std::mutex> lock(self->input_queue_mutex);
+ self->curr_event_queue.push(event);
+}
+
+static gboolean gst_hailonet_handle_queued_event(GstHailoNet *self, GstEvent *event)
+{
+ switch (GST_EVENT_TYPE(event)) {
+ case GST_EVENT_CAPS:
+ {
+ GstCaps *caps;
+ gst_event_parse_caps(event, &caps);
+ auto result = gst_pad_set_caps(self->srcpad, caps);
+ gst_event_unref(event);
+ return result;
+ }
+ default:
+ return gst_pad_push_event(self->srcpad, event);
+ }
+}
+
+static void gst_hailonet_handle_buffer_events(GstHailoNet *self, GstBuffer *buffer)
+{
+ if (self->events_queue_per_buffer.find(buffer) == self->events_queue_per_buffer.end()) {
+ // The buffer does not have any events to send
+ return;
+ }
+
+ while (!self->events_queue_per_buffer.at(buffer).empty()) {
+ GstEvent* event = self->events_queue_per_buffer.at(buffer).front();
+ (void)gst_hailonet_handle_queued_event(self, event);
+ self->events_queue_per_buffer.at(buffer).pop();
+ }
+ self->events_queue_per_buffer.erase(buffer);
+}
+
static hailo_status gst_hailonet_configure(GstHailoNet *self)
{
if (self->is_configured) {
for (const auto &input_name : self->infer_model->get_input_names()) {
if(self->props.m_no_transform.get()) {
// In case transformation is disabled - format order will be the same as we get from the HW (stream info).
- auto input_stream_infos = self->infer_model->hef().get_stream_info_by_name(input_name, HAILO_H2D_STREAM);
- CHECK_EXPECTED_AS_STATUS(input_stream_infos);
- self->infer_model->input(input_name)->set_format_order(input_stream_infos.value().format.order);
+ TRY(const auto input_stream_infos, self->infer_model->hef().get_stream_info_by_name(input_name, HAILO_H2D_STREAM));
+ self->infer_model->input(input_name)->set_format_order(input_stream_infos.format.order);
} else if (self->infer_model->input(input_name)->format().order == HAILO_FORMAT_ORDER_NHWC) {
self->infer_model->input(input_name)->set_format_order(HAILO_FORMAT_ORDER_RGB4);
}
if (self->props.m_no_transform.get()) {
for (const auto &output_name : self->infer_model->get_output_names()) {
// In case transformation is disabled - format order will be the same as we get from the HW (stream info).
- auto output_stream_infos = self->infer_model->hef().get_stream_info_by_name(output_name, HAILO_D2H_STREAM);
- CHECK_EXPECTED_AS_STATUS(output_stream_infos);
- self->infer_model->output(output_name)->set_format_order(output_stream_infos.value().format.order);
+ TRY(const auto output_stream_infos, self->infer_model->hef().get_stream_info_by_name(output_name, HAILO_D2H_STREAM));
+ self->infer_model->output(output_name)->set_format_order(output_stream_infos.format.order);
}
}
- auto configured_infer_model = self->infer_model->configure();
- CHECK_EXPECTED_AS_STATUS(configured_infer_model);
+ TRY(auto configured_infer_model, self->infer_model->configure());
- auto ptr = make_shared_nothrow<ConfiguredInferModel>(configured_infer_model.release());
+ auto ptr = make_shared_nothrow<ConfiguredInferModel>(std::move(configured_infer_model));
CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY);
self->configured_infer_model = ptr;
return HAILO_SUCCESS;
}
+static void gst_hailonet_init_allocator(GstHailoNet *self)
+{
+ gchar *parent_name = gst_object_get_name(GST_OBJECT(self));
+ gchar *name = g_strconcat(parent_name, ":hailo_allocator", NULL);
+ g_free(parent_name);
+
+ if (gst_hailo_should_use_dma_buffers()) {
+ self->dmabuf_allocator = GST_HAILO_DMABUF_ALLOCATOR(g_object_new(GST_TYPE_HAILO_DMABUF_ALLOCATOR, "name", name, NULL));
+ gst_object_ref_sink(self->dmabuf_allocator);
+ } else {
+ self->allocator = GST_HAILO_ALLOCATOR(g_object_new(GST_TYPE_HAILO_ALLOCATOR, "name", name, NULL));
+ gst_object_ref_sink(self->allocator);
+ }
+
+ g_free(name);
+}
+
static hailo_status gst_hailonet_allocate_infer_resources(GstHailoNet *self)
{
- auto bindings = self->configured_infer_model->create_bindings();
- CHECK_EXPECTED_AS_STATUS(bindings);
- self->infer_bindings = std::move(bindings.release());
+ TRY(self->infer_bindings, self->configured_infer_model->create_bindings());
self->output_buffer_pools = std::unordered_map<std::string, GstBufferPool*>();
self->output_vstream_infos = std::unordered_map<std::string, hailo_vstream_info_t>();
- auto async_queue_size = self->configured_infer_model->get_async_queue_size();
- CHECK_EXPECTED_AS_STATUS(async_queue_size);
- self->input_queue = gst_queue_array_new(static_cast<guint>(async_queue_size.value()));
- self->thread_queue = gst_queue_array_new(static_cast<guint>(async_queue_size.value()));
+ TRY(const auto async_queue_size, self->configured_infer_model->get_async_queue_size());
+ self->input_queue = gst_queue_array_new(static_cast<guint>(async_queue_size));
+ self->thread_queue = gst_queue_array_new(static_cast<guint>(async_queue_size));
self->is_thread_running = true;
self->thread = std::thread([self] () {
while (self->is_thread_running) {
self->thread_cv.notify_all();
if (GST_IS_PAD(self->srcpad)) { // Checking because we fail here when exiting the application
GstFlowReturn ret = gst_pad_push(self->srcpad, buffer);
- if ((GST_FLOW_OK != ret) && (GST_FLOW_FLUSHING != ret) && (!self->has_got_eos)) {
+ if ((GST_FLOW_OK != ret) && (GST_FLOW_FLUSHING != ret) && ((GST_FLOW_EOS != ret)) && (!self->has_got_eos)) {
ERROR("gst_pad_push failed with status = %d\n", ret);
break;
}
}
});
+ gst_hailonet_init_allocator(self);
for (auto &output : self->infer_model->outputs()) {
- auto buffer_pool = gst_hailonet_create_buffer_pool(self, output.get_frame_size());
- CHECK_EXPECTED_AS_STATUS(buffer_pool);
-
- self->output_buffer_pools[output.name()] = buffer_pool.release();
+ TRY(self->output_buffer_pools[output.name()], gst_hailonet_create_buffer_pool(self, output.get_frame_size()));
}
- auto vstream_infos = self->infer_model->hef().get_output_vstream_infos();
- CHECK_EXPECTED_AS_STATUS(vstream_infos);
-
- for (const auto &vstream_info : vstream_infos.value()) {
+ TRY(const auto vstream_infos, self->infer_model->hef().get_output_vstream_infos());
+ for (const auto &vstream_info : vstream_infos) {
self->output_vstream_infos[vstream_info.name] = vstream_info;
}
return HAILO_SUCCESS;
}
+static GstPadProbeReturn gst_hailonet_sink_probe(GstPad */*pad*/, GstPadProbeInfo */*info*/, gpointer user_data)
+{
+ GstHailoNet *self = static_cast<GstHailoNet*>(user_data);
+ std::unique_lock<std::mutex> lock(self->sink_probe_change_state_mutex);
+
+ if (self->did_critical_failure_happen) {
+ return GST_PAD_PROBE_REMOVE;
+ }
+
+ auto status = gst_hailonet_configure(self);
+ if (HAILO_SUCCESS != status) {
+ return GST_PAD_PROBE_REMOVE;
+ }
+
+ status = gst_hailonet_allocate_infer_resources(self);
+ if (HAILO_SUCCESS != status) {
+ return GST_PAD_PROBE_REMOVE;
+ }
+
+ if (HAILO_SCHEDULING_ALGORITHM_NONE != self->props.m_scheduling_algorithm.get()) {
+ self->props.m_is_active = true;
+ return GST_PAD_PROBE_REMOVE;
+ }
+
+ if ((1 == hailonet_count) && (!self->props.m_is_active.was_changed())) {
+ self->props.m_is_active = true;
+ }
+
+ if (self->props.m_is_active.get()) {
+ status = self->configured_infer_model->activate();
+ if (HAILO_SUCCESS != status) {
+ return GST_PAD_PROBE_REMOVE;
+ }
+ }
+
+ self->has_called_activate = true;
+ return GST_PAD_PROBE_REMOVE;
+}
+
static GstStateChangeReturn gst_hailonet_change_state(GstElement *element, GstStateChange transition)
{
GstStateChangeReturn ret = GST_ELEMENT_CLASS(gst_hailonet_parent_class)->change_state(element, transition);
if (HAILO_SUCCESS != status) {
return GST_STATE_CHANGE_FAILURE;
}
+
+ gst_pad_add_probe(self->sinkpad, GST_PAD_PROBE_TYPE_BUFFER, static_cast<GstPadProbeCallback>(gst_hailonet_sink_probe), self, nullptr);
break;
}
default:
g_warning("The network was already configured so changing the HEF path will not take place!");
break;
}
- if (nullptr != self->props.m_hef_path.get()) {
- g_free(self->props.m_hef_path.get());
- }
- self->props.m_hef_path = g_strdup(g_value_get_string(value));
+ self->props.m_hef_path = g_value_get_string(value);
break;
case PROP_BATCH_SIZE:
if (self->is_configured) {
g_warning("The network was already configured so changing the device ID will not take place!");
break;
}
- if (nullptr != self->props.m_device_id.get()) {
- g_free(self->props.m_device_id.get());
- }
- self->props.m_device_id = g_strdup(g_value_get_string(value));
+ self->props.m_device_id = g_value_get_string(value);
break;
case PROP_DEVICE_COUNT:
- if (nullptr != self->props.m_device_id.get()) {
+ if (!self->props.m_device_id.get().empty()) {
g_error("device-id and device-count excludes eachother. received device-id=%s, device-count=%d",
- self->props.m_device_id.get(), g_value_get_uint(value));
+ self->props.m_device_id.get().c_str(), g_value_get_uint(value));
break;
}
if (self->is_configured) {
g_warning("The network was already configured so changing the vdevice group ID will not take place!");
break;
}
- if (nullptr != self->props.m_vdevice_group_id.get()) {
- g_free(self->props.m_vdevice_group_id.get());
- }
- self->props.m_vdevice_group_id = g_strdup(g_value_get_string(value));
+ self->props.m_vdevice_group_id = g_value_get_string(value);
break;
case PROP_IS_ACTIVE:
(void)gst_hailonet_toggle_activation(self, self->props.m_is_active.get(), g_value_get_boolean(value));
GstHailoNet *self = GST_HAILONET(object);
switch (property_id) {
case PROP_HEF_PATH:
- g_value_set_string(value, self->props.m_hef_path.get());
+ g_value_set_string(value, self->props.m_hef_path.get().c_str());
break;
case PROP_BATCH_SIZE:
g_value_set_uint(value, self->props.m_batch_size.get());
break;
case PROP_DEVICE_ID:
- g_value_set_string(value, self->props.m_device_id.get());
+ g_value_set_string(value, self->props.m_device_id.get().c_str());
break;
case PROP_DEVICE_COUNT:
g_value_set_uint(value, self->props.m_device_count.get());
break;
case PROP_VDEVICE_GROUP_ID:
- g_value_set_string(value, self->props.m_vdevice_group_id.get());
+ g_value_set_string(value, self->props.m_vdevice_group_id.get().c_str());
break;
case PROP_IS_ACTIVE:
g_value_set_boolean(value, self->props.m_is_active.get());
static hailo_status gst_hailonet_fill_multiple_input_bindings_dma_buffers(GstHailoNet *self, GstBuffer *buffer)
{
- auto input_buffers = gst_hailonet_read_input_dma_buffers_from_meta(self, buffer);
- CHECK_EXPECTED_AS_STATUS(input_buffers);
- for (const auto &name : self->infer_model->get_input_names())
- {
- auto status = self->infer_bindings.input(name)->set_dma_buffer(input_buffers.value().at(name));
+ TRY(auto input_buffers, gst_hailonet_read_input_dma_buffers_from_meta(self, buffer));
+ for (const auto &name : self->infer_model->get_input_names()) {
+ auto status = self->infer_bindings.input(name)->set_dma_buffer(input_buffers.at(name));
CHECK_SUCCESS(status);
}
static hailo_status gst_hailonet_fill_multiple_input_bindings(GstHailoNet *self, GstBuffer *buffer)
{
- auto input_buffers = gst_hailonet_read_input_buffers_from_meta(self, buffer);
- CHECK_EXPECTED_AS_STATUS(input_buffers);
+ TRY(auto input_buffers, gst_hailonet_read_input_buffers_from_meta(self, buffer));
for (const auto &name : self->infer_model->get_input_names()) {
- auto status = self->infer_bindings.input(name)->set_buffer(MemoryView(input_buffers.value().at(name),
+ auto status = self->infer_bindings.input(name)->set_buffer(MemoryView(input_buffers.at(name),
self->infer_model->input(name)->get_frame_size()));
CHECK_SUCCESS(status);
}
return HAILO_SUCCESS;
}
+static void store_buffer_events(GstHailoNet *self, GstBuffer *buffer)
+{
+ self->events_queue_per_buffer[buffer] = std::queue<GstEvent*>();
+ while (!self->curr_event_queue.empty()) {
+ GstEvent *event = self->curr_event_queue.front();
+ self->events_queue_per_buffer[buffer].push(event);
+ self->curr_event_queue.pop();
+ }
+}
+
static hailo_status gst_hailonet_push_buffer_to_input_queue(GstHailoNet *self, GstBuffer *buffer)
{
std::unique_lock<std::mutex> lock(self->input_queue_mutex);
+ store_buffer_events(self, buffer);
gst_queue_array_push_tail(self->input_queue, buffer);
return HAILO_SUCCESS;
self->ongoing_frames++;
}
- auto job = self->configured_infer_model->run_async(self->infer_bindings, [self, tensors] (const AsyncInferCompletionInfo &/*completion_info*/) {
+ TRY(auto job, self->configured_infer_model->run_async(self->infer_bindings, [self, tensors] (const AsyncInferCompletionInfo &/*completion_info*/) {
GstBuffer *buffer = nullptr;
{
std::unique_lock<std::mutex> lock(self->input_queue_mutex);
buffer = static_cast<GstBuffer*>(gst_queue_array_pop_head(self->input_queue));
+ gst_hailonet_handle_buffer_events(self, buffer);
}
for (auto &output : self->infer_model->outputs()) {
self->flush_cv.notify_all();
gst_hailonet_push_buffer_to_thread(self, buffer);
- });
- CHECK_EXPECTED_AS_STATUS(job);
- job->detach();
+ }));
+ job.detach();
return HAILO_SUCCESS;
}
if (HAILO_STREAM_ABORT == tensors.status()) {
return HAILO_SUCCESS;
}
- CHECK_EXPECTED_AS_STATUS(tensors);
+ CHECK_EXPECTED_AS_STATUS(tensors); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
status = gst_hailonet_call_run_async(self, tensors.value());
CHECK_SUCCESS(status);
if (HAILO_STREAM_ABORT == tensors.status()) {
return HAILO_SUCCESS;
}
- CHECK_EXPECTED_AS_STATUS(tensors);
+ CHECK_EXPECTED_AS_STATUS(tensors); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
status = gst_hailonet_call_run_async(self, tensors.value());
CHECK_SUCCESS(status);
GstHailoNet *self = GST_HAILONET(parent);
std::unique_lock<std::mutex> lock(self->infer_mutex);
+ if (self->did_critical_failure_happen) {
+ return GST_FLOW_ERROR;
+ }
+
if (self->props.m_pass_through.get() || !self->props.m_is_active.get() || !self->is_configured) {
gst_hailonet_push_buffer_to_thread(self, buffer);
return GST_FLOW_OK;
if (self->props.m_should_force_writable.get()) {
buffer = gst_buffer_make_writable(buffer);
if (nullptr == buffer) {
- ERROR("Failed to make buffer writable!");
+ ERROR("Failed to make buffer writable!\n");
return GST_FLOW_ERROR;
}
} else {
- ERROR("Input buffer is not writable! Use force-writable property to force the buffer to be writable");
+ ERROR("Input buffer is not writable! Use force-writable property to force the buffer to be writable\n");
return GST_FLOW_ERROR;
}
}
hailo_device_id_t device_id = {0};
if (self->props.m_device_id.was_changed()) {
- auto expected_device_id = HailoRTCommon::to_device_id(self->props.m_device_id.get());
- CHECK_EXPECTED_AS_STATUS(expected_device_id);
- device_id = std::move(expected_device_id.release());
-
+ TRY(device_id, HailoRTCommon::to_device_id(self->props.m_device_id.get()));
vdevice_params.device_ids = &device_id;
}
if (self->props.m_device_count.was_changed()) {
vdevice_params.device_count = self->props.m_device_count.get();
}
if (self->props.m_vdevice_group_id.was_changed()) {
- vdevice_params.group_id = self->props.m_vdevice_group_id.get();
+ vdevice_params.group_id = self->props.m_vdevice_group_id.get().c_str();
} else if (self->props.m_vdevice_key.was_changed()) {
auto key_str = std::to_string(self->props.m_vdevice_key.get());
vdevice_params.group_id = key_str.c_str();
"To use multi-process-service please set scheduling-algorithm to a value other than 'none'");
}
- auto vdevice = VDevice::create(vdevice_params);
- CHECK_EXPECTED_AS_STATUS(vdevice);
- self->vdevice = std::move(vdevice.release());
-
- auto infer_model = self->vdevice->create_infer_model(self->props.m_hef_path.get());
- CHECK_EXPECTED_AS_STATUS(infer_model);
- self->infer_model = infer_model.release();
+ TRY(self->vdevice, VDevice::create(vdevice_params));
+ TRY(self->infer_model, self->vdevice->create_infer_model(self->props.m_hef_path.get()));
if(!(self->props.m_input_from_meta.get())){
CHECK(self->infer_model->inputs().size() == 1, HAILO_INVALID_OPERATION,
return nullptr;
}
+ if (nullptr != self->input_caps) {
+ return gst_caps_copy(self->input_caps);
+ }
+
if (nullptr == self->vdevice) {
auto status = gst_hailonet_init_infer_model(self);
if (HAILO_SUCCESS != status) {
}
}
- // TODO (HRT-12491): check caps based on incoming metadata
if (self->props.m_input_from_meta.get()) {
GstCaps *new_caps = gst_caps_new_any();
+ std::unique_lock<std::mutex> lock(self->input_caps_mutex);
+ gst_hailonet_unref_input_caps(self);
self->input_caps = new_caps;
return gst_caps_copy(new_caps);
}
return nullptr;
}
+ std::unique_lock<std::mutex> lock(self->input_caps_mutex);
+ gst_hailonet_unref_input_caps(self);
self->input_caps = new_caps;
return gst_caps_copy(new_caps);
}
}
}
-static gboolean gst_hailonet_handle_caps_event(GstHailoNet *self, GstCaps */*caps*/)
-{
- if (nullptr == self->input_caps) {
- return FALSE;
- }
-
- GstCaps *caps_result = gst_pad_peer_query_caps(self->srcpad, self->input_caps);
- if (gst_caps_is_empty(caps_result)) {
- return FALSE;
- }
-
- if (gst_caps_is_any(caps_result)) {
- gst_caps_unref(caps_result);
- return TRUE;
- }
-
- GstCaps *outcaps = gst_caps_fixate(caps_result);
- gboolean res = gst_pad_set_caps(self->srcpad, outcaps);
- gst_caps_unref(outcaps);
- return res;
-}
-
static gboolean gst_hailonet_sink_event(GstPad *pad, GstObject *parent, GstEvent *event)
{
GstHailoNet *self = GST_HAILONET(parent);
- switch (GST_EVENT_TYPE(event)) {
- case GST_EVENT_CAPS:
- {
- GstCaps *caps;
- gst_event_parse_caps(event, &caps);
- auto result = gst_hailonet_handle_caps_event(self, caps);
- gst_event_unref(event);
- return result;
- }
- case GST_EVENT_EOS:
+ if (GST_EVENT_TYPE(event) == GST_EVENT_EOS) {
self->has_got_eos = true;
return gst_pad_push_event(self->srcpad, event);
- default:
- return gst_pad_event_default(pad, parent, event);
- }
-}
-
-static GstPadProbeReturn gst_hailonet_sink_probe(GstPad */*pad*/, GstPadProbeInfo */*info*/, gpointer user_data)
-{
- GstHailoNet *self = static_cast<GstHailoNet*>(user_data);
- std::unique_lock<std::mutex> lock(self->sink_probe_change_state_mutex);
-
- auto status = gst_hailonet_configure(self);
- if (HAILO_SUCCESS != status) {
- return GST_PAD_PROBE_DROP;
- }
-
- status = gst_hailonet_allocate_infer_resources(self);
- if (HAILO_SUCCESS != status) {
- return GST_PAD_PROBE_DROP;
- }
-
- if (HAILO_SCHEDULING_ALGORITHM_NONE != self->props.m_scheduling_algorithm.get()) {
- self->props.m_is_active = true;
- return GST_PAD_PROBE_REMOVE;
- }
-
- if ((1 == hailonet_count) && (!self->props.m_is_active.was_changed())) {
- self->props.m_is_active = true;
}
-
- if (self->props.m_is_active.get()) {
- status = self->configured_infer_model->activate();
- if (HAILO_SUCCESS != status) {
- return GST_PAD_PROBE_DROP;
- }
+ if (GST_EVENT_IS_STICKY(event)) {
+ gst_hailonet_push_event_to_queue(self, event);
+ return TRUE;
+ } else {
+ return gst_pad_event_default(pad, parent, event);
}
-
- self->has_called_activate = true;
- return GST_PAD_PROBE_REMOVE;
}
static void gst_hailonet_flush_callback(GstHailoNet *self, gpointer /*data*/)
self->has_called_activate = false;
self->ongoing_frames = 0;
self->did_critical_failure_happen = false;
-
- gchar *parent_name = gst_object_get_name(GST_OBJECT(self));
- gchar *name = g_strconcat(parent_name, ":hailo_allocator", NULL);
- g_free(parent_name);
-
- if (gst_hailo_should_use_dma_buffers()) {
- self->dma_allocator = gst_dmabuf_allocator_new();
- } else {
- self->allocator = GST_HAILO_ALLOCATOR(g_object_new(GST_TYPE_HAILO_ALLOCATOR, "name", name, NULL));
- gst_object_ref_sink(self->allocator);
- g_free(name);
- }
+ self->events_queue_per_buffer = std::unordered_map<GstBuffer*, std::queue<GstEvent*>>();
+ self->curr_event_queue = std::queue<GstEvent*>();
g_signal_connect(self, "flush", G_CALLBACK(gst_hailonet_flush_callback), nullptr);
#include "hailo/infer_model.hpp"
#include "common.hpp"
+#include "gsthailo_allocator.hpp"
+#include "gsthailo_dmabuf_allocator.hpp"
#include <queue>
#include <condition_variable>
G_BEGIN_DECLS
-#define GST_TYPE_HAILO_ALLOCATOR (gst_hailo_allocator_get_type())
-#define GST_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocator))
-#define GST_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocatorClass))
-#define GST_IS_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_HAILO_ALLOCATOR))
-#define GST_IS_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_HAILO_ALLOCATOR))
-
#define MIN_OUTPUTS_POOL_SIZE (MAX_GSTREAMER_BATCH_SIZE)
#define MAX_OUTPUTS_POOL_SIZE (MAX_GSTREAMER_BATCH_SIZE * 4)
-#define GST_HAILO_USE_DMA_BUFFER_ENV_VAR "GST_HAILO_USE_DMA_BUFFER"
-
-struct GstHailoAllocator
-{
- GstAllocator parent;
- std::unordered_map<GstMemory*, Buffer> buffers;
-};
-
-struct GstHailoAllocatorClass
-{
- GstAllocatorClass parent;
-};
-
-GType gst_hailo_allocator_get_type(void);
-
struct HailoNetProperties final
{
public:
- HailoNetProperties() : m_hef_path(nullptr), m_batch_size(HAILO_DEFAULT_BATCH_SIZE),
- m_device_id(nullptr), m_device_count(0), m_vdevice_group_id(nullptr), m_is_active(false), m_pass_through(false),
+ HailoNetProperties() : m_hef_path(""), m_batch_size(HAILO_DEFAULT_BATCH_SIZE),
+ m_device_id(""), m_device_count(0), m_vdevice_group_id(""), m_is_active(false), m_pass_through(false),
m_outputs_min_pool_size(MIN_OUTPUTS_POOL_SIZE), m_outputs_max_pool_size(MAX_OUTPUTS_POOL_SIZE),
m_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN), m_scheduler_timeout_ms(HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS),
m_scheduler_threshold(HAILO_DEFAULT_SCHEDULER_THRESHOLD), m_scheduler_priority(HAILO_SCHEDULER_PRIORITY_NORMAL),
m_vdevice_key(DEFAULT_VDEVICE_KEY)
{}
- void free_strings()
- {
- if (m_hef_path.was_changed()) {
- g_free(m_hef_path.get());
- }
- if (m_device_id.was_changed()) {
- g_free(m_device_id.get());
- }
- if (m_vdevice_group_id.was_changed()) {
- g_free(m_vdevice_group_id.get());
- }
- }
-
- HailoElemProperty<gchar*> m_hef_path;
+ HailoElemStringProperty m_hef_path;
HailoElemProperty<guint16> m_batch_size;
- HailoElemProperty<gchar*> m_device_id;
+ HailoElemStringProperty m_device_id;
HailoElemProperty<guint16> m_device_count;
- HailoElemProperty<gchar*> m_vdevice_group_id;
+ HailoElemStringProperty m_vdevice_group_id;
HailoElemProperty<gboolean> m_is_active;
HailoElemProperty<gboolean> m_pass_through;
HailoElemProperty<guint> m_outputs_min_pool_size;
};
typedef struct _GstHailoNet {
- GstElement element;
- GstPad *sinkpad;
- GstPad *srcpad;
- GstQueueArray *input_queue;
- GstQueueArray *thread_queue;
- std::atomic_uint32_t buffers_in_thread_queue;
- std::thread thread;
- HailoNetProperties props;
- GstCaps *input_caps;
- std::atomic_bool is_thread_running;
- std::atomic_bool has_got_eos;
- std::mutex sink_probe_change_state_mutex;
- bool did_critical_failure_happen;
-
- std::unique_ptr<VDevice> vdevice;
- std::shared_ptr<InferModel> infer_model;
- std::shared_ptr<ConfiguredInferModel> configured_infer_model;
- ConfiguredInferModel::Bindings infer_bindings;
- bool is_configured;
- std::mutex infer_mutex;
-
- bool has_called_activate;
- std::atomic_uint32_t ongoing_frames;
- std::condition_variable flush_cv;
- std::mutex flush_mutex;
-
- GstVideoInfo input_frame_info;
-
- GstHailoAllocator *allocator;
- GstAllocator *dma_allocator;
- std::unordered_map<std::string, GstBufferPool*> output_buffer_pools;
- std::unordered_map<std::string, hailo_vstream_info_t> output_vstream_infos;
-
- std::mutex input_queue_mutex;
- std::mutex thread_queue_mutex;
- std::condition_variable thread_cv;
+ GstElement element;
+ GstPad *sinkpad;
+ GstPad *srcpad;
+
+ std::unordered_map<GstBuffer*, std::queue<GstEvent*>> events_queue_per_buffer;
+ std::queue<GstEvent*> curr_event_queue;
+ GstQueueArray *input_queue;
+
+ GstQueueArray *thread_queue;
+ std::atomic_uint32_t buffers_in_thread_queue;
+ std::thread thread;
+ HailoNetProperties props;
+ GstCaps *input_caps;
+ std::atomic_bool is_thread_running;
+ std::atomic_bool has_got_eos;
+ std::mutex sink_probe_change_state_mutex;
+ bool did_critical_failure_happen;
+
+ std::unique_ptr<VDevice> vdevice;
+ std::shared_ptr<InferModel> infer_model;
+ std::shared_ptr<ConfiguredInferModel> configured_infer_model;
+ ConfiguredInferModel::Bindings infer_bindings;
+ bool is_configured;
+ std::mutex infer_mutex;
+
+ bool has_called_activate;
+ std::atomic_uint32_t ongoing_frames;
+ std::condition_variable flush_cv;
+ std::mutex flush_mutex;
+ std::mutex input_caps_mutex;
+
+ GstVideoInfo input_frame_info;
+
+ GstHailoAllocator *allocator;
+ GstHailoDmabufAllocator *dmabuf_allocator;
+ std::unordered_map<std::string, GstBufferPool*> output_buffer_pools;
+ std::unordered_map<std::string, hailo_vstream_info_t> output_vstream_infos;
+
+ std::mutex input_queue_mutex;
+ std::mutex thread_queue_mutex;
+ std::condition_variable thread_cv;
} GstHailoNet;
typedef struct _GstHailoNetClass {
cmake_minimum_required(VERSION 3.11.0)
-include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/pybind11.cmake)
add_subdirectory(src)
return os.path.join(_ROOT, 'hailo_platform', 'drivers', path)
+import hailo_platform.pyhailort._pyhailort as _pyhailort
from hailo_platform.tools.udp_rate_limiter import UDPRateLimiter
from hailo_platform.pyhailort.hw_object import PcieDevice, EthernetDevice
from hailo_platform.pyhailort.pyhailort import (HEF, ConfigureParams,
InputVStreams, OutputVStreams,
InferVStreams, HailoStreamDirection, HailoFormatFlags, HailoCpuId, Device, VDevice,
DvmTypes, PowerMeasurementTypes, SamplingPeriod, AveragingFactor, MeasurementBufferIndex,
- HailoRTException, HailoSchedulingAlgorithm, HailoRTStreamAbortedByUser)
+ HailoRTException, HailoSchedulingAlgorithm, HailoRTStreamAbortedByUser, AsyncInferJob)
def _verify_pyhailort_lib_exists():
python_version = "".join(str(i) for i in sys.version_info[:2])
_verify_pyhailort_lib_exists()
-def get_version(package_name):
- # See: https://packaging.python.org/guides/single-sourcing-package-version/ (Option 5)
- # We assume that the installed package is actually the same one we import. This assumption may
- # break in some edge cases e.g. if the user modifies sys.path manually.
-
- # hailo_platform package has been renamed to hailort, but the import is still hailo_platform
- if package_name == "hailo_platform":
- package_name = "hailort"
- try:
- import pkg_resources
- return pkg_resources.get_distribution(package_name).version
- except:
- return 'unknown'
+__version__ = "4.18.0"
+if _pyhailort.__version__ != __version__:
+ raise ImportError(
+ f"_pyhailort version ({_pyhailort.__version__}) does not match pyhailort version ({__version__})"
+ )
-__version__ = get_version('hailo_platform')
__all__ = ['EthernetDevice', 'DvmTypes', 'PowerMeasurementTypes',
'SamplingPeriod', 'AveragingFactor', 'MeasurementBufferIndex', 'UDPRateLimiter', 'PcieDevice', 'HEF',
'ConfigureParams', 'FormatType', 'FormatOrder', 'MipiDataTypeRx', 'MipiPixelsPerClock', 'MipiClockSelection',
'MipiIspImageInOrder', 'MipiIspImageOutDataType', 'join_drivers_path', 'IspLightFrequency', 'HailoPowerMode',
'Endianness', 'HailoStreamInterface', 'InputVStreamParams', 'OutputVStreamParams',
'InputVStreams', 'OutputVStreams', 'InferVStreams', 'HailoStreamDirection', 'HailoFormatFlags', 'HailoCpuId',
- 'Device', 'VDevice', 'HailoRTException', 'HailoSchedulingAlgorithm', 'HailoRTStreamAbortedByUser']
+ 'Device', 'VDevice', 'HailoRTException', 'HailoSchedulingAlgorithm', 'HailoRTStreamAbortedByUser', 'AsyncInferJob']
from enum import Enum, IntEnum
import signal
import struct
-import pkg_resources
-# hailo_platform package has been renamed to hailort, but the import is still hailo_platform
-__version__ = pkg_resources.get_distribution("hailort").version
import sys
+from collections import deque
+from dataclasses import dataclass
from argparse import ArgumentTypeError
+from datetime import timedelta
import numpy
import time
-from hailo_platform.common.logger.logger import default_logger
import gc
import os
+from hailo_platform.common.logger.logger import default_logger
import hailo_platform.pyhailort._pyhailort as _pyhailort
-if _pyhailort.__version__ != __version__:
- raise ImportError("_pyhailort version ({}) does not match pyhailort version ({})".format(_pyhailort.__version__, __version__))
from hailo_platform.pyhailort._pyhailort import (TemperatureInfo, # noqa F401
DvmTypes, PowerMeasurementTypes, # noqa F401
MIN_UDP_PADDED_PAYLOAD_SIZE = HailoSocketDefs.MIN_UDP_PADDED_PAYLOAD_SIZE()
MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP = HailoSocketDefs.MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP()
+
class HailoSchedulingAlgorithm(_pyhailort.SchedulingAlgorithm):
pass
else:
raise
- def _raise_indicative_status_exception(self, libhailort_exception):
- error_code = int(libhailort_exception.args[0])
+ @staticmethod
+ def create_exception_from_status(error_code):
string_error_code = get_status_message(error_code)
if string_error_code == "HAILO_ETH_RECV_FAILURE":
- raise UdpRecvError("Failed to receive data") from libhailort_exception
+ return UdpRecvError("Failed to receive data")
if string_error_code == "HAILO_UNSUPPORTED_CONTROL_PROTOCOL_VERSION":
- raise InvalidProtocolVersionException("HailoRT has failed because an invalid protocol version was received from device") from libhailort_exception
+ return InvalidProtocolVersionException("HailoRT has failed because an invalid protocol version was received from device")
if string_error_code == "HAILO_FW_CONTROL_FAILURE":
- raise HailoRTFirmwareControlFailedException("libhailort control operation failed") from libhailort_exception
+ return HailoRTFirmwareControlFailedException("libhailort control operation failed")
if string_error_code == "HAILO_UNSUPPORTED_OPCODE":
- raise HailoRTUnsupportedOpcodeException("HailoRT has failed because an unsupported opcode was sent to device") from libhailort_exception
+ return HailoRTUnsupportedOpcodeException("HailoRT has failed because an unsupported opcode was sent to device")
if string_error_code == "HAILO_INVALID_FRAME":
- raise HailoRTInvalidFrameException("An invalid frame was received") from libhailort_exception
+ return HailoRTInvalidFrameException("An invalid frame was received")
if string_error_code == "HAILO_TIMEOUT":
- raise HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred") from libhailort_exception
+ return HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred")
if string_error_code == "HAILO_STREAM_ABORT":
- raise HailoRTStreamAborted("Stream was aborted") from libhailort_exception
+ return HailoRTStreamAborted("Stream was aborted")
if string_error_code == "HAILO_INVALID_OPERATION":
- raise HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information") from libhailort_exception
+ return HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information")
if string_error_code == "HAILO_INVALID_ARGUMENT":
- raise HailoRTInvalidArgumentException("Invalid argument. See hailort.log for more information") from libhailort_exception
+ return HailoRTInvalidArgumentException("Invalid argument. See hailort.log for more information")
if string_error_code == "HAILO_NOT_FOUND":
- raise HailoRTNotFoundException("Item not found. See hailort.log for more information") from libhailort_exception
+ return HailoRTNotFoundException("Item not found. See hailort.log for more information")
if string_error_code == "HAILO_INVALID_HEF":
- raise HailoRTInvalidHEFException("Invalid HEF. See hailort.log for more information") from libhailort_exception
+ return HailoRTInvalidHEFException("Invalid HEF. See hailort.log for more information")
if string_error_code == "HAILO_ETH_FAILURE":
- raise HailoRTEthException("Ethernet failure. See hailort.log for more information") from libhailort_exception
+ return HailoRTEthException("Ethernet failure. See hailort.log for more information")
if string_error_code == "HAILO_DRIVER_FAIL":
- raise HailoRTPCIeDriverException("PCIe driver failure. run 'dmesg | grep hailo' for more information") from libhailort_exception
+ return HailoRTPCIeDriverException("PCIe driver failure. run 'dmesg | grep hailo' for more information")
if string_error_code == "HAILO_NETWORK_GROUP_NOT_ACTIVATED":
- raise HailoRTNetworkGroupNotActivatedException("Network group is not activated") from libhailort_exception
+ return HailoRTNetworkGroupNotActivatedException("Network group is not activated")
else:
- raise HailoRTException("libhailort failed with error: {} ({})".format(error_code, string_error_code)) from libhailort_exception
+ return HailoRTException("libhailort failed with error: {} ({})".format(error_code, string_error_code))
+
+
+ def _raise_indicative_status_exception(self, libhailort_exception):
+ error_code = int(libhailort_exception.args[0])
+ raise self.create_exception_from_status(error_code) from libhailort_exception
+
def get_status_message(status_code):
status_str = _pyhailort.get_status_message(status_code)
def set_scheduler_timeout(self, timeout_ms, network_name=None):
"""Sets the maximum time period that may pass before receiving run time from the scheduler.
This will occur providing at least one send request has been sent, there is no minimum requirement for send
- requests, (e.g. threshold - see set_scheduler_threshold()).
+ requests, (e.g. threshold - see :func:`ConfiguredNetwork.set_scheduler_threshold`).
Args:
timeout_ms (int): Timeout in milliseconds.
def set_scheduler_threshold(self, threshold):
"""Sets the minimum number of send requests required before the network is considered ready to get run time from the scheduler.
- If at least one send request has been sent, but the threshold is not reached within a set time period (e.g. timeout - see hailo_set_scheduler_timeout()),
+ If at least one send request has been sent, but the threshold is not reached within a set time period (e.g. timeout - see :func:`ConfiguredNetwork.set_scheduler_timeout`),
the scheduler will consider the network ready regardless.
Args:
"""
return self._configured_network.set_scheduler_priority(priority)
+ def init_cache(self, read_offset, write_offset_delta):
+ return self._configured_network.init_cache(read_offset, write_offset_delta)
+
+ def get_cache_info(self):
+ return self._configured_network.get_cache_info()
+
+ def update_cache_offset(self, offset_delta_bytes):
+ return self._configured_network.update_cache_offset(offset_delta_bytes)
+
class EmptyContextManager(object):
"""An empty context manager that returns instead of activated network group when scheduler is enabled`."""
time_before_infer_calcs = time.perf_counter()
if not isinstance(input_data, dict):
- input_stream_infos = self._configured_net_group.get_input_stream_infos()
- if len(input_stream_infos) != 1:
+ input_vstream_infos = self._configured_net_group.get_input_vstream_infos()
+ if len(input_vstream_infos) != 1:
raise Exception("when there is more than one input, the input_data should be of type dict,"
- " mapping between each input_name, and his input_data tensor. number of inputs: {}".format(len(input_stream_infos)))
- input_data = {input_stream_infos[0].name : input_data}
+ " mapping between each input_name, and his input_data tensor. number of inputs: {}".format(len(input_vstream_infos)))
+ input_data = {input_vstream_infos[0].name : input_data}
batch_size = InferVStreams._get_number_of_frames(input_data)
output_buffers, output_buffers_info = self._make_output_buffers_and_infos(input_data, batch_size)
if output_buffers_info[name].output_order == FormatOrder.HAILO_NMS_WITH_BYTE_MASK:
nms_shape = output_buffers_info[name].vstream_info.nms_shape
output_dtype = output_buffers_info[name].output_dtype
- input_stream_infos = self._configured_net_group.get_input_stream_infos()
- if len(input_stream_infos) != 1:
- raise Exception("Output format HAILO_NMS_WITH_BYTE_MASK should have 1 input. Number of inputs: {}".format(len(input_stream_infos)))
- input_height = input_stream_infos[0].shape[0]
- input_width = input_stream_infos[0].shape[1]
+ input_vstream_infos = self._configured_net_group.get_input_vstream_infos()
+ if len(input_vstream_infos) != 1:
+ raise Exception("Output format HAILO_NMS_WITH_BYTE_MASK should have 1 input. Number of inputs: {}".format(len(input_vstream_infos)))
+ input_height = input_vstream_infos[0].shape[0]
+ input_width = input_vstream_infos[0].shape[1]
output_buffers[name] = HailoRTTransformUtils._output_raw_buffer_to_nms_with_byte_mask_format(result_array,
nms_shape.number_of_classes, batch_size, input_height, input_width,
nms_shape.max_bboxes_per_class, output_dtype, self._tf_nms_format)
SUPPORTED_PROTOCOL_VERSION = 2
SUPPORTED_FW_MAJOR = 4
-SUPPORTED_FW_MINOR = 17
-SUPPORTED_FW_REVISION = 1
+SUPPORTED_FW_MINOR = 18
+SUPPORTED_FW_REVISION = 0
MEGA_MULTIPLIER = 1000.0 * 1000.0
HAILO8L = 2
HAILO15H = 3
PLUTO = 4
+ HAILO15M = 5
+ HAILO10H = 6
def __str__(self):
return self.name
if ((device_arch == DeviceArchitectureTypes.HAILO8) or
(device_arch == DeviceArchitectureTypes.HAILO8L)):
return 'hailo8'
- elif device_arch == DeviceArchitectureTypes.HAILO15H:
+ elif ((device_arch == DeviceArchitectureTypes.HAILO15H) or
+ (device_arch == DeviceArchitectureTypes.HAILO15M)):
return 'hailo15'
+ elif (device_arch == DeviceArchitectureTypes.HAILO10H):
+ return 'hailo10'
else:
raise HailoRTException("Unsupported device architecture.")
with ExceptionWrapper():
return self._device.remove_notification_callback(notification_id)
+ def _init_cache_info(self, cache_info):
+ with ExceptionWrapper():
+ return self._device._init_cache_info(cache_info)
+
+ def _get_cache_info(self):
+ with ExceptionWrapper():
+ return self._device._get_cache_info()
+
+ def _update_cache_read_offset(self, read_offset_delta):
+ with ExceptionWrapper():
+ return self._device._update_cache_read_offset(read_offset_delta)
+
def _get_device_handle(self):
return self._device
return self._loaded_network_groups[0]
+class AsyncInferCompletionInfo:
+ """
+ Holds information about the async infer job
+ """
+
+ def __init__(self, exception):
+ """
+ Args:
+ exception (:obj:`HailoRTException`): an exception corresponding to the error that happened inside the async infer job.
+ """
+ self._exception = exception
+
+ @property
+ def exception(self):
+ """
+ Returns the exception that was set on this Infer job. if the job finished succesfully, returns None.
+ """
+ return self._exception
+
+
+class InferModel:
+ """
+ Contains all of the necessary information for configuring the network for inference.
+ This class is used to set up the model for inference and includes methods for setting and getting the model's parameters.
+ By calling the configure function, the user can create a :obj:`ConfiguredInferModel` object, which is used to run inference.
+ """
+ class InferStream:
+ """
+ Represents the parameters of a stream.
+ In default, the stream's parameters are set to the default values of the model.
+ The user can change the stream's parameters by calling the setter functions.
+ """
+ def __init__(self, infer_stream):
+ #"""
+ #Args:
+ # infer_stream (_pyhailort.InferStream): The C++ InferStream object.
+ #"""
+ self._infer_stream = infer_stream
+
+ @property
+ def name(self):
+ """
+ Returns:
+ name (str): the name of the edge.
+ """
+ with ExceptionWrapper():
+ return self._infer_stream.name()
+
+ @property
+ def shape(self):
+ """
+ Returns:
+ shape (list[int]): the shape of the edge.
+ """
+ with ExceptionWrapper():
+ return self._infer_stream.shape()
+
+ @property
+ def format(self):
+ """
+ Returns:
+ format (_pyhailort.hailo_format_t): the format of the edge.
+ """
+ with ExceptionWrapper():
+ return self._infer_stream.format()
+
+ def set_format_type(self, type):
+ """
+ Set the format type of the stream.
+
+ Args:
+ type (_pyhailort.hailo_format_type_t): the format type
+ """
+ with ExceptionWrapper():
+ self._infer_stream.set_format_type(type)
+
+ def set_format_order(self, order):
+ """
+ Set the format order of the stream.
+
+ Args:
+ order (_pyhailort.hailo_format_order_t): the format order
+ """
+ with ExceptionWrapper():
+ self._infer_stream.set_format_order(order)
+
+ @property
+ def quant_infos(self):
+ """
+ Returns:
+ quant_infos (list[_pyhailort.hailo_quant_info_t]): List of the quantization information of the edge.
+ """
+ with ExceptionWrapper():
+ return self._infer_stream.get_quant_infos()
+
+ @property
+ def is_nms(self):
+ """
+ Returns:
+ is_nms (bool): whether the stream is NMS.
+ """
+ with ExceptionWrapper():
+ return self._infer_stream.is_nms()
+
+ def set_nms_score_threshold(self, threshold):
+ """
+ Set NMS score threshold, used for filtering out candidates. Any box with score<TH is suppressed.
+
+ Args:
+ threshold (float): NMS score threshold to set.
+
+ Note:
+ This function is invalid in cases where the edge has no NMS operations on the CPU. It will not fail,
+ but make the :func:`~hailo_platform.pyhailort.pyhailort.pyhailort.InferModel.configure()` function fail.
+ """
+ with ExceptionWrapper():
+ self._infer_stream.set_nms_score_threshold(threshold)
+
+ def set_nms_iou_threshold(self, threshold):
+ """
+ Set NMS intersection over union overlap Threshold,
+ used in the NMS iterative elimination process where potential duplicates of detected items are suppressed.
+
+ Args:
+ threshold (float): NMS IoU threshold to set.
+
+ Note:
+ This function is invalid in cases where the edge has no NMS operations on the CPU. It will not fail,
+ but make the `configure()` function fail.
+ """
+ with ExceptionWrapper():
+ self._infer_stream.set_nms_iou_threshold(threshold)
+
+ def set_nms_max_proposals_per_class(self, max_proposals):
+ """
+ Set a limit for the maximum number of boxes per class.
+
+ Args:
+ max_proposals (int): NMS max proposals per class to set.
+
+ Note:
+ This function is invalid in cases where the edge has no NMS operations on the CPU. It will not fail,
+ but make the `configure()` function fail.
+ """
+ with ExceptionWrapper():
+ self._infer_stream.set_nms_max_proposals_per_class(max_proposals)
+
+ def set_nms_max_accumulated_mask_size(self, max_accumulated_mask_size):
+ """
+ Set maximum accumulated mask size for all the detections in a frame.
+
+ Args:
+ max_accumalated_mask_size (int): NMS max accumulated mask size.
+
+ Note:
+ Used in order to change the output buffer frame size in cases where the
+ output buffer is too small for all the segmentation detections.
+
+ Note:
+ This function is invalid in cases where the edge has no NMS operations on the CPU. It will not fail,
+ but make the `configure()` function fail.
+ """
+ with ExceptionWrapper():
+ self._infer_stream.set_nms_max_accumulated_mask_size(max_accumulated_mask_size)
+
+
+ def __init__(self, infer_model, hef_path):
+ #"""
+ #Args:
+ # infer_model (_pyhailort.InferModel): The internal InferModel object.
+ # hef_path (str): The path to the HEF file.
+ #"""
+ self._infer_model = infer_model
+ self._hef_path = hef_path
+ self._hef = None
+
+ @property
+ def hef(self):
+ """
+ Returns:
+ :class:`HEF`: the HEF object of the model
+ """
+ # TODO: https://hailotech.atlassian.net/browse/HRT-13659
+ if not self._hef:
+ with ExceptionWrapper():
+ self._hef = HEF(self._hef_path)
+
+ return self._hef
+
+ def set_batch_size(self, batch_size):
+ """
+ Sets the batch size of the InferModel. This parameter determines the number of frames to be sent for inference
+ in a single batch. If a scheduler is enabled, this parameter determines the 'burst size': the max number of
+ frames after which the scheduler will attempt to switch to another model.
+
+ Note:
+ Default value is `HAILO_DEFAULT_BATCH_SIZE`. It means automatic batch determined by hailort.
+
+ Args:
+ batch_size (int): The new batch size to be set.
+ """
+ with ExceptionWrapper():
+ self._infer_model.set_batch_size(batch_size)
+
+ def set_power_mode(self, power_mode):
+ """
+ Sets the power mode of the InferModel
+
+ Args:
+ power_mode (_pyhailort.hailo_power_mode_t): The power mode to set.
+ """
+ with ExceptionWrapper():
+ self._infer_model.set_power_mode(power_mode)
+
+ def configure(self):
+ """
+ Configures the InferModel object. Also checks the validity of the configuration's formats.
+
+ Returns:
+ configured_infer_model (:class:`ConfiguredInferModel`): The configured :class:`InferModel` object.
+
+ Raises:
+ :class:`HailoRTException`: In case the configuration is invalid (example: see :func:`InferStream.set_nms_iou_threshold`).
+
+ Note:
+ A :obj:`ConfiguredInferModel` should be used inside a context manager, and should not be passed to a different process.
+ """
+ with ExceptionWrapper():
+ configured_infer_model_cpp_obj = self._infer_model.configure()
+ return ConfiguredInferModel(configured_infer_model_cpp_obj, self)
+
+ @property
+ def input_names(self):
+ """
+ Returns:
+ names (list[str]): The input names of the :class:`InferModel`.
+ """
+ with ExceptionWrapper():
+ return self._infer_model.get_input_names()
+
+ @property
+ def output_names(self):
+ """
+ Returns:
+ names (list[str]): The output names of the :class:`InferModel`.
+ """
+ with ExceptionWrapper():
+ return self._infer_model.get_output_names()
+
+ @property
+ def inputs(self):
+ """
+ Returns:
+ inputs (list[`InferStream`]): List of input :class:`InferModel.InferStream`.
+ """
+ with ExceptionWrapper():
+ return [self.InferStream(infer_stream) for infer_stream in self._infer_model.inputs()]
+
+ @property
+ def outputs(self):
+ """
+ Returns:
+ outputs (list[`InferStream`]): List of output :class:`InferModel.InferStream`.
+ """
+ with ExceptionWrapper():
+ return [self.InferStream(infer_stream) for infer_stream in self._infer_model.outputs()]
+
+ def input(self, name=""):
+ """
+ Gets an input's :class:`InferModel.InferStream`.
+
+ Args:
+ name (str, optional): the name of the input stream. Required in case of multiple inputs.
+
+ Returns:
+ :class:`ConfiguredInferModel.Bindings.InferStream` - the input infer stream of the configured infer model.
+
+ Raises:
+ :class:`HailoRTNotFoundException` in case a non-existing input is requested or no name is given
+ but multiple inputs exist.
+ """
+ with ExceptionWrapper():
+ return self.InferStream(self._infer_model.input(name))
+
+ def output(self, name=""):
+ """
+ Gets an output's :class:`InferModel.InferStream`.
+
+ Args:
+ name (str, optional): the name of the output stream. Required in case of multiple outputs.
+
+ Returns:
+ :obj:`ConfiguredInferModel.Bindings.InferStream` - the output infer stream of the configured infer model.
+
+ Raises:
+ :class:`HailoRTNotFoundException` in case a non-existing output is requested or no name is given
+ but multiple outputs exist.
+ """
+ with ExceptionWrapper():
+ return self.InferStream(self._infer_model.output(name))
+
+
+class ConfiguredInferModel:
+ """
+ Configured :class:`InferModel` that can be used to perform an asynchronous inference.
+
+ Note:
+ Passing an instance of :class:`ConfiguredInferModel` to a different process is not supported and would lead to an undefined behavior.
+ """
+
+ @dataclass
+ class NmsTransformationInfo:
+ """
+ class for NMS transformation info.
+ """
+ format_order: FormatOrder
+ input_height: int
+ input_width: int
+ number_of_classes: int
+ max_bboxes_per_class: int
+ quant_info: _pyhailort.QuantInfo
+ output_dtype: numpy.dtype = numpy.dtype('float32')
+ batch_size: int = 1
+
+ @dataclass
+ class NmsHailoTransformationInfo(NmsTransformationInfo):
+ """
+ class for NMS transformation info when using hailo format
+ """
+ use_tf_nms_format: bool = False
+
+ @dataclass
+ class NmsTfTransformationInfo(NmsTransformationInfo):
+ """
+ class for NMS transformation info when using tf format
+ """
+ use_tf_nms_format: bool = True
+
+ class Bindings:
+ """
+ Represents an asynchronous infer request - holds the input and output buffers of the request.
+ A request represents a single frame.
+ """
+
+ class InferStream:
+ """
+ Holds the input and output buffers of the Bindings infer request
+ """
+ def __init__(self, infer_stream, nms_info=None):
+ #"""
+ #Args:
+ # infer_stream (_pyhailort.InferStream): The internal infer stream object.
+ # nmw_info (class:`ConfiguredInferModel.NmsTransformationInfo`, optional): The NMS transformation info.
+ #"""
+ self._infer_stream = infer_stream
+ self._buffer = None
+ if nms_info:
+ self._nms_info = nms_info
+ self._quantized_empty_bbox = numpy.asarray(
+ [0] * BBOX_PARAMS,
+ dtype=nms_info.output_dtype,
+ )
+
+ HailoRTTransformUtils.dequantize_output_buffer_in_place(
+ self._quantized_empty_bbox,
+ nms_info.output_dtype,
+ BBOX_PARAMS,
+ nms_info.quant_info,
+ )
+ else:
+ self._nms_info = None
+
+ def set_buffer(self, buffer):
+ """
+ Sets the edge's buffer to a new one.
+
+ Args:
+ buffer (numpy.array): The new buffer to set. The array's shape should match the edge's shape.
+ """
+ with ExceptionWrapper():
+ self._infer_stream.set_buffer(buffer)
+
+ self._buffer = buffer
+
+ def get_buffer(self, tf_format=False):
+ """
+ Gets the edge's buffer.
+
+ Args:
+ tf_format (bool, optional): Whether the output format is tf or hailo. Relevant for NMS outputs. The output
+ can be re-formatted into two formats (TF, Hailo) and the user through choosing the True/False function
+ parameter, can decide which format to receive.
+ For detection outputs:
+ TF format is an :obj:`numpy.array` with shape [number of classes, bounding box params, max bounding boxes per class]
+ where the 3rd dimension (bounding box params) is of a fixed length of 5 (y_min, x_min, y_max, x_max, score).
+
+ Hailo format is a list of detections per class: [[class_0 detections], [class_1 detections], ... [class_n-1 detections]]
+ where each detection is an :obj:`numpy.array` with shape (y_min, x_min, y_max, x_max, score).
+
+ For segmentation outputs:
+ TF format is an :obj:`numpy.array` with shape [1, image_size + number_of_params, max bounding boxes per class]
+ where the 3rd dimension (image_size + number_of_params) is calculated as: mask (image_width - image_height) + (y_min, x_min, y_max, x_max, score, class_id).
+ The mask is a binary mask of the segmentation output where the ROI (region of interest) is mapped to 1 and the background is mapped to 0.
+
+ Hailo format is a list of detections per class: [detecion0, detection1, ... detection_m]
+ where each detection is an :obj:`HailoDetection`
+
+ Returns:
+ buffer (numpy.array): the buffer of the edge.
+ """
+ buffer = self._buffer
+
+ if tf_format is None:
+ # the user wants the raw buffer, with no transformation. Useful when the output is not ready, and
+ # NMS transformation might fail.
+ return buffer
+
+ if self._nms_info:
+ nms_info_class = ConfiguredInferModel.NmsTfTransformationInfo if tf_format else ConfiguredInferModel.NmsHailoTransformationInfo
+ nms_info = nms_info_class(**self._nms_info.__dict__)
+
+ if nms_info.format_order == FormatOrder.HAILO_NMS_WITH_BYTE_MASK:
+ buffer = HailoRTTransformUtils._output_raw_buffer_to_nms_with_byte_mask_format(
+ [self._buffer],
+ nms_info.number_of_classes,
+ nms_info.batch_size,
+ nms_info.input_height,
+ nms_info.input_width,
+ nms_info.max_bboxes_per_class,
+ nms_info.output_dtype,
+ nms_info.use_tf_nms_format,
+ )
+ else:
+ if nms_info.use_tf_nms_format:
+ nms_shape = [
+ nms_info.number_of_classes,
+ BBOX_PARAMS,
+ nms_info.max_bboxes_per_class,
+ ]
+
+ shape = [nms_info.batch_size, *nms_shape]
+ flat_result = self._buffer.reshape(-1)
+
+ buffer = HailoRTTransformUtils.output_raw_buffer_to_nms_tf_format(
+ flat_result,
+ shape,
+ nms_info.output_dtype,
+ self._quantized_empty_bbox,
+ )
+ else:
+ buffer = HailoRTTransformUtils.output_raw_buffer_to_nms_format(
+ [self._buffer],
+ nms_info.number_of_classes,
+ )
+
+ if tf_format:
+ buffer = buffer[0]
+
+ return buffer
+
+ def __init__(self, bindings, input_names, output_names, nms_infos):
+ #"""
+ #Args:
+ # bindings (_pyhailort.ConfiguredInferModelBindingsWrapper): The internal bindings object.
+ # input_names (list[str]): The input names of the model.
+ # output_names (list[str]): The output names of the model.
+ # nms_infos (dict[str : class:`ConfiguredInferModel.NmsTransformationInfo`]): The NMS transformation info per output.
+ #"""
+ self._bindings = bindings
+ self._inputs = {}
+ self._outputs = {}
+ self._input_names = input_names
+ self._output_names = output_names
+ self._nms_infos = nms_infos
+
+ def input(self, name=""):
+ """
+ Gets an input's InferStream object.
+
+ Args:
+ name (str, optional): the name of the input stream. Required in case of multiple inputs.
+
+ Returns:
+ :class:`ConfiguredInferModel.Bindings.InferStream` - the input infer stream of the configured infer model.
+
+ Raises:
+ :class:`HailoRTNotFoundException` in case a non-existing input is requested or no name is given
+ but multiple inputs exist.
+ """
+ if name == "" and len(self._input_names) == 1:
+ name = self._input_names[0]
+
+ if name not in self._inputs:
+ with ExceptionWrapper():
+ self._inputs[name] = self.InferStream(self._bindings.input(name))
+
+ return self._inputs[name]
+
+ def output(self, name=""):
+ """
+ Gets an output's InferStream object.
+
+ Args:
+ name (str, optional): the name of the output stream. Required in cae of multiple outputs.
+
+ Returns:
+ :class:`ConfiguredInferModel.Bindings.InferStream` - the output infer stream of the configured infer model.
+
+ Raises:
+ :class:`HailoRTNotFoundException` in case a non-existing output is requested or no name is given
+ but multiple outputs exist.
+ """
+ if name == "" and len(self._output_names) == 1:
+ name = self._output_names[0]
+
+ if name not in self._outputs:
+ with ExceptionWrapper():
+ self._outputs[name] = self.InferStream(self._bindings.output(name), self._nms_infos.get(name, None))
+
+ return self._outputs[name]
+
+ def get(self):
+ """
+ Gets the internal bindings object.
+
+ Returns:
+ _bindings (_pyhailort.ConfiguredInferModelBindingsWrapper): the internal bindings object.
+ """
+ return self._bindings
+
+
+ def __init__(self, configured_infer_model, infer_model):
+ #"""
+ #Args:
+ # configured_infer_model (_pyhailort.ConfiguredInferModelWrapper): The internal configured_infer_model object.
+ # infer_model (:class:`InferModel`): The InferModel object.
+ #"""
+ self._configured_infer_model = configured_infer_model
+ self._input_names = infer_model.input_names
+ self._output_names = infer_model.output_names
+ self._infer_model = infer_model
+ self._buffer_guards = deque()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self._configured_infer_model = None
+
+ def activate(self):
+ """
+ Activates hailo device inner-resources for inference.
+ Calling this function is invalid in case scheduler is enabled.
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ """
+ with ExceptionWrapper():
+ self._configured_infer_model.activate()
+
+ def deactivate(self):
+ """
+ Deactivates hailo device inner-resources for inference.
+ Calling this function is invalid in case scheduler is enabled.
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ """
+ with ExceptionWrapper():
+ self._configured_infer_model.deactivate()
+
+ def create_bindings(self, input_buffers=None, output_buffers=None):
+ """
+ Creates a Bindings object.
+
+ Args:
+ input_buffers (dict[str: numpy.array], optional): The input buffers for the Bindings object. Keys are the input names, and values are their corresponding buffers. See :func:`~hailo_platform.pyhailort.pyhailort.ConfiguredInferModel.Bindings.InferStream.get_buffer` for more information.
+ output_buffers (dict[str: numpy.array], optional): The output buffers for the Bindings object. Keys are the output names, and values are their corresponding buffers. See :func:`~hailo_platform.pyhailort.pyhailort.ConfiguredInferModel.Bindings.InferStream.get_buffer` for more information.
+
+ Returns:
+ :obj:`ConfiguredInferModel.Bindings`: Bindings object
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ """
+ with ExceptionWrapper():
+ bindings_cpp_obj = self._configured_infer_model.create_bindings()
+
+ bindings = self.Bindings(bindings_cpp_obj, self._input_names, self._output_names, self._get_nms_infos())
+
+ if input_buffers:
+ for input_name, buffer in input_buffers.items():
+ bindings.input(input_name).set_buffer(buffer)
+
+ if output_buffers:
+ for output_name, buffer in output_buffers.items():
+ bindings.output(output_name).set_buffer(buffer)
+
+ return bindings
+
+ def wait_for_async_ready(self, timeout_ms=1000, frames_count=1):
+ """
+ Waits until the model is ready to launch a new asynchronous inference operation.
+ The readiness of the model is determined by the ability to push buffers to the asynchronous inference pipeline.
+
+ args:
+ timeout_ms (int, optional): Amount of time to wait until the model is ready in milliseconds.
+ frames_count (int, optional): The count of buffers you intent to infer in the next request. Useful for batch inference. Default is 1
+
+ Raises:
+ :class:`HailoRTTimeout` in case the model is not ready in the given timeout.
+ :class:`HailoRTException` in case of an error.
+ """
+ with ExceptionWrapper():
+ self._configured_infer_model.wait_for_async_ready(timedelta(milliseconds=timeout_ms), frames_count)
+
+ def run(self, bindings, timeout):
+ """
+ Launches a synchronous inference operation with the provided bindings.
+
+ Args:
+ list of bindings (:obj:`ConfiguredInferModel.Bindings`): The bindings for the inputs and outputs of the model.
+ A list with a single binding is valid. Multiple bindings are useful for batch inference.
+ timeout (int): The timeout in milliseconds.
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ :class:`HailoRTTimeout` in case the job did not finish in the given timeout.
+ """
+ with ExceptionWrapper():
+ job = self.run_async(bindings)
+ job.wait(timeout)
+
+ def run_async(self, bindings, callback=None):
+ """
+ Launches an asynchronous inference operation with the provided bindings.
+
+ Args:
+ list of bindings (:obj:`ConfiguredInferModel.Bindings`): The bindings for the inputs and outputs of the model.
+ A list with a single binding is valid. Multiple bindings are useful for batch inference.
+ callback (Callable, optional): A callback that will be called upon completion of the asynchronous
+ inference operation. The function will be called with an info argument
+ (:class:`AsyncInferCompletionInfo`) holding the information about the async job. If the async job was
+ unsuccessful, the info parameter will hold an exception method that will raise an exception. The
+ callback must accept a 'completion_info' keyword argument
+
+ Note:
+ As a standard, callbacks should be executed as quickly as possible.
+ In case of an error, the pipeline will be shut down.
+
+ Returns:
+ AsyncInferJob: The async inference job object.
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ """
+ # keep the buffers alive until the job and the callback are completed
+ buffers = []
+ for b in bindings:
+ for name in self._input_names:
+ buffers.append(b.input(name).get_buffer())
+ for name in self._output_names:
+ buffers.append(b.output(name).get_buffer(None))
+ self._buffer_guards.append(buffers)
+
+ def callback_wrapper(error_code):
+ cpp_cb_exception = ExceptionWrapper.create_exception_from_status(error_code) if error_code else None
+ if callback:
+ completion_info = AsyncInferCompletionInfo(cpp_cb_exception)
+ callback(completion_info=completion_info)
+
+ # remove the buffers - they are no longer needed
+ self._buffer_guards.popleft()
+
+ with ExceptionWrapper():
+ cpp_job = self._configured_infer_model.run_async(
+ [b.get() for b in bindings], callback_wrapper
+ )
+
+ job = AsyncInferJob(cpp_job)
+ return job
+
+ def set_scheduler_timeout(self, timeout_ms):
+ """
+ Sets the minimum number of send requests required before the network is considered ready to get run time from the scheduler.
+ Sets the maximum time period that may pass before receiving run time from the scheduler.
+ This will occur providing at least one send request has been sent, there is no minimum requirement for send
+ requests, (e.g. threshold - see :func:`ConfiguredInferModel.set_scheduler_threshold`).
+
+ The new time period will be measured after the previous time the scheduler allocated run time to this network group.
+ Using this function is only allowed when scheduling_algorithm is not `HAILO_SCHEDULING_ALGORITHM_NONE`.
+ The default timeout is 0ms.
+
+ Args:
+ timeout_ms (int): The maximum time to wait for the scheduler to provide run time, in milliseconds.
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ """
+ with ExceptionWrapper():
+ self._configured_infer_model.set_scheduler_timeout(timedelta(milliseconds=timeout_ms))
+
+ def set_scheduler_threshold(self, threshold):
+ """
+ Sets the minimum number of send requests required before the network is considered ready to get run time from the scheduler.
+
+ Args:
+ threshold (int): Threshold in number of frames.
+
+ Using this function is only allowed when scheduling_algorithm is not `HAILO_SCHEDULING_ALGORITHM_NONE`.
+ The default threshold is 1.
+ If at least one send request has been sent, but the threshold is not reached within a set time period (e.g. timeout - see
+ :func:`ConfiguredInferModel.set_scheduler_timeout`), the scheduler will consider the network ready regardless.
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ """
+ with ExceptionWrapper():
+ self._configured_infer_model.set_scheduler_threshold(threshold)
+
+ def set_scheduler_priority(self, priority):
+ """
+ Sets the priority of the network.
+ When the network group scheduler will choose the next network, networks with higher priority will be prioritized in the selection.
+ bigger number represent higher priority.
+
+ Using this function is only allowed when scheduling_algorithm is not `HAILO_SCHEDULING_ALGORITHM_NONE`.
+ The default priority is HAILO_SCHEDULER_PRIORITY_NORMAL.
+
+ Args:
+ priority (int): Priority as a number between `HAILO_SCHEDULER_PRIORITY_MIN` - `HAILO_SCHEDULER_PRIORITY_MAX`.
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ """
+ with ExceptionWrapper():
+ self._configured_infer_model.set_scheduler_priority(priority)
+
+ def get_async_queue_size(self):
+ """
+ Returns Expected of a the number of inferences that can be queued simultaneously for execution.
+
+ Returns:
+ size (int): the number of inferences that can be queued simultaneously for execution
+
+ Raises:
+ :class:`HailoRTException` in case of an error.
+ """
+ with ExceptionWrapper():
+ return self._configured_infer_model.get_async_queue_size()
+
+ def shutdown(self):
+ """
+ Shuts the inference down. After calling this method, the model is no longer usable.
+ """
+ with ExceptionWrapper():
+ return self._configured_infer_model.shutdown()
+
+ def _get_nms_infos(self):
+ nms_infos = {}
+
+ for name in self._output_names:
+ output = self._infer_model.output(name)
+ format_order = output.format.order
+
+ if format_order in (
+ FormatOrder.HAILO_NMS_WITH_BYTE_MASK,
+ FormatOrder.HAILO_NMS,
+ ):
+ output_vstream_info = next(
+ filter(
+ lambda item: item.name == name,
+ self._infer_model.hef.get_output_vstream_infos(),
+ )
+ )
+
+ if format_order == FormatOrder.HAILO_NMS_WITH_BYTE_MASK:
+ if (len(self._input_names)) != 1:
+ raise HailoRTInvalidHEFException(
+ f"Output format order {format_order} should have 1 input. Number of inputs: {len(self._input_names)}"
+ )
+
+ input = self._infer_model.input()
+ input_height, input_width = input.shape[:2]
+ else:
+ input_height, input_width = -1, -1 # not accessed
+
+ nms_infos[name] = self.NmsTransformationInfo(
+ output.format.order,
+ input_height,
+ input_width,
+ output_vstream_info.nms_shape.number_of_classes,
+ output_vstream_info.nms_shape.max_bboxes_per_class,
+ output.quant_infos[0],
+ )
+
+ return nms_infos
+
+
+class AsyncInferJob:
+ """
+ Hailo Asynchronous Inference Job Wrapper.
+ It holds the result of the inference job (once ready), and provides an async poll method to check the job status.
+ """
+
+ MILLISECOND = (1 / 1000)
+
+ def __init__(self, job):
+ #"""
+ #Args:
+ # job (:obj:`_pyhailort.AsyncInferJob`): The internal AsyncInferJob object.
+ #"""
+ self._job = job
+
+ def wait(self, timeout_ms):
+ """
+ Waits for the asynchronous inference job to finish.
+ If the async job and its callback have not completed within the given timeout, a HailoRTTimeout exception will be raised.
+
+ Args:
+ timeout_ms (int): timeout The maximum time to wait.
+
+ Raises:
+ :class:`HailoRTTimeout` in case the job did not finish in the given timeout.
+ """
+ with ExceptionWrapper():
+ self._job.wait(timedelta(milliseconds=timeout_ms))
+
+
class VDevice(object):
"""Hailo virtual device representation."""
with ExceptionWrapper():
return self._vdevice.get_physical_devices_ids()
+ def create_infer_model(self, hef_source, network_name=""):
+ """
+ Creates the infer model from an hef.
+
+ Args:
+ hef_source (str or bytes): The source from which the HEF object will be created. If the
+ source type is `str`, it is treated as a path to an hef file. If the source type is
+ `bytes`, it is treated as a buffer. Any other type will raise a ValueError.
+ network_name (str, optional): The string of the network name.
+
+ Returns:
+ :obj:`InferModel`: The infer model object.
+
+ Raises:
+ :class:`HailoRTException`: In case the infer model creation failed.
+
+ Note:
+ create_infer_model must be called from the same process the VDevice is created in,
+ otherwise an :class:`HailoRTException` will be raised.
+
+ Note:
+ as long as the InferModel object is alive, the VDevice object is alive as well.
+ """
+ if os.getpid() != self._creation_pid:
+ raise HailoRTException("InferModel can be created only from the process VDevice was created in.")
+
+ with ExceptionWrapper():
+ if type(hef_source) is bytes:
+ infer_model_cpp_obj = self._vdevice.create_infer_model_from_buffer(hef_source, network_name)
+ else:
+ infer_model_cpp_obj = self._vdevice.create_infer_model_from_file(hef_source, network_name)
+
+ infer_model = InferModel(infer_model_cpp_obj, hef_source)
+ return infer_model
+
@property
def loaded_network_groups(self):
"""Getter for the property _loaded_network_groups.
if self.output_order == FormatOrder.HAILO_NMS_WITH_BYTE_MASK:
nms_shape = self._vstream_info.nms_shape
if len(self._input_stream_infos) != 1:
- raise Exception("Output format HAILO_NMS_WITH_BYTE_MASK should have 1 input. Number of inputs: {}".format(len(self._input_stream_infos)))
+ raise HailoRTInvalidHEFException(
+ f"Output format HAILO_NMS_WITH_BYTE_MASK should have 1 input. Number of inputs: {len(self._input_stream_infos)}"
+ )
input_height = self._input_stream_infos[0].shape[0]
input_width = self._input_stream_infos[0].shape[1]
res = HailoRTTransformUtils._output_raw_buffer_to_nms_with_byte_mask_format(result_array,
import subprocess
import sys
-import pkg_resources
+import importlib.util
import hailo_platform
from hailo_platform.tools.hailocli.base_utils import HailortCliUtil
def _check_requirements(self):
missing_pkgs = []
- working_set = pkg_resources.WorkingSet()
for req in self.TUTORIALS_REQUIREMENTS:
- try:
- working_set.require(req)
- except pkg_resources.DistributionNotFound:
+ if importlib.util.find_spec(req) is None:
missing_pkgs.append(req)
if missing_pkgs:
"\n",
"# Python inference tutorial\n",
"\n",
- "This tutorial will walk you through the inference process.\n",
+ "This tutorial will describe how to use the Inference Process.\n",
+ "\n",
"\n",
"**Requirements:**\n",
"\n",
"## Standalone hardware deployment\n",
"\n",
"The standalone flow allows direct access to the HW, developing applications directly on top of Hailo\n",
- "core HW, using HailoRT. This way we can use the Hailo hardware without Tensorflow, and\n",
+ "core HW, using HailoRT. This way the Hailo hardware can be used without Tensorflow, and\n",
"even without the Hailo SDK (after the HEF is built).\n",
"\n",
"An HEF is Hailo's binary format for neural networks. The HEF files contain:\n",
"* Weights\n",
"* Metadata for HailoRT (e.g. input/output scaling)\n",
"\n",
- "First create the desired target object. In our example we use the Hailo-8 PCIe interface:\n"
+ "First create the desired target object.\n",
+ "In this example the Hailo-8 PCIe interface is used."
]
},
{
"\n",
"# Python Inference Tutorial - Multi Process Service and Model Scheduler\n",
"\n",
- "This tutorial will walk you through the inference process using The Model Scheduler.\n",
+ "This tutorial describes how to run an inference process using the multi-process service.\n",
+ "\n",
"\n",
"**Requirements:**\n",
"\n",
- "* Enable HailoRT Multi-Process Service before running inference\n",
+ "* Enable HailoRT Multi-Process Service before running inference. For instructions, see [Multi Process Service](https://hailo.ai/developer-zone/documentation/hailort/latest/?sp_referrer=inference/inference.html#multi-process-service).\n",
"* Run the notebook inside the Python virtual environment: ```source hailo_virtualenv/bin/activate```\n",
"\n",
- "It is recommended to use the command ``hailo tutorial`` (when inside the virtualenv) to open a Jupyter server that contains the tutorials."
+ "It is recommended to use the command ``hailo tutorial`` (when inside the ```virtualenv```) to open a Jupyter server that contains the tutorials."
]
},
{
"# Creating the VDevice target with scheduler enabled\n",
"params = VDevice.create_params()\n",
"params.scheduling_algorithm = HailoSchedulingAlgorithm.ROUND_ROBIN\n",
+ "params.multi_process_service = True\n",
"with VDevice(params) as target:\n",
" infer_processes = []\n",
"\n",
--- /dev/null
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "# Python inference tutorial\n",
+ "\n",
+ "This tutorial will describe how to use the Inference Process.\n",
+ "\n",
+ "\n",
+ "**Requirements:**\n",
+ "\n",
+ "* Run the notebook inside the Python virtual environment: ```source hailo_virtualenv/bin/activate```\n",
+ "\n",
+ "It is recommended to use the command ``hailo tutorial`` (when inside the virtualenv) to open a Jupyter server that contains the tutorials."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Standalone hardware deployment\n",
+ "\n",
+ "The standalone flow allows direct access to the HW, developing applications directly on top of Hailo\n",
+ "core HW, using HailoRT. This way the Hailo hardware can be used without Tensorflow, and\n",
+ "even without the Hailo SDK (after the HEF is built).\n",
+ "\n",
+ "An HEF is Hailo's binary format for neural networks. The HEF files contain:\n",
+ "\n",
+ "* Target HW configuration\n",
+ "* Weights\n",
+ "* Metadata for HailoRT (e.g. input/output scaling)\n",
+ "\n",
+ "First create the desired target object.\n",
+ "In this example the Hailo-8 PCIe interface is used."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "from hailo_platform import VDevice, HailoSchedulingAlgorithm\n",
+ "\n",
+ "timeout_ms = 1000\n",
+ "\n",
+ "params = VDevice.create_params()\n",
+ "params.scheduling_algorithm = HailoSchedulingAlgorithm.ROUND_ROBIN\n",
+ "\n",
+ "# The vdevice is used as a context manager (\"with\" statement) to ensure it's released on time.\n",
+ "with VDevice(params) as vdevice:\n",
+ "\n",
+ " # Create an infer model from an HEF:\n",
+ " infer_model = vdevice.create_infer_model('../hefs/resnet_v1_18.hef')\n",
+ "\n",
+ " # Configure the infer model and create bindings for it\n",
+ " with infer_model.configure() as configured_infer_model:\n",
+ " bindings = configured_infer_model.create_bindings()\n",
+ "\n",
+ " # Set input and output buffers\n",
+ " buffer = np.empty(infer_model.input().shape).astype(np.uint8)\n",
+ " bindings.input().set_buffer(buffer)\n",
+ "\n",
+ " buffer = np.empty(infer_model.output().shape).astype(np.uint8)\n",
+ " bindings.output().set_buffer(buffer)\n",
+ "\n",
+ " # Run synchronous inference and access the output buffers\n",
+ " configured_infer_model.run([bindings], timeout_ms)\n",
+ " buffer = bindings.output().get_buffer()\n",
+ "\n",
+ " # Run asynchronous inference\n",
+ " job = configured_infer_model.run_async([bindings])\n",
+ " job.wait(timeout_ms)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
--- /dev/null
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "# Python Async Inference Tutorial - Multiple Models with Model Scheduler\n",
+ "\n",
+ "This tutorial will describe how to run an inference process.\n",
+ "\n",
+ "\n",
+ "**Requirements:**\n",
+ "\n",
+ "* Run the notebook inside the Python virtual environment: ```source hailo_virtualenv/bin/activate```\n",
+ "\n",
+ "It is recommended to use the command ``hailo tutorial`` (when inside the ```virtualenv```) to open a Jupyter server that contains the tutorials."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Running Inference using HailoRT\n",
+ "\n",
+ "In this example we will use the Model Scheduler to run inference on multiple models.\n",
+ "Each model is represented by an HEF which is built using the Hailo Dataflow Compiler.\n",
+ "An HEF is Hailo's binary format for neural networks. The HEF files contain:\n",
+ "\n",
+ "* Target HW configuration\n",
+ "* Weights\n",
+ "* Metadata for HailoRT (e.g. input/output scaling)\n",
+ "\n",
+ "The Model Scheduler is an HailoRT component that comes to enhance and simplify the usage\n",
+ "of the same Hailo device by multiple networks. The responsibility for activating/deactivating the network\n",
+ "groups is now under HailoRT, and done **automatically** without user application intervention.\n",
+ "In order to use the Model Scheduler, create the VDevice with scheduler enabled, configure all models to the device, and start inference on all models:\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Optional: define a callback function that will run after the inference job is done\n",
+ "# The callback must have a keyword argument called \"completion_info\".\n",
+ "# That argument will be passed by the framework.\n",
+ "def example_callback(completion_info, bindings):\n",
+ " if completion_info.exception:\n",
+ " # handle exception\n",
+ " pass\n",
+ " \n",
+ " _ = bindings.output().get_buffer()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "from functools import partial\n",
+ "from hailo_platform import VDevice, HailoSchedulingAlgorithm, FormatType\n",
+ "\n",
+ "number_of_frames = 4\n",
+ "timeout_ms = 10000\n",
+ "\n",
+ "def infer(multi_process_service):\n",
+ " # Create a VDevice\n",
+ " params = VDevice.create_params()\n",
+ " params.scheduling_algorithm = HailoSchedulingAlgorithm.ROUND_ROBIN\n",
+ " params.group_id = \"SHARED\" \n",
+ " if multi_process_service:\n",
+ " params.multi_process_service = multi_process_service\n",
+ " \n",
+ " with VDevice(params) as vdevice:\n",
+ "\n",
+ " # Create an infer model from an HEF:\n",
+ " infer_model = vdevice.create_infer_model('../hefs/resnet_v1_18.hef')\n",
+ "\n",
+ " # Set optional infer model parameters\n",
+ " infer_model.set_batch_size(2)\n",
+ "\n",
+ " # For a single input / output model, the input / output object \n",
+ " # can be accessed with a name parameter ...\n",
+ " infer_model.input(\"input_layer1\").set_format_type(FormatType.FLOAT32)\n",
+ " # ... or without\n",
+ " infer_model.output().set_format_type(FormatType.FLOAT32)\n",
+ "\n",
+ " # Once the infer model is set, configure the infer model\n",
+ " with infer_model.configure() as configured_infer_model:\n",
+ " for _ in range(number_of_frames):\n",
+ " # Create bindings for it and set buffers\n",
+ " bindings = configured_infer_model.create_bindings()\n",
+ " bindings.input().set_buffer(np.empty(infer_model.input().shape).astype(np.float32))\n",
+ " bindings.output().set_buffer(np.empty(infer_model.output().shape).astype(np.float32))\n",
+ "\n",
+ " # Wait for the async pipeline to be ready, and start an async inference job\n",
+ " configured_infer_model.wait_for_async_ready(timeout_ms=10000)\n",
+ "\n",
+ " # Any callable can be passed as callback (lambda, function, functools.partial), as long\n",
+ " # as it has a keyword argument \"completion_info\"\n",
+ " job = configured_infer_model.run_async([bindings], partial(example_callback, bindings=bindings))\n",
+ "\n",
+ " # Wait for the last job\n",
+ " job.wait(timeout_ms)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Running multiple models concurrently\n",
+ "\n",
+ "The models can be run concurrently using either multiple `Thread` objects or multiple `Process` objects"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from threading import Thread\n",
+ "\n",
+ "pool = [\n",
+ " Thread(target=infer, args=(False,)),\n",
+ " Thread(target=infer, args=(False,))\n",
+ "]\n",
+ "\n",
+ "print('Starting async inference on multiple models using threads')\n",
+ "\n",
+ "for job in pool:\n",
+ " job.start()\n",
+ "for job in pool:\n",
+ " job.join()\n",
+ "\n",
+ "print('Done inference')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If the models are run in different processes, the multi-process service must be enabled first."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from multiprocessing import Process\n",
+ "\n",
+ "pool = [\n",
+ " Process(target=infer, args=(True,)),\n",
+ " Process(target=infer, args=(True,))\n",
+ "]\n",
+ "\n",
+ "print('Starting async inference on multiple models using processes')\n",
+ "\n",
+ "for job in pool:\n",
+ " job.start()\n",
+ "for job in pool:\n",
+ " job.join()\n",
+ "\n",
+ "print('Done inference')"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
+"""
+builds hailo_platform python package and its C++ dependencies using cmake
+"""
+import platform
import os
-import json
-from setuptools import setup, find_packages
+import subprocess
+import sys
+
+from pathlib import Path
+from setuptools import setup, Extension, find_packages
+from setuptools.command.build_ext import build_ext as orig_build_ext
from wheel.bdist_wheel import bdist_wheel as orig_bdist_wheel
-class NonPurePythonBDistWheel(orig_bdist_wheel):
- """makes the wheel platform-dependant so it can be based on the _pyhailort architecture"""
+_plat_name = None
+def _fix_plat_name(s):
+ # plat_name does not require the "linux_" prefix
+ return s.replace(platform.processor(), _plat_name.replace("linux_", ""))
+
+class bdist_wheel(orig_bdist_wheel):
+ """makes the wheel platform-dependant so it can be based on the _pyhailort architecture"""
def finalize_options(self):
+ # Save the plat_name option and pass it along to build_ext which will use it to change the processor in the
+ # extension name.
+ # All other paths will still use the naive processor, but that's ok, since the only thing that is packed into
+ # the wheel is the actual shared library, so only its name is relevant. Fixing all paths will require tweaking
+ # build_py, install, install_lib commands or fixing this somehow all accross setuptools
+ global _plat_name
+ _plat_name = self.plat_name
orig_bdist_wheel.finalize_options(self)
self.root_is_pure = False
-def _get_pyhailort_lib_path():
- conf_file_path = os.path.join(os.path.abspath(os.path.dirname( __file__ )), "wheel_conf.json")
- extension = {
- "posix": "so",
- "nt": "pyd", # Windows
- }[os.name]
- if not os.path.isfile(conf_file_path):
- return None
+class build_ext(orig_build_ext):
+ OPTIONAL_CMAKE_ENV_VARIABLES = [
+ "CMAKE_TOOLCHAIN_FILE",
+ "HAILORT_INCLUDE_DIR",
+ "LIBHAILORT_PATH",
+ "PYTHON_INCLUDE_DIRS",
+ "CMAKE_GENERATOR",
+ "PYTHON_LIBRARY",
+ ]
+
+ """defines a cmake command that will be called from the python build process"""
+ def run(self):
+ cfg = 'Debug' if self.debug else 'Release'
+
+ build_args = f"--config {cfg}"
+ build_directory = os.path.abspath(self.build_temp)
+ cmake_list_dir = Path(__file__).absolute().parents[1] / "src"
+ python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+ cmake_args = [
+ f'-DCMAKE_BUILD_TYPE={cfg}',
+ f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={build_directory}',
+ f'-DPYBIND11_PYTHON_VERSION={python_version}',
+ f'-DPYTHON_EXECUTABLE={sys.executable}',
+ ]
+
+ for env_var in self.OPTIONAL_CMAKE_ENV_VARIABLES:
+ if env_var in os.environ:
+ if env_var == "CMAKE_GENERATOR":
+ cmake_args.append(f'-G "{os.environ[env_var]}"')
+ else:
+ cmake_args.append(f"-D{env_var}={os.environ[env_var]}")
- with open(conf_file_path, "r") as conf_file:
- content = json.load(conf_file)
- # TODO (HRT-8637): change this hard-coded path
- return f"../hailo_platform/pyhailort/_pyhailort*{content['py_version']}*{content['arch']}*.{extension}"
+ if not os.path.exists(self.build_temp):
+ os.makedirs(self.build_temp)
-def _get_package_paths():
- packages = []
- pyhailort_lib = _get_pyhailort_lib_path()
- if pyhailort_lib:
- packages.append(pyhailort_lib)
- packages.append("../hailo_tutorials/notebooks/*")
- packages.append("../hailo_tutorials/hefs/*")
- return packages
+ subprocess.run(
+ f"cmake {cmake_list_dir} {' '.join(cmake_args)}",
+ cwd=self.build_temp,
+ shell=True,
+ check=True
+ )
+
+ subprocess.run(
+ f"cmake --build . {build_args}",
+ cwd=self.build_temp,
+ shell=True,
+ check=True,
+ )
+
+ for ext in self.extensions:
+ ext_filename = self.get_ext_filename(ext.name)
+ if platform.system() == "Linux" and _plat_name:
+ ext_filename = _fix_plat_name(ext_filename)
+
+ dst = Path(self.get_ext_fullpath(ext.name)).resolve().parent / "hailo_platform/pyhailort/"
+
+ build_temp = Path(self.build_temp).resolve()
+ if os.name == "nt":
+ src = build_temp / cfg / ext_filename
+ else:
+ src = build_temp / ext_filename
+
+ self.copy_file(src, dst)
if __name__ == "__main__":
author="Hailo team",
author_email="contact@hailo.ai",
cmdclass={
- "bdist_wheel": NonPurePythonBDistWheel,
+ "bdist_wheel": bdist_wheel,
+ "build_ext": build_ext, # Build the C++ extension (_pyhailort) using cmake
},
description="HailoRT",
entry_points={
"hailo=hailo_platform.tools.hailocli.main:main",
]
},
+ ext_modules= [
+ Extension('_pyhailort', sources=[]),
+ ],
install_requires=[
"argcomplete",
"contextlib2",
],
name="hailort",
package_data={
- "hailo_platform": _get_package_paths(),
+ "hailo_platform": [
+ "../hailo_tutorials/notebooks/*",
+ "../hailo_tutorials/hefs/*"
+ ]
},
packages=find_packages(),
platforms=[
"linux_aarch64",
],
url="https://hailo.ai/",
- version="4.17.1",
+ version="4.18.0",
zip_safe=False,
)
cmake_minimum_required(VERSION 3.0.0)
+project(pyhailort)
+
+get_filename_component(HAILORT_PROJECT_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../../../" ABSOLUTE)
+get_filename_component(HAILORT_COMMON_DIR "${HAILORT_PROJECT_SOURCE_DIR}/hailort/" ABSOLUTE)
+get_filename_component(PYHAILORT_DIR "${CMAKE_CURRENT_LIST_DIR}" ABSOLUTE)
+
+set(HAILO_EXTERNAL_DIR ${HAILORT_COMMON_DIR}/external)
+set(HAILO_EXTERNALS_CMAKE_SCRIPTS ${HAILORT_COMMON_DIR}/cmake/external/)
+
+option(LIBHAILORT_PATH "Path to libhailort to link against" "")
+option(HAILORT_INCLUDE_DIR "Path to include dir of libhailort" "")
include(ExternalProject)
+include(GNUInstallDirs)
+include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/pybind11.cmake)
+include_directories(${HAILORT_COMMON_DIR})
FUNCTION(exclude_archive_libs_symbols target) # should be same as in common_compiler_options.cmake
if(WIN32)
get_property(TEMP_LINK_FLAGS TARGET ${target} PROPERTY LINK_FLAGS)
set(TEMP_LINK_FLAGS "${TEMP_LINK_FLAGS} -Wl,--exclude-libs=ALL")
set_property(TARGET ${target} PROPERTY LINK_FLAGS ${TEMP_LINK_FLAGS})
- else()
- message(FATAL_ERROR "Unexpeced host, stopping build")
endif()
ENDFUNCTION()
set(PYTHON_MODULE_EXTENSION ".cpython-${dpython}${m_flag}-${CMAKE_SYSTEM_PROCESSOR}-linux-gnu.so")
endif()
-option(HAILO_BUILD_PYHAILORT_INTERNAL OFF)
-
-set(PYHAILORT_DIR ${CMAKE_CURRENT_LIST_DIR})
-
pybind11_add_module(_pyhailort
pyhailort.cpp
device_api.cpp
+ vdevice_api.cpp
+ infer_model_api.cpp
network_group_api.cpp
hef_api.cpp
vstream_api.cpp
# VISIBILITY_INLINES_HIDDEN YES
)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+# allow user to inject a specific libhailort (and headers) to link against.
+# use case: cross compilation
+if(LIBHAILORT_PATH AND HAILORT_INCLUDE_DIR)
+ message(STATUS "LIBHAILORT_PATH is set. Will link against given libhailort: ${LIBHAILORT_PATH}")
+ message(STATUS "HAILORT_INCLUDE_DIR is set. Will include given include dir: ${HAILORT_INCLUDE_DIR}")
+
+ # the library to link against
+ target_link_libraries(_pyhailort PRIVATE ${LIBHAILORT_PATH})
+
+ # the include dir
+ include_directories(${HAILORT_INCLUDE_DIR})
+
+ # since we are linking against an injected libhailort, we need to define the version
+ target_compile_definitions(
+ _pyhailort
+ PUBLIC
+ HAILORT_MAJOR_VERSION=4
+ HAILORT_MINOR_VERSION=18
+ HAILORT_REVISION_VERSION=0
+ )
+elseif(LIBHAILORT_PATH OR HAILORT_INCLUDE_DIR)
+ message(FATAL_ERROR "Both LIBHAILORT_PATH and HAILORT_INCLUDE_DIR must be defined or none of them")
+else()
+ find_package(HailoRT 4.18.0 EXACT REQUIRED)
+ target_link_libraries(_pyhailort PRIVATE HailoRT::libhailort)
+endif()
-target_link_libraries(_pyhailort PRIVATE HailoRT::libhailort)
if(WIN32)
target_link_libraries(_pyhailort PRIVATE Ws2_32)
target_compile_options(_pyhailort PRIVATE
target_compile_options(_pyhailort PRIVATE ${HAILORT_COMPILE_OPTIONS})
exclude_archive_libs_symbols(_pyhailort)
-if (HAILO_BUILD_PYHAILORT_INTERNAL)
- add_subdirectory(internal)
- # copy files to a path the venv will look for
- add_custom_target(pyhailort_internal_venv ALL
- COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:_pyhailort_internal> ${PROJECT_SOURCE_DIR}/platform_internals/hailo_platform_internals/pyhailort/
- )
- add_dependencies(pyhailort_internal_venv _pyhailort_internal)
-endif()
-
# TODO (HRT-8637): change this hard-coded path
set(HAILO_PYHAILORT_TARGET_DIR ${CMAKE_CURRENT_LIST_DIR}/../platform/hailo_platform/pyhailort/)
-# copy files to a path the venv and whl will look for
-message(STATUS "Copying _pyhailort artifacts into " ${HAILO_PYHAILORT_TARGET_DIR})
add_custom_target(pyhailort_venv ALL
COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:_pyhailort> ${HAILO_PYHAILORT_TARGET_DIR}
)
install(TARGETS _pyhailort
LIBRARY DESTINATION ${HAILO_PYHAILORT_TARGET_DIR}
CONFIGURATIONS Release
-)
\ No newline at end of file
+)
+
}
}
- static std::vector<size_t> get_pybind_shape(const hailo_vstream_info_t& vstream_info, const hailo_format_t &user_format)
+ static std::vector<size_t> get_pybind_shape(
+ const hailo_3d_image_shape_t &shape,
+ const hailo_nms_shape_t &nms_shape,
+ const hailo_format_t &user_format)
{
- // We are using user_format instead of hw format inside the vstream_info
- const auto shape = vstream_info.shape;
- // TODO: support no transformations (i.e. use stream_info.hw_shape) (SDK-16811)
switch (user_format.order)
{
case HAILO_FORMAT_ORDER_HAILO_NMS:
- return { HailoRTCommon::get_nms_host_shape_size(vstream_info.nms_shape) };
- case HAILO_FORMAT_ORDER_HAILO_NMS_WITH_BYTE_MASK: {
- return {HailoRTCommon::get_nms_host_frame_size(vstream_info.nms_shape, user_format) / HailoRTCommon::get_format_data_bytes(user_format)};
- }
+ return { HailoRTCommon::get_nms_host_shape_size(nms_shape) };
+ case HAILO_FORMAT_ORDER_HAILO_NMS_WITH_BYTE_MASK:
+ return {HailoRTCommon::get_nms_host_frame_size(nms_shape, user_format) / HailoRTCommon::get_format_data_bytes(user_format)};
case HAILO_FORMAT_ORDER_NC:
return {shape.features};
case HAILO_FORMAT_ORDER_NHW:
void DeviceWrapper::set_notification_callback(const std::function<void(uintptr_t, const hailo_notification_t&, py::object)> &callback,
hailo_notification_id_t notification_id, py::object opaque)
{
- // we capture opaque and move it because when opaque goes out of score it will be deleted,
+ // capture the opaque and move it, this is because when opaque goes out of scope it will be automatically deleted,
// so capturing it ensures that it will not be deleted
hailo_status status = device().set_notification_callback(
[callback, op = std::move(opaque)] (Device &device, const hailo_notification_t ¬ification, void* opaque) {
VALIDATE_STATUS(status);
}
-void DeviceWrapper::add_to_python_module(py::module &m)
+void DeviceWrapper::_init_cache_info(const hailo_cache_info_t &cache_info)
+{
+ auto status = device().init_cache_info(cache_info);
+ VALIDATE_STATUS(status);
+}
+
+hailo_cache_info_t DeviceWrapper::_get_cache_info()
+{
+ auto cache_info = device().get_cache_info();
+ VALIDATE_EXPECTED(cache_info);
+
+ return cache_info.release();
+}
+
+void DeviceWrapper::_update_cache_read_offset(int32_t read_offset_delta)
+{
+ auto status = device().update_cache_read_offset(read_offset_delta);
+ VALIDATE_STATUS(status);
+}
+
+void DeviceWrapper::bind(py::module &m)
{
py::class_<DeviceWrapper>(m, "Device")
.def("is_valid", &DeviceWrapper::is_valid)
.def("set_notification_callback", &DeviceWrapper::set_notification_callback)
.def("remove_notification_callback", &DeviceWrapper::remove_notification_callback)
.def("set_sleep_state", &DeviceWrapper::set_sleep_state)
+ .def("_init_cache_info", &DeviceWrapper::_init_cache_info)
+ .def("_get_cache_info", &DeviceWrapper::_get_cache_info)
+ .def("_update_cache_read_offset", &DeviceWrapper::_update_cache_read_offset)
;
}
py::bytes direct_read_memory(uint32_t address, uint32_t size);
const char *get_dev_id() const;
void set_sleep_state(hailo_sleep_state_t sleep_state);
+ void _init_cache_info(const hailo_cache_info_t &cache_info);
+ hailo_cache_info_t _get_cache_info();
+ void _update_cache_read_offset(int32_t read_offset_delta);
- static void add_to_python_module(py::module &m);
+ static void bind(py::module &m);
private:
DeviceWrapper(std::unique_ptr<Device> &&device)
}
}
-HefWrapper HefWrapper::create_from_buffer(py::bytes data)
+HefWrapper HefWrapper::create_from_buffer(const py::bytes &data)
{
- return HefWrapper(MemoryView((uint8_t*)std::string(data).c_str(), std::string(data).size()));
+ // TODO: HRT-13713 - When adding support to read the hef from pre-allocated memory,
+ // we will need to make sure the hef memory is not released here.
+ py::buffer_info info(py::buffer(data).request());
+ return HefWrapper(MemoryView::create_const(info.ptr, info.size));
}
HefWrapper HefWrapper::create_from_file(const std::string &hef_path)
return py::cast(res);
}
-void HefWrapper::initialize_python_module(py::module &m)
+void HefWrapper::bind(py::module &m)
{
py::class_<HefWrapper>(m, "Hef")
.def("create_from_buffer", &HefWrapper::create_from_buffer)
public:
HefWrapper(const std::string &hef_path);
HefWrapper(const MemoryView &hef_buffer);
- static HefWrapper create_from_buffer(py::bytes data);
+ static HefWrapper create_from_buffer(const py::bytes &data);
static HefWrapper create_from_file(const std::string &hef_path);
py::list get_network_group_names();
py::list get_network_groups_infos();
py::dict create_configure_params_mipi_input(hailo_stream_interface_t output_interface,
const hailo_mipi_input_stream_params_t &mipi_params);
py::list get_networks_names(const std::string &net_group_name);
- static void initialize_python_module(py::module &m);
+ static void bind(py::module &m);
private:
std::unique_ptr<Hef> hef;
--- /dev/null
+/**
+ * Copyright (c) 2020-2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file infer_model_api.cpp
+ * @brief Defines binding to a infer model class family usage over Python.
+ **/
+#include "infer_model_api.hpp"
+#include "bindings_common.hpp"
+#include "hailo/infer_model.hpp"
+
+#include <chrono>
+#include <condition_variable>
+#include <cstddef>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <pybind11/gil.h> // py::gil_scoped_release
+#include <pybind11/stl.h> // handle std::vector
+#include <pybind11/functional.h> // handle std::function
+#include <pybind11/chrono.h> // handle std::chrono::milliseconds
+
+using namespace hailort;
+
+void InferModelWrapper::set_batch_size(uint16_t batch_size)
+{
+ m_infer_model->set_batch_size(batch_size);
+}
+
+void InferModelWrapper::set_power_mode(hailo_power_mode_t power_mode)
+{
+ m_infer_model->set_power_mode(power_mode);
+}
+
+std::vector<InferModelInferStreamWrapper> InferModelWrapper::inputs()
+{
+ auto infer_streams = m_infer_model->inputs();
+ std::vector<InferModelInferStreamWrapper> wrappers;
+ for (auto &infer_stream : infer_streams)
+ {
+ wrappers.push_back(InferModelInferStreamWrapper(std::move(infer_stream)));
+ }
+ return wrappers;
+}
+
+std::vector<InferModelInferStreamWrapper> InferModelWrapper::outputs()
+{
+ auto infer_streams = m_infer_model->outputs();
+ std::vector<InferModelInferStreamWrapper> wrappers;
+ for (auto &infer_stream : infer_streams)
+ {
+ wrappers.push_back(InferModelInferStreamWrapper(std::move(infer_stream)));
+ }
+ return wrappers;
+}
+
+ConfiguredInferModelWrapper InferModelWrapper::configure()
+{
+ auto configured_infer_model = m_infer_model->configure();
+ VALIDATE_EXPECTED(configured_infer_model);
+ return ConfiguredInferModelWrapper(configured_infer_model.release(), m_is_using_service,
+ m_infer_model->get_output_names());
+}
+
+std::vector<std::string> InferModelWrapper::get_input_names()
+{
+ return m_infer_model->get_input_names();
+}
+
+std::vector<std::string> InferModelWrapper::get_output_names()
+{
+ return m_infer_model->get_output_names();
+}
+
+InferModelInferStreamWrapper InferModelWrapper::input(const std::string &name)
+{
+ auto infer_stream = name.empty() ? m_infer_model->input() : m_infer_model->input(name);
+ VALIDATE_EXPECTED(infer_stream);
+ return InferModelInferStreamWrapper(infer_stream.release());
+}
+
+InferModelInferStreamWrapper InferModelWrapper::output(const std::string &name)
+{
+ auto infer_stream = name.empty() ? m_infer_model->output() : m_infer_model->output(name);
+ VALIDATE_EXPECTED(infer_stream);
+ return InferModelInferStreamWrapper(infer_stream.release());
+}
+
+ConfiguredInferModelBindingsWrapper ConfiguredInferModelWrapper::create_bindings()
+{
+ auto bindings = m_configured_infer_model.create_bindings();
+ VALIDATE_EXPECTED(bindings);
+ return ConfiguredInferModelBindingsWrapper(bindings.release(), m_output_names);
+}
+
+void ConfiguredInferModelWrapper::activate()
+{
+ auto status = m_configured_infer_model.activate();
+ VALIDATE_STATUS(status);
+}
+
+void ConfiguredInferModelWrapper::deactivate()
+{
+ auto status = m_configured_infer_model.deactivate();
+ VALIDATE_STATUS(status);
+}
+
+ConfiguredInferModelBindingsInferStreamWrapper ConfiguredInferModelBindingsWrapper::input(const std::string &name)
+{
+ auto infer_stream = name.empty() ? m_bindings.input() : m_bindings.input(name);
+ VALIDATE_EXPECTED(infer_stream);
+ return ConfiguredInferModelBindingsInferStreamWrapper(infer_stream.release());
+}
+
+ConfiguredInferModelBindingsInferStreamWrapper ConfiguredInferModelBindingsWrapper::output(const std::string &name)
+{
+ auto infer_stream = name.empty() ? m_bindings.output() : m_bindings.output(name);
+ VALIDATE_EXPECTED(infer_stream);
+ return ConfiguredInferModelBindingsInferStreamWrapper(infer_stream.release());
+}
+
+const std::string InferModelInferStreamWrapper::name() const
+{
+ return m_infer_stream.name();
+}
+
+void InferModelInferStreamWrapper::set_format_type(hailo_format_type_t type)
+{
+ m_infer_stream.set_format_type(type);
+}
+
+void InferModelInferStreamWrapper::set_format_order(hailo_format_order_t order)
+{
+ m_infer_stream.set_format_order(order);
+}
+
+std::vector<hailo_quant_info_t> InferModelInferStreamWrapper::get_quant_infos() const
+{
+ return m_infer_stream.get_quant_infos();
+}
+
+std::vector<size_t> InferModelInferStreamWrapper::shape() const
+{
+ auto shape = m_infer_stream.shape();
+ auto format = m_infer_stream.format();
+ hailo_nms_shape_t nms_shape; // if the format is non-NMS, this struct won't be used
+
+ if (HailoRTCommon::is_nms(format.order))
+ {
+ auto expected = m_infer_stream.get_nms_shape();
+ VALIDATE_EXPECTED(expected);
+ nms_shape = expected.release();
+ }
+
+ return HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format);
+}
+
+hailo_format_t InferModelInferStreamWrapper::format() const
+{
+ return m_infer_stream.format();
+}
+
+bool InferModelInferStreamWrapper::is_nms() const
+{
+ return m_infer_stream.is_nms();
+}
+
+void InferModelInferStreamWrapper::set_nms_score_threshold(float32_t threshold)
+{
+ m_infer_stream.set_nms_score_threshold(threshold);
+}
+
+void InferModelInferStreamWrapper::set_nms_iou_threshold(float32_t threshold)
+{
+ m_infer_stream.set_nms_iou_threshold(threshold);
+}
+
+void InferModelInferStreamWrapper::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class)
+{
+ m_infer_stream.set_nms_max_proposals_per_class(max_proposals_per_class);
+}
+
+void InferModelInferStreamWrapper::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size)
+{
+ m_infer_stream.set_nms_max_accumulated_mask_size(max_accumulated_mask_size);
+}
+
+void ConfiguredInferModelBindingsInferStreamWrapper::set_buffer(py::array buffer)
+{
+ MemoryView view(buffer.mutable_data(), static_cast<size_t>(buffer.nbytes()));
+ auto status = m_infer_stream.set_buffer(view);
+ VALIDATE_STATUS(status);
+}
+
+void ConfiguredInferModelWrapper::wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count)
+{
+ auto status = m_configured_infer_model.wait_for_async_ready(timeout, frames_count);
+ VALIDATE_STATUS(status);
+}
+
+void ConfiguredInferModelWrapper::run(
+ ConfiguredInferModelBindingsWrapper bindings,
+ std::chrono::milliseconds timeout)
+{
+ auto status = m_configured_infer_model.run(bindings.get(), timeout);
+ VALIDATE_STATUS(status);
+}
+
+AsyncInferJobWrapper ConfiguredInferModelWrapper::run_async(
+ std::vector<ConfiguredInferModelBindingsWrapper> &user_bindings,
+ AsyncInferCallBack pythonic_cb)
+{
+ std::function<void(const AsyncInferCompletionInfo &info)> cb;
+
+ // create an event that will be held by AsyncInferJobWrapper to know if the callback has been executed
+ auto is_callback_done_expected = Event::create_shared(Event::State::not_signalled);
+ VALIDATE_EXPECTED(is_callback_done_expected);
+ auto is_callback_done = is_callback_done_expected.release();
+
+ if (!m_is_using_service)
+ {
+ // a worker thread will handle the user callbacks
+ // this function should register the callback and notify the thread once the callback is ready to be executed
+ if(!m_callbacks_thread)
+ {
+ m_callbacks_thread = std::make_shared<std::thread>(&ConfiguredInferModelWrapper::execute_callbacks, this);
+ }
+
+ // the pythonic_cb (i.e. user-defined callback) will signal once the callback has been executed
+ auto pythonic_cb_wrapper = [pythonic_cb, is_callback_done](const int error_code)
+ {
+ pythonic_cb(error_code);
+ is_callback_done->signal();
+ };
+
+ // enqueue the pythonic_cb (i.e. user-defined callback) only when libhailort's
+ // async job is done. Once enqueued, the job will be dequeued by the worker thread
+ cb = [this, pythonic_cb_wrapper](const AsyncInferCompletionInfo &info)
+ {
+ {
+ std::lock_guard<std::mutex> lock(m_queue_mutex);
+ // push the callback and the status to the queue. If the status is not HAILO_SUCCESS,
+ // the user-defined callback will be get an indication of the failure
+ m_callbacks_queue->push(std::make_pair(pythonic_cb_wrapper, info));
+ }
+ m_cv.notify_one();
+ };
+ }
+ else
+ {
+ // the user callbacks will be called from libahilort, and this object won't have a worker thread to notify
+ cb = [pythonic_cb, is_callback_done](const AsyncInferCompletionInfo &info)
+ {
+ pythonic_cb(info.status);
+ is_callback_done->signal();
+ };
+ }
+
+ std::vector<ConfiguredInferModel::Bindings> bindings;
+ std::transform(user_bindings.begin(), user_bindings.end(), std::back_inserter(bindings),
+ [](ConfiguredInferModelBindingsWrapper &wrapper) { return wrapper.get(); });
+
+ std::vector<void*> user_output_buffers;
+ std::vector<BufferPtr> aligned_output_buffers;
+ for (auto &binding : bindings)
+ {
+ for (const auto &name : m_output_names)
+ {
+ auto stream = binding.output(name);
+ VALIDATE_EXPECTED(stream);
+
+ auto buffer = stream->get_buffer();
+ VALIDATE_EXPECTED(buffer);
+
+ user_output_buffers.push_back(buffer->data());
+
+ auto aligned_buffer = Buffer::create_shared(buffer->size(), BufferStorageParams::create_dma());
+ VALIDATE_EXPECTED(aligned_buffer);
+
+ auto buf = aligned_buffer.release();
+ aligned_output_buffers.push_back(buf);
+
+ auto status = stream->set_buffer(MemoryView(buf->data(), buf->size()));
+ VALIDATE_STATUS(status);
+ }
+ }
+
+ auto cb_wrapper_with_output_copy = [cb, user_output_buffers, aligned_output_buffers](const AsyncInferCompletionInfo &info)
+ {
+ for (size_t i = 0; i < user_output_buffers.size(); i++)
+ {
+ std::memcpy(user_output_buffers[i], aligned_output_buffers[i]->data(), aligned_output_buffers[i]->size());
+ }
+
+ cb(info);
+ };
+
+ auto job = m_configured_infer_model.run_async(bindings, cb_wrapper_with_output_copy);
+ VALIDATE_EXPECTED(job);
+
+ // don't wait for the job at this location. The user should call job.wait() from the python side
+ job->detach();
+
+ return AsyncInferJobWrapper(job.release(), is_callback_done);
+}
+
+void ConfiguredInferModelWrapper::set_scheduler_timeout(const std::chrono::milliseconds &timeout)
+{
+ auto status = m_configured_infer_model.set_scheduler_timeout(timeout);
+ VALIDATE_STATUS(status);
+}
+
+void ConfiguredInferModelWrapper::set_scheduler_threshold(uint32_t threshold)
+{
+ auto status = m_configured_infer_model.set_scheduler_threshold(threshold);
+ VALIDATE_STATUS(status);
+}
+
+void ConfiguredInferModelWrapper::set_scheduler_priority(uint8_t priority)
+{
+ auto status = m_configured_infer_model.set_scheduler_priority(priority);
+ VALIDATE_STATUS(status);
+}
+
+size_t ConfiguredInferModelWrapper::get_async_queue_size()
+{
+ auto size = m_configured_infer_model.get_async_queue_size();
+ VALIDATE_EXPECTED(size);
+ return size.release();
+}
+
+void ConfiguredInferModelWrapper::shutdown()
+{
+ auto status = m_configured_infer_model.shutdown();
+ VALIDATE_STATUS(status);
+}
+
+void ConfiguredInferModelWrapper::execute_callbacks()
+{
+ while (true)
+ {
+ std::unique_lock<std::mutex> lock(m_queue_mutex);
+ m_cv.wait_for(lock, std::chrono::minutes(1), [this](){ return !m_callbacks_queue->empty() || !m_is_alive.load(); });
+
+ if (!m_is_alive.load())
+ {
+ while (!m_callbacks_queue->empty()) {
+ auto cb_status_pair = m_callbacks_queue->front();
+ auto &cb = cb_status_pair.first;
+ auto status = cb_status_pair.second;
+ cb(status.status);
+ m_callbacks_queue->pop();
+ }
+ return;
+ }
+
+ auto cb_status_pair = m_callbacks_queue->front();
+
+ m_callbacks_queue->pop();
+ lock.unlock(); // release the lock before calling the callback, allowing other threads to push to the queue
+
+ auto &cb = cb_status_pair.first;
+ auto status = cb_status_pair.second;
+ cb(status.status);
+ }
+}
+
+void AsyncInferJobWrapper::wait(std::chrono::milliseconds timeout)
+{
+ // TODO: currently waiting for 2 TIMEOUT (worst case). Fix it
+ auto status = m_job.wait(timeout);
+ VALIDATE_STATUS(status);
+
+ status = m_is_callback_done->wait(timeout);
+ VALIDATE_STATUS(status);
+}
+
+void InferModelWrapper::bind(py::module &m)
+{
+ py::class_<
+ InferModelWrapper,
+ std::shared_ptr<InferModelWrapper>
+ >(m, "InferModel")
+ //.def("hef", &InferModelWrapper::hef)
+ .def("configure", &InferModelWrapper::configure)
+ .def("set_batch_size", &InferModelWrapper::set_batch_size)
+ .def("set_power_mode", &InferModelWrapper::set_power_mode)
+ .def("get_input_names", &InferModelWrapper::get_input_names)
+ .def("get_output_names", &InferModelWrapper::get_output_names)
+ .def("inputs", &InferModelWrapper::inputs)
+ .def("outputs", &InferModelWrapper::outputs)
+ .def("input", &InferModelWrapper::input)
+ .def("output", &InferModelWrapper::output)
+ ;
+}
+
+void ConfiguredInferModelWrapper::bind(py::module &m)
+{
+ py::class_<
+ ConfiguredInferModelWrapper,
+ std::shared_ptr<ConfiguredInferModelWrapper>
+ >(m, "ConfiguredInferModel")
+ .def("create_bindings", &ConfiguredInferModelWrapper::create_bindings)
+ .def("activate", &ConfiguredInferModelWrapper::activate)
+ .def("deactivate", &ConfiguredInferModelWrapper::deactivate)
+ // wait_for_async_ready is a blocking call which could take a long time, and in the meantime, the pythonic
+ // callback that was passed to run_async might be called. Therefore, the GIL before calling must be released.
+ // This is done in order to avoid any further deadlock. Any c++ function that is called from python and that might
+ // be still running while another thread is trying to execute a python callback, should be able to release the GIL.
+ .def("wait_for_async_ready", &ConfiguredInferModelWrapper::wait_for_async_ready, py::call_guard<py::gil_scoped_release>())
+ .def("run", &ConfiguredInferModelWrapper::run)
+ // run_async is not calling python callbacks*, so there is no need to keep the GIL while calling.
+ // Releasing the GIL before calling run_async will allow the callbacks already registered to be called.
+ // * callbacks will be called from another thread, and will acquire the GIL by themselves
+ .def("run_async", &ConfiguredInferModelWrapper::run_async, py::call_guard<py::gil_scoped_release>())
+ .def("set_scheduler_timeout", &ConfiguredInferModelWrapper::set_scheduler_timeout)
+ .def("set_scheduler_threshold", &ConfiguredInferModelWrapper::set_scheduler_threshold)
+ .def("set_scheduler_priority", &ConfiguredInferModelWrapper::set_scheduler_priority)
+ .def("get_async_queue_size", &ConfiguredInferModelWrapper::get_async_queue_size)
+ .def("shutdown", &ConfiguredInferModelWrapper::shutdown)
+ ;
+}
+
+void ConfiguredInferModelBindingsWrapper::bind(py::module &m)
+{
+ py::class_<
+ ConfiguredInferModelBindingsWrapper,
+ std::shared_ptr<ConfiguredInferModelBindingsWrapper>
+ >(m, "ConfiguredInferModelBindings")
+ .def("input", &ConfiguredInferModelBindingsWrapper::input)
+ .def("output", &ConfiguredInferModelBindingsWrapper::output)
+ ;
+}
+
+void ConfiguredInferModelBindingsInferStreamWrapper::bind(py::module &m)
+{
+ py::class_<
+ ConfiguredInferModelBindingsInferStreamWrapper,
+ std::shared_ptr<ConfiguredInferModelBindingsInferStreamWrapper>
+ >(m, "ConfiguredInferModelInferStream")
+ .def("set_buffer", &ConfiguredInferModelBindingsInferStreamWrapper::set_buffer)
+ ;
+}
+
+void InferModelInferStreamWrapper::bind(py::module &m)
+{
+ py::class_<
+ InferModelInferStreamWrapper,
+ std::shared_ptr<InferModelInferStreamWrapper>
+ >(m, "InferModelInferStream")
+ .def("name", &InferModelInferStreamWrapper::name)
+ .def("set_format_type", &InferModelInferStreamWrapper::set_format_type)
+ .def("set_format_order", &InferModelInferStreamWrapper::set_format_order)
+ .def("get_quant_infos", &InferModelInferStreamWrapper::get_quant_infos)
+ .def("shape", &InferModelInferStreamWrapper::shape)
+ .def("format", &InferModelInferStreamWrapper::format)
+ .def("is_nms", &InferModelInferStreamWrapper::is_nms)
+ .def("set_nms_score_threshold", &InferModelInferStreamWrapper::set_nms_score_threshold)
+ .def("set_nms_iou_threshold", &InferModelInferStreamWrapper::set_nms_iou_threshold)
+ .def("set_nms_max_proposals_per_class", &InferModelInferStreamWrapper::set_nms_max_proposals_per_class)
+ .def("set_nms_max_accumulated_mask_size", &InferModelInferStreamWrapper::set_nms_max_accumulated_mask_size)
+ ;
+}
+
+void AsyncInferJobWrapper::bind(py::module &m)
+{
+ py::class_<
+ AsyncInferJobWrapper, std::shared_ptr<AsyncInferJobWrapper>
+ >(m, "AsyncInferJob")
+ // wait is a blocking call that can take a long time, and in the meantime, the pythonic
+ // callback that was passed to run_async might be called. Therefore, we release the GIL before calling.
+ // This is done in order to avoid a deadlock. Any c++ function that is called from python and that might
+ // be still running while another thread is trying to execute a python callback should release the GIL.
+ .def("wait", &AsyncInferJobWrapper::wait, py::call_guard<py::gil_scoped_release>())
+ ;
+}
--- /dev/null
+/**
+ * Copyright (c) 2020-2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file infer_model_api.hpp
+ * @brief Defines binding to a infer model class family usage over Python.
+ **/
+
+#ifndef INFER_MODEL_API_HPP_
+#define INFER_MODEL_API_HPP_
+
+#include "hailo/hailort.h"
+#include "hailo/event.hpp"
+#include "hailo/infer_model.hpp"
+#include "utils.hpp"
+#include <functional>
+#include <pybind11/numpy.h>
+#include <thread>
+#include <vector>
+#include <queue>
+
+namespace hailort {
+
+class ConfiguredInferModelWrapper;
+class ConfiguredInferModelBindingsWrapper;
+class ConfiguredInferModelBindingsInferStreamWrapper;
+class InferModelInferStreamWrapper;
+class AsyncInferJobWrapper;
+
+using AsyncInferCallBack = std::function<void(const int)>;
+using AsyncInferCallBackAndStatus = std::pair<AsyncInferCallBack, AsyncInferCompletionInfo>;
+
+class InferModelWrapper final
+{
+public:
+ InferModelWrapper(std::shared_ptr<InferModel> infer_model, bool is_using_service) :
+ m_infer_model(std::move(infer_model)), m_is_using_service(is_using_service) {}
+ void set_batch_size(uint16_t batch_size);
+ void set_power_mode(hailo_power_mode_t power_mode);
+ void set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency);
+ ConfiguredInferModelWrapper configure();
+ std::vector<std::string> get_input_names();
+ std::vector<std::string> get_output_names();
+ std::vector<InferModelInferStreamWrapper> inputs();
+ std::vector<InferModelInferStreamWrapper> outputs();
+ InferModelInferStreamWrapper input(const std::string &name);
+ InferModelInferStreamWrapper output(const std::string &name);
+
+ static void bind(py::module &m);
+
+private:
+ std::shared_ptr<InferModel> m_infer_model;
+ bool m_is_using_service;
+};
+
+class ConfiguredInferModelWrapper final
+{
+public:
+ ConfiguredInferModelWrapper(ConfiguredInferModel &&configured_infer_model, bool is_using_service,
+ const std::vector<std::string> &output_names) :
+ m_configured_infer_model(std::move(configured_infer_model)),
+ m_callbacks_queue(std::make_shared<std::queue<AsyncInferCallBackAndStatus>>()),
+ m_is_alive(true),
+ m_is_using_service(is_using_service),
+ m_output_names(output_names)
+ {
+ }
+
+ ConfiguredInferModelWrapper(ConfiguredInferModelWrapper &&other) :
+ m_configured_infer_model(std::move(other.m_configured_infer_model)),
+ m_callbacks_queue(std::move(other.m_callbacks_queue)),
+ m_callbacks_thread(std::move(other.m_callbacks_thread)),
+ m_is_alive(true),
+ m_is_using_service(other.m_is_using_service),
+ m_output_names(other.m_output_names)
+ {
+ other.m_is_alive = false;
+ }
+
+ ~ConfiguredInferModelWrapper()
+ {
+ if (m_is_alive) {
+ py::gil_scoped_release release; // don't block the user-defined callbacks while join()-ing
+ m_configured_infer_model.shutdown();
+ m_is_alive = false; // signal the thread to stop
+ m_cv.notify_all(); // wake up the thread so it can return
+ if (m_callbacks_thread && m_callbacks_thread->joinable())
+ {
+ m_callbacks_thread->join();
+ }
+ }
+ }
+
+ ConfiguredInferModelBindingsWrapper create_bindings();
+ void activate();
+ void deactivate();
+ void wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count = 1);
+ void run(ConfiguredInferModelBindingsWrapper bindings, std::chrono::milliseconds timeout);
+ AsyncInferJobWrapper run_async(
+ std::vector<ConfiguredInferModelBindingsWrapper> &bindings,
+ AsyncInferCallBack pythonic_cb);
+ void set_scheduler_timeout(const std::chrono::milliseconds &timeout);
+ void set_scheduler_threshold(uint32_t threshold);
+ void set_scheduler_priority(uint8_t priority);
+ size_t get_async_queue_size();
+ void shutdown();
+
+ static void bind(py::module &m);
+
+private:
+ void execute_callbacks();
+
+ ConfiguredInferModel m_configured_infer_model;
+ std::mutex m_queue_mutex;
+ std::condition_variable m_cv;
+ std::shared_ptr<std::queue<AsyncInferCallBackAndStatus>> m_callbacks_queue;
+ std::shared_ptr<std::thread> m_callbacks_thread; // worker thread. Executes user defined (pythonic) callbacks
+ std::atomic_bool m_is_alive; // allow main thread to write, while worker thread is reading
+ bool m_is_using_service;
+ std::vector<std::string> m_output_names;
+};
+
+class ConfiguredInferModelBindingsWrapper final
+{
+public:
+ ConfiguredInferModelBindingsWrapper(ConfiguredInferModel::Bindings&& bindings, std::vector<std::string> output_names) :
+ m_bindings(std::move(bindings)),
+ m_output_names(output_names)
+ {}
+ ConfiguredInferModelBindingsInferStreamWrapper input(const std::string &name);
+ ConfiguredInferModelBindingsInferStreamWrapper output(const std::string &name);
+ ConfiguredInferModel::Bindings get() { return m_bindings; }
+
+ static void bind(py::module &m);
+
+private:
+ ConfiguredInferModel::Bindings m_bindings;
+ std::vector<std::string> m_output_names;
+};
+
+class ConfiguredInferModelBindingsInferStreamWrapper final
+{
+public:
+ ConfiguredInferModelBindingsInferStreamWrapper(ConfiguredInferModel::Bindings::InferStream&& infer_stream) :
+ m_infer_stream(std::move(infer_stream)) {}
+ void set_buffer(py::array buffer);
+
+ static void bind(py::module &m);
+
+private:
+ ConfiguredInferModel::Bindings::InferStream m_infer_stream;
+};
+
+class InferModelInferStreamWrapper final
+{
+public:
+ InferModelInferStreamWrapper(InferModel::InferStream&& infer_stream) :
+ m_infer_stream(std::move(infer_stream)) {}
+ const std::string name() const;
+ void set_format_type(hailo_format_type_t type);
+ void set_format_order(hailo_format_order_t order);
+ std::vector<hailo_quant_info_t> get_quant_infos() const;
+ std::vector<size_t> shape() const;
+ hailo_format_t format() const;
+ bool is_nms() const;
+ void set_nms_score_threshold(float32_t threshold);
+ void set_nms_iou_threshold(float32_t threshold);
+ void set_nms_max_proposals_per_class(uint32_t max_proposals_per_class);
+ void set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size);
+
+ static void bind(py::module &m);
+private:
+ InferModel::InferStream m_infer_stream;
+};
+
+class AsyncInferJobWrapper final
+{
+public:
+ AsyncInferJobWrapper(AsyncInferJob&& job, EventPtr is_callback_done) :
+ m_job(std::move(job)),
+ m_is_callback_done(is_callback_done)
+ {}
+ void wait(std::chrono::milliseconds timeout);
+
+ static void bind(py::module &m);
+
+private:
+ AsyncInferJob m_job;
+ EventPtr m_is_callback_done;
+};
+
+}
+
+#endif /* INFER_MODEL_API_HPP_ */
namespace hailort
{
-void ConfiguredNetworkGroupWrapper::add_to_python_module(py::module &m)
+void ConfiguredNetworkGroupWrapper::bind(py::module &m)
{
py::class_<ConfiguredNetworkGroupWrapper, ConfiguredNetworkGroupWrapperPtr>(m, "ConfiguredNetworkGroup")
.def("is_scheduled", &ConfiguredNetworkGroupWrapper::is_scheduled)
.def("set_scheduler_timeout", &ConfiguredNetworkGroupWrapper::set_scheduler_timeout)
.def("set_scheduler_threshold", &ConfiguredNetworkGroupWrapper::set_scheduler_threshold)
.def("set_scheduler_priority", &ConfiguredNetworkGroupWrapper::set_scheduler_priority)
+ .def("init_cache", &ConfiguredNetworkGroupWrapper::init_cache)
+ .def("get_cache_info", &ConfiguredNetworkGroupWrapper::get_cache_info)
+ .def("update_cache_offset", &ConfiguredNetworkGroupWrapper::update_cache_offset)
.def("get_networks_names", &ConfiguredNetworkGroupWrapper::get_networks_names)
.def("get_sorted_output_names", &ConfiguredNetworkGroupWrapper::get_sorted_output_names)
.def("get_input_vstream_infos", &ConfiguredNetworkGroupWrapper::get_input_vstream_infos)
m_activated_net_group.reset();
}
-void ActivatedAppContextManagerWrapper::add_to_python_module(py::module &m)
+void ActivatedAppContextManagerWrapper::bind(py::module &m)
{
py::class_<ActivatedAppContextManagerWrapper>(m, "ActivatedApp")
.def("__enter__", &ActivatedAppContextManagerWrapper::enter, py::return_value_policy::reference)
;
}
-void NetworkGroup_api_initialize_python_module(py::module &m)
-{
- ConfiguredNetworkGroupWrapper::add_to_python_module(m);
- ActivatedAppContextManagerWrapper::add_to_python_module(m);
-}
-
} /* namespace hailort */
const ActivatedNetworkGroup& enter();
void exit();
- static void add_to_python_module(py::module &m);
+ static void bind(py::module &m);
private:
std::unique_ptr<ActivatedNetworkGroup> m_activated_net_group;
ConfiguredNetworkGroup &m_net_group;
VALIDATE_STATUS(status);
}
+ void init_cache(uint32_t read_offset, int32_t write_offset_delta)
+ {
+ auto status = get().init_cache(read_offset, write_offset_delta);
+ VALIDATE_STATUS(status);
+ }
+
+ hailo_cache_info_t get_cache_info()
+ {
+ auto cache_info = get().get_cache_info();
+ VALIDATE_EXPECTED(cache_info);
+
+ return cache_info.release();
+ }
+
+ void update_cache_offset(int32_t offset_delta_bytes)
+ {
+ auto status = get().update_cache_offset(offset_delta_bytes);
+ VALIDATE_STATUS(status);
+ }
+
auto get_networks_names()
{
auto network_infos = get().get_network_infos();
return std::make_shared<ConfiguredNetworkGroupWrapper>(net_group.release(), store_guard_for_multi_process);
}
- static void add_to_python_module(py::module &m);
+ static void bind(py::module &m);
private:
// Normally, the ownership of the network group is the Device/VDevice objects. We keep weak_ptr
#include "hailo/hailort_defaults.hpp"
#include "hailo/network_rate_calculator.hpp"
+#include "infer_model_api.hpp"
#include "hef_api.hpp"
#include "vstream_api.hpp"
#include "vdevice_api.hpp"
return name.value();
}
- static void add_to_python_module(py::module &m)
+ static void bind(py::module &m)
{
py::class_<NetworkRateLimiter>(m, "NetworkRateLimiter")
.def("set_rate_limit", &NetworkRateLimiter::set_rate_limit)
.value("HAILO15H", HAILO_ARCH_HAILO15H)
.value("PLUTO", HAILO_ARCH_PLUTO)
.value("HAILO15M", HAILO_ARCH_HAILO15M)
+ .value("HAILO10H", HAILO_ARCH_HAILO10H)
;
/* TODO: SDK-15648 */
py::class_<VDeviceParamsWrapper>(m, "VDeviceParams")
.def(py::init<>())
- // Add device_ids
+ .def_property("device_ids",
+ [](const VDeviceParamsWrapper ¶ms) -> py::list {
+ py::list ids;
+ if (params.orig_params.device_ids != nullptr) {
+ for (size_t i = 0; i < params.orig_params.device_count; i++) {
+ ids.append(std::string(params.orig_params.device_ids[i].id));
+ }
+ }
+ return ids;
+ },
+ [](VDeviceParamsWrapper ¶ms, const py::list &device_ids) {
+ uint32_t count = static_cast<uint32_t>(py::len(device_ids));
+ params.ids.resize(count);
+ for (size_t i = 0; i < count; i++) {
+ std::string id_str = py::cast<std::string>(device_ids[i]);
+ auto expected_device_id = HailoRTCommon::to_device_id(id_str);
+ VALIDATE_EXPECTED(expected_device_id);
+ params.ids[i] = expected_device_id.release();
+ }
+ params.orig_params.device_ids = params.ids.data();
+ params.orig_params.device_count = count;
+ }
+ )
.def_property("device_count",
[](const VDeviceParamsWrapper& params) -> uint32_t {
return params.orig_params.device_count;
},
[](VDeviceParamsWrapper& params, hailo_scheduling_algorithm_t scheduling_algorithm) {
params.orig_params.scheduling_algorithm = scheduling_algorithm;
- params.orig_params.multi_process_service = (HAILO_SCHEDULING_ALGORITHM_NONE != scheduling_algorithm);
}
)
.def_property("group_id",
params.orig_params.group_id = params.group_id_str.c_str();
}
)
- .def_property_readonly("multi_process_service",
+ .def_property("multi_process_service",
[](const VDeviceParamsWrapper& params) -> bool {
return params.orig_params.multi_process_service;
+ },
+ [](VDeviceParamsWrapper& params, bool multi_process_service) {
+ params.orig_params.multi_process_service = multi_process_service;
}
)
.def_static("default", []() {
auto orig_params = HailoRTDefaults::get_vdevice_params();
orig_params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_NONE;
- VDeviceParamsWrapper params_wrapper{orig_params, ""};
+ VDeviceParamsWrapper params_wrapper{orig_params, "", {}};
return params_wrapper;
});
;
))
;
+ py::class_<hailo_cache_info_t>(m, "CacheInfo")
+ .def(py::init<>())
+ .def(py::init<const uint32_t, const uint32_t, const int32_t>())
+ .def_readwrite("cache_size", &hailo_cache_info_t::cache_size)
+ .def_readwrite("current_read_offset", &hailo_cache_info_t::current_read_offset)
+ .def_readwrite("write_offset_delta", &hailo_cache_info_t::write_offset_delta)
+ ;
+
py::class_<hailo_throttling_level_t>(m, "ThrottlingLevel", py::module_local())
.def_readonly("temperature_threshold", &hailo_throttling_level_t::temperature_threshold)
.def_readonly("hysteresis_temperature_threshold", &hailo_throttling_level_t::hysteresis_temperature_threshold)
.def_static("MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP", []() { return 1472;} )
;
- HefWrapper::initialize_python_module(m);
- VStream_api_initialize_python_module(m);
- VDevice_api_initialize_python_module(m);
- NetworkGroup_api_initialize_python_module(m);
- DeviceWrapper::add_to_python_module(m);
-
- NetworkRateLimiter::add_to_python_module(m);
+ ActivatedAppContextManagerWrapper::bind(m);
+ AsyncInferJobWrapper::bind(m);
+ ConfiguredInferModelBindingsInferStreamWrapper::bind(m);
+ ConfiguredInferModelBindingsWrapper::bind(m);
+ ConfiguredInferModelWrapper::bind(m);
+ ConfiguredNetworkGroupWrapper::bind(m);
+ DeviceWrapper::bind(m);
+ HefWrapper::bind(m);
+ InferModelInferStreamWrapper::bind(m);
+ InferModelWrapper::bind(m);
+ InferVStreamsWrapper::bind(m);
+ InputVStreamWrapper::bind(m);
+ InputVStreamsWrapper::bind(m);
+ NetworkRateLimiter::bind(m);
+ OutputVStreamWrapper::bind(m);
+ OutputVStreamsWrapper::bind(m);
+ VDeviceWrapper::bind(m);
std::stringstream version;
version << HAILORT_MAJOR_VERSION << "." << HAILORT_MINOR_VERSION << "." << HAILORT_REVISION_VERSION;
--- /dev/null
+/**
+ * Copyright (c) 2020-2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file infer_model_api.cpp
+ * @brief Defines binding to a infer model class family usage over Python.
+ **/
+#include "vdevice_api.hpp"
+#include "infer_model_api.hpp"
+
+using namespace hailort;
+
+InferModelWrapper VDeviceWrapper::create_infer_model_from_file(const std::string &hef_path, const std::string &network_name)
+{
+ auto infer_model = m_vdevice->create_infer_model(hef_path, network_name);
+ VALIDATE_EXPECTED(infer_model);
+
+ return InferModelWrapper(infer_model.release(), m_is_using_service);
+}
+
+InferModelWrapper VDeviceWrapper::create_infer_model_from_buffer(const py::bytes &buffer, const std::string &network_name)
+{
+ // there are 3 ways to get the buffer from python and convert it to MemoryView:
+ // 1. py::bytes -> std::string -> MemoryView
+ // 2. py::bytes -> py::buffer -> MemoryView (this is the one used here)
+ // 3. std::string -> MemoryView
+ //
+ // 1+3 are copying the data, while 2 isn't, resulting in 700X faster transfer between python and c++ (tested on yolov5s [~15MB])
+ py::buffer_info info(py::buffer(buffer).request());
+ MemoryView hef_buffer(MemoryView(info.ptr, static_cast<size_t>(info.size)));
+ auto infer_model = m_vdevice->create_infer_model(hef_buffer, network_name);
+ VALIDATE_EXPECTED(infer_model);
+
+ return InferModelWrapper(infer_model.release(), m_is_using_service);
+}
#ifndef VDEVICE_API_HPP_
#define VDEVICE_API_HPP_
+#include "hef_api.hpp"
#include "utils.hpp"
#include "network_group_api.hpp"
#include "hailo/hailort_common.hpp"
#include <iostream>
+#include <memory>
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
#include <pybind11/detail/common.h>
namespace hailort
{
+class InferModelWrapper;
+
struct VDeviceParamsWrapper {
hailo_vdevice_params_t orig_params;
std::string group_id_str;
+ std::vector<hailo_device_id_t> ids;
};
-
class VDeviceWrapper;
using VDeviceWrapperPtr = std::shared_ptr<VDeviceWrapper>;
std::cerr << "VDevice device_ids can be set in params or device_ids argument. Both parameters were passed to the c'tor";
throw HailoRTStatusException(std::to_string(HAILO_INVALID_OPERATION));
}
- auto modified_params = params;
- auto device_ids_vector = HailoRTCommon::to_device_ids_vector(device_ids);
- VALIDATE_EXPECTED(device_ids_vector);
- modified_params.orig_params.device_ids = device_ids_vector->data();
- return std::make_shared<VDeviceWrapper>(modified_params.orig_params);
+ if (!device_ids.empty()) {
+ return create_from_ids(device_ids);
+ }
+ return create(params);
}
static VDeviceWrapperPtr create_from_ids(const std::vector<std::string> &device_ids)
VALIDATE_EXPECTED(vdevice_expected);
m_vdevice = vdevice_expected.release();
+ m_is_using_service = params.multi_process_service;
};
py::list get_physical_devices_ids() const
m_vdevice.reset();
}
+ InferModelWrapper create_infer_model_from_file(const std::string &hef_path, const std::string &network_name);
+ InferModelWrapper create_infer_model_from_buffer(const py::bytes &buffer, const std::string &network_name);
+
+ static void bind(py::module &m)
+ {
+ py::class_<VDeviceWrapper, VDeviceWrapperPtr>(m, "VDevice")
+ .def("create", py::overload_cast<const hailo_vdevice_params_t&>(&VDeviceWrapper::create))
+ .def("create", py::overload_cast<const VDeviceParamsWrapper&>(&VDeviceWrapper::create))
+ .def("create", py::overload_cast<const VDeviceParamsWrapper&, const std::vector<std::string>&>(&VDeviceWrapper::create))
+ .def("create_from_ids", &VDeviceWrapper::create_from_ids)
+ .def("get_physical_devices_ids", &VDeviceWrapper::get_physical_devices_ids)
+ .def("configure", &VDeviceWrapper::configure)
+ .def("release", &VDeviceWrapper::release)
+ .def("create_infer_model_from_file", &VDeviceWrapper::create_infer_model_from_file)
+ .def("create_infer_model_from_buffer", &VDeviceWrapper::create_infer_model_from_buffer)
+ ;
+ }
+
private:
std::unique_ptr<VDevice> m_vdevice;
std::vector<ConfiguredNetworkGroupWrapperPtr> m_net_groups;
+ bool m_is_using_service;
#ifdef HAILO_IS_FORK_SUPPORTED
AtForkRegistry::AtForkGuard m_atfork_guard;
#endif
};
-void VDevice_api_initialize_python_module(py::module &m)
-{
- py::class_<VDeviceWrapper, VDeviceWrapperPtr>(m, "VDevice")
- .def("create", py::overload_cast<const hailo_vdevice_params_t&>(&VDeviceWrapper::create))
- .def("create", py::overload_cast<const VDeviceParamsWrapper&>(&VDeviceWrapper::create))
- .def("create", py::overload_cast<const VDeviceParamsWrapper&, const std::vector<std::string>&>(&VDeviceWrapper::create))
- .def("create_from_ids", &VDeviceWrapper::create_from_ids)
- .def("get_physical_devices_ids", &VDeviceWrapper::get_physical_devices_ids)
- .def("configure", &VDeviceWrapper::configure)
- .def("release", &VDeviceWrapper::release)
- ;
-}
-
} /* namespace hailort */
#endif /* VDEVICE_API_HPP_ */
namespace hailort
{
-void InputVStreamWrapper::add_to_python_module(py::module &m)
+void InputVStreamWrapper::bind(py::module &m)
{
py::class_<InputVStream, std::shared_ptr<InputVStream>>(m, "InputVStream")
.def("send", [](InputVStream &self, py::array data)
})
.def_property_readonly("shape", [](InputVStream &self)
{
- return *py::array::ShapeContainer(HailoRTBindingsCommon::get_pybind_shape(self.get_info(), self.get_user_buffer_format()));
+ auto shape = self.get_info().shape;
+ auto nms_shape = self.get_info().nms_shape;
+ auto format = self.get_user_buffer_format();
+ return *py::array::ShapeContainer(HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format));
})
;
}
VALIDATE_STATUS(status);
}
-void InputVStreamsWrapper::add_to_python_module(py::module &m)
+void InputVStreamsWrapper::bind(py::module &m)
{
py::class_<InputVStreamsWrapper, InputVStreamsWrapperPtr>(m, "InputVStreams")
.def(py::init(&InputVStreamsWrapper::create))
auto OutputVStreamWrapper::get_shape(OutputVStream &self)
{
- return *py::array::ShapeContainer(HailoRTBindingsCommon::get_pybind_shape(self.get_info(), self.get_user_buffer_format()));
+ auto shape = self.get_info().shape;
+ auto nms_shape = self.get_info().nms_shape;
+ auto format = self.get_user_buffer_format();
+ return *py::array::ShapeContainer(HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format));
}
-void OutputVStreamWrapper::add_to_python_module(py::module &m)
+void OutputVStreamWrapper::bind(py::module &m)
{
py::class_<OutputVStream, std::shared_ptr<OutputVStream>>(m, "OutputVStream")
.def("recv", [](OutputVStream &self)
}
}
-void OutputVStreamsWrapper::add_to_python_module(py::module &m)
+void OutputVStreamsWrapper::bind(py::module &m)
{
py::class_<OutputVStreamsWrapper, OutputVStreamsWrapperPtr>(m, "OutputVStreams")
.def(py::init(&OutputVStreamsWrapper::create))
{
auto input = m_infer_pipeline->get_input_by_name(stream_name);
if (HAILO_SUCCESS == input.status()) {
- return HailoRTBindingsCommon::get_pybind_shape(input->get().get_info(), input->get().get_user_buffer_format());
+ auto shape = input->get().get_info().shape;
+ auto nms_shape = input->get().get_info().nms_shape;
+ auto format = input->get().get_user_buffer_format();
+ return HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format);
}
auto output = m_infer_pipeline->get_output_by_name(stream_name);
if (HAILO_SUCCESS == output.status()) {
- return HailoRTBindingsCommon::get_pybind_shape(output->get().get_info(), output->get().get_user_buffer_format());
+ auto shape = output->get().get_info().shape;
+ auto nms_shape = output->get().get_info().nms_shape;
+ auto format = output->get().get_user_buffer_format();
+ return HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format);
}
std::cerr << "Stream " << stream_name << " not found";
}
}
-void InferVStreamsWrapper::add_to_python_module(py::module &m)
+void InferVStreamsWrapper::bind(py::module &m)
{
py::class_<InferVStreamsWrapper>(m, "InferVStreams")
.def(py::init(&InferVStreamsWrapper::create))
: m_infer_pipeline(std::move(infer_pipeline))
{}
-void VStream_api_initialize_python_module(py::module &m)
-{
- InputVStreamWrapper::add_to_python_module(m);
- InputVStreamsWrapper::add_to_python_module(m);
- OutputVStreamWrapper::add_to_python_module(m);
- OutputVStreamsWrapper::add_to_python_module(m);
- InferVStreamsWrapper::add_to_python_module(m);
-}
-
} /* namespace hailort */
class InputVStreamWrapper final
{
public:
- static void add_to_python_module(py::module &m);
+ static void bind(py::module &m);
};
void after_fork_in_parent();
void after_fork_in_child();
- static void add_to_python_module(py::module &m);
+ static void bind(py::module &m);
InputVStreamsWrapper(std::unordered_map<std::string, std::shared_ptr<InputVStream>> &input_vstreams);
static py::dtype get_dtype(OutputVStream &self);
static hailo_format_t get_user_buffer_format(OutputVStream &self);
static auto get_shape(OutputVStream &self);
- static void add_to_python_module(py::module &m);
+ static void bind(py::module &m);
};
class OutputVStreamsWrapper;
void before_fork();
void after_fork_in_parent();
void after_fork_in_child();
- static void add_to_python_module(py::module &m);
+ static void bind(py::module &m);
OutputVStreamsWrapper(std::unordered_map<std::string, std::shared_ptr<OutputVStream>> &output_vstreams);
void before_fork();
void after_fork_in_parent();
void after_fork_in_child();
- static void add_to_python_module(py::module &m);
+ static void bind(py::module &m);
private:
InferVStreamsWrapper(std::shared_ptr<InferVStreams> &infer_pipeline);
DIRECTORY "${DOXYGEN_OUTPUT_DIR}" DESTINATION "doc/"
CONFIGURATIONS Release)
- add_custom_target(doc DEPENDS doxygen)
+ add_custom_target(doc ALL DEPENDS doxygen)
endif()
\ No newline at end of file
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(data_quantization_example.c PROPERTIES LANGUAGE C)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(infer_pipeline_example.c PROPERTIES LANGUAGE C)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(multi_device_example.c PROPERTIES LANGUAGE C)
find_package(Threads REQUIRED)
set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(multi_network_vstream_example.c PROPERTIES LANGUAGE C)
(void)hailo_deactivate_network_group(activated_network_group);
l_release_vstreams:
for (size_t net_index = 0; net_index < NET_COUNT; net_index++) {
- if (NULL != output_vstreams[net_index]) {
+ if (NULL != output_vstreams[net_index][0]) {
(void)hailo_release_output_vstreams(output_vstreams[net_index], output_vstreams_size[net_index]);
}
}
for (size_t net_index = 0; net_index < NET_COUNT; net_index++) {
- if (NULL != input_vstreams[net_index]) {
+ if (NULL != input_vstreams[net_index][0]) {
(void)hailo_release_input_vstreams(input_vstreams[net_index], input_vstreams_size[net_index]);
}
}
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(notification_callback_example.c PROPERTIES LANGUAGE C)
cmake_minimum_required(VERSION 3.0.0)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(power_measurement_example.c PROPERTIES LANGUAGE C)
cmake_minimum_required(VERSION 3.0.0)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(raw_async_streams_single_thread_example.c PROPERTIES LANGUAGE C)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(raw_streams_example.c PROPERTIES LANGUAGE C)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(switch_network_groups_example.c PROPERTIES LANGUAGE C)
for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) {
for (size_t i = 0; i < num_input_vstreams[hef_index]; i++) {
- if (NULL != src_data[hef_index] && NULL != src_data[hef_index][i]) {
+ if (NULL != src_data[hef_index][i]) {
FREE(src_data[hef_index][i]);
}
}
for (size_t i = 0; i < num_output_vstreams[hef_index]; i++) {
- if (NULL != dst_data[hef_index] && NULL != dst_data[hef_index][i]) {
+ if (NULL != dst_data[hef_index][i]) {
FREE(dst_data[hef_index][i]);
}
}
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(switch_network_groups_manually_example.c PROPERTIES LANGUAGE C)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
SET_SOURCE_FILES_PROPERTIES(vstreams_example.c PROPERTIES LANGUAGE C)
cmake_minimum_required(VERSION 3.0.0)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_async_infer_advanced_example async_infer_advanced_example.cpp)
target_link_libraries(cpp_async_infer_advanced_example PRIVATE HailoRT::libhailort)
#include <sys/mman.h>
#endif
-#define FRAMES_COUNT (100)
+#define BATCH_COUNT (100)
+#define BATCH_SIZE (2)
using namespace hailort;
std::cerr << "Failed create vdevice, status = " << vdevice.status() << std::endl;
return vdevice.status();
}
+ std::cout << "VDevice created" << std::endl;
// Create infer model from HEF file.
auto infer_model_exp = vdevice.value()->create_infer_model("hefs/shortcut_net_nv12.hef");
std::cerr << "Failed to create infer model, status = " << infer_model_exp.status() << std::endl;
return infer_model_exp.status();
}
+ std::cout << "InferModel created" << std::endl;
auto infer_model = infer_model_exp.release();
infer_model->output()->set_format_type(HAILO_FORMAT_TYPE_FLOAT32);
+ std::cout << "Set output format_type to float32" << std::endl;
+ infer_model->set_batch_size(BATCH_SIZE);
+ std::cout << "Set batch_size to " << BATCH_SIZE << std::endl;
// Configure the infer model
auto configured_infer_model = infer_model->configure();
std::cerr << "Failed to create configured infer model, status = " << configured_infer_model.status() << std::endl;
return configured_infer_model.status();
}
+ std::cout << "ConfiguredInferModel created" << std::endl;
// The buffers are stored here as a guard for the memory. The buffer will be freed only after
// configured_infer_model will be released.
std::vector<std::shared_ptr<uint8_t>> buffer_guards;
- // Create infer bindings
- auto bindings = configured_infer_model->create_bindings();
- if (!bindings) {
- std::cerr << "Failed to create infer bindings, status = " << bindings.status() << std::endl;
- return bindings.status();
- }
-
- // Set the input buffers of the bindings.
+ // Create input buffers.
+ std::unordered_map<std::string, hailo_pix_buffer_t> input_buffers;
for (const auto &input_name : infer_model->get_input_names()) {
size_t input_frame_size = infer_model->input(input_name)->get_frame_size();
pix_buffer.planes[1].plane_size = UV_PLANE_SIZE;
pix_buffer.planes[1].user_ptr = reinterpret_cast<void*>(uv_plane_buffer.get());
- auto status = bindings->input(input_name)->set_pix_buffer(pix_buffer);
- if (HAILO_SUCCESS != status) {
- std::cerr << "Failed to set infer input buffer, status = " << status << std::endl;
- return status;
- }
-
+ input_buffers[input_name] = pix_buffer;
buffer_guards.push_back(y_plane_buffer);
buffer_guards.push_back(uv_plane_buffer);
}
- // Set the output buffers of the bindings.
+ // Create output buffers.
+ std::unordered_map<std::string, MemoryView> output_buffers;
for (const auto &output_name : infer_model->get_output_names()) {
size_t output_frame_size = infer_model->output(output_name)->get_frame_size();
auto output_buffer = page_aligned_alloc(output_frame_size);
- auto status = bindings->output(output_name)->set_buffer(MemoryView(output_buffer.get(), output_frame_size));
- if (HAILO_SUCCESS != status) {
- std::cerr << "Failed to set infer output buffer, status = " << status << std::endl;
- return status;
- }
+ output_buffers[output_name] = MemoryView(output_buffer.get(), output_frame_size);
buffer_guards.push_back(output_buffer);
}
+ std::cout << "Running inference..." << std::endl;
AsyncInferJob last_infer_job;
- for (uint32_t i = 0; i < FRAMES_COUNT; i++) {
+ for (uint32_t i = 0; i < BATCH_COUNT; i++) {
// Waiting for available requests in the pipeline
- auto status = configured_infer_model->wait_for_async_ready(std::chrono::milliseconds(1000));
+ auto status = configured_infer_model->wait_for_async_ready(std::chrono::milliseconds(1000), BATCH_SIZE);
if (HAILO_SUCCESS != status) {
std::cerr << "Failed to wait for async ready, status = " << status << std::endl;
return status;
}
- auto job = configured_infer_model->run_async(bindings.value(), [] (const AsyncInferCompletionInfo &/*completion_info*/) {
- // Use completion_info to get the job status and the corresponding bindings
+ // In this example we infer the same buffers, so setting 'BATCH_SIZE' identical bindings in the 'multiple_bindings' vector
+ std::vector<ConfiguredInferModel::Bindings> bindings_batch;
+ for (uint32_t b = 0; b < BATCH_SIZE; b++) {
+ auto bindings = configured_infer_model->create_bindings();
+ if (!bindings) {
+ std::cerr << "Failed to create infer bindings, status = " << bindings.status() << std::endl;
+ return bindings.status();
+ }
+
+ for (auto &input_buffer : input_buffers) {
+ status = bindings->input(input_buffer.first)->set_pix_buffer(input_buffer.second);
+ if (HAILO_SUCCESS != status) {
+ std::cerr << "Failed to set infer input buffer, status = " << status << std::endl;
+ return status;
+ }
+ }
+ for (auto &output_buffer : output_buffers) {
+ status = bindings->output(output_buffer.first)->set_buffer(output_buffer.second);
+ if (HAILO_SUCCESS != status) {
+ std::cerr << "Failed to set infer output buffer, status = " << status << std::endl;
+ return status;
+ }
+ }
+
+ bindings_batch.emplace_back(bindings.release());
+ }
+
+ auto job = configured_infer_model->run_async(bindings_batch, [] (const AsyncInferCompletionInfo &completion_info) {
+ // Use completion_info to get the async operation status
+ // Note that this callback must be executed as quickly as possible
+ (void)completion_info.status;
});
if (!job) {
std::cerr << "Failed to start async infer job, status = " << job.status() << std::endl;
// detach() is called in order for jobs to run in parallel (and not one after the other)
job->detach();
- if (i == FRAMES_COUNT - 1) {
+ if (i == BATCH_COUNT - 1) {
last_infer_job = job.release();
}
}
return status;
}
- std::cout << "Inference finished successfully" << std::endl;
+ std::cout << "Inference finished successfully on " << BATCH_COUNT * BATCH_SIZE << " frames" << std::endl;
return HAILO_SUCCESS;
}
cmake_minimum_required(VERSION 3.0.0)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_async_infer_basic_example async_infer_basic_example.cpp)
target_link_libraries(cpp_async_infer_basic_example PRIVATE HailoRT::libhailort)
std::cerr << "Failed create vdevice, status = " << vdevice.status() << std::endl;
return vdevice.status();
}
+ std::cout << "VDevice created" << std::endl;
// Create infer model from HEF file.
auto infer_model_exp = vdevice.value()->create_infer_model(HEF_FILE);
std::cerr << "Failed to create infer model, status = " << infer_model_exp.status() << std::endl;
return infer_model_exp.status();
}
+ std::cout << "InferModel created" << std::endl;
auto infer_model = infer_model_exp.release();
// Configure the infer model
std::cerr << "Failed to create configured infer model, status = " << configured_infer_model.status() << std::endl;
return configured_infer_model.status();
}
+ std::cout << "ConfiguredInferModel created" << std::endl;
// The buffers are stored here as a guard for the memory. The buffer will be freed only after
// configured_infer_model will be released.
buffer_guards.push_back(output_buffer);
}
+ std::cout << "ConfiguredInferModel::Bindings created and configured" << std::endl;
+ std::cout << "Running inference..." << std::endl;
// Run the async infer job.
auto job = configured_infer_model->run_async(bindings.value());
if (!job) {
cmake_minimum_required(VERSION 3.0.0)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_infer_pipeline_example infer_pipeline_example.cpp)
target_link_libraries(cpp_infer_pipeline_example PRIVATE HailoRT::libhailort)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_multi_device_example multi_device_example.cpp)
target_link_libraries(cpp_multi_device_example PRIVATE HailoRT::libhailort Threads::Threads)
find_package(Threads REQUIRED)
set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_multi_network_vstream_example multi_network_vstream_example.cpp)
target_link_libraries(cpp_multi_network_vstream_example PRIVATE HailoRT::libhailort Threads::Threads)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_multi_process_example multi_process_example.cpp)
target_link_libraries(cpp_multi_process_example PRIVATE HailoRT::libhailort Threads::Threads)
cmake_minimum_required(VERSION 3.0.0)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_notification_callback_example notification_callback_example.cpp)
target_link_libraries(cpp_notification_callback_example PRIVATE HailoRT::libhailort)
cmake_minimum_required(VERSION 3.0.0)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_power_measurement_example power_measurement_example.cpp)
target_link_libraries(cpp_power_measurement_example PRIVATE HailoRT::libhailort)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_raw_async_streams_multi_thread_example raw_async_streams_multi_thread_example.cpp)
target_link_libraries(cpp_raw_async_streams_multi_thread_example PRIVATE HailoRT::libhailort Threads::Threads)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_raw_async_streams_single_thread_example raw_async_streams_single_thread_example.cpp)
target_link_libraries(cpp_raw_async_streams_single_thread_example PRIVATE HailoRT::libhailort Threads::Threads)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_raw_streams_example raw_streams_example.cpp)
target_link_libraries(cpp_raw_streams_example PRIVATE HailoRT::libhailort Threads::Threads)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_switch_network_groups_example switch_network_groups_example.cpp)
target_link_libraries(cpp_switch_network_groups_example PRIVATE HailoRT::libhailort Threads::Threads)
find_package(Threads REQUIRED)
set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_switch_network_groups_manually_example switch_network_groups_manually_example.cpp)
target_link_libraries(cpp_switch_network_groups_manually_example PRIVATE HailoRT::libhailort Threads::Threads)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
-find_package(HailoRT 4.17.1 EXACT REQUIRED)
+find_package(HailoRT 4.18.0 EXACT REQUIRED)
add_executable(cpp_vstreams_example vstreams_example.cpp)
target_link_libraries(cpp_vstreams_example PRIVATE HailoRT::libhailort Threads::Threads)
PROTO__HW_ARCH__HAILO8L = 3;
PROTO__HW_ARCH__HAILO15H = 103;
PROTO__HW_ARCH__HAILO15M = 4;
+ PROTO__HW_ARCH__HAILO10H = 5;
// Reserving low numbers to public hw archs
PROTO__HW_ARCH__SAGE_A0 = 100;
ProtoHEFActionEnableNMS enable_nms = 13;
ProtoHEFActionWriteDataByType write_data_by_type = 14;
ProtoHEFActionSwitchLcuBatch switch_lcu_batch = 15;
+ ProtoHEFActionWriteDataCcwPtr write_data_ccw_ptr = 16;
}
}
+message ProtoHEFActionWriteDataCcwPtr {
+ // The offset which the data is in
+ uint64 offset = 1;
+
+ // The size of the data in bytes
+ uint32 size = 2;
+
+ // cfg_channel_index
+ uint32 cfg_channel_index = 3;
+}
+
message ProtoHEFActionWriteData {
// The address to write the data to
uint64 address = 1;
PROTO__EDGE_CONNECTION_TYPE__BOUNDARY = 0;
PROTO__EDGE_CONNECTION_TYPE__INTERMEDIATE = 1;
PROTO__EDGE_CONNECTION_TYPE__DDR = 2;
+ PROTO__EDGE_CONNECTION_TYPE__CACHE = 3;
}
message ProtoHEFContextSwitchInformation {
uint32 buffers = 6;
uint32 connected_sys_index = 7;
repeated ProtoHEFConnectedContextInfo connected_contexts = 8;
+ int32 cache_id = 9;
}
message ProtoHEFConnectedContextInfo {
*/
virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction);
+ /**
+ * Maps the dmabuf represented by the file descriptor @a dmabuf_fd for DMA transfers to/from this device, in the specified
+ * @a data_direction.
+ * DMA mapping of buffers in advance may improve the performance of async API. This improvement will become
+ * apparent when the buffer is reused multiple times across different async operations.
+ *
+ * For high level API (aka InferModel), buffers bound using ConfiguredInferModel::Bindings::InferStream::set_buffer
+ * can be mapped.
+ *
+ * For low level API (aka InputStream/OutputStream), buffers passed to InputStream::write_async and
+ * OutputStream::read_async can be mapped.
+ *
+ * @param[in] dmabuf_fd The file descriptor of the dmabuf to be mapped.
+ * @param[in] size The buffer's size in bytes.
+ * @param[in] direction The direction of the mapping. For input streams, use `HAILO_DMA_BUFFER_DIRECTION_H2D`
+ * and for output streams, use `HAILO_DMA_BUFFER_DIRECTION_D2H`.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
+ *
+ * @note The DMA mapping will be released upon calling dma_unmap() with @a dmabuf_fd, @a size and @a data_direction, or
+ * when the @a Device object is destroyed.
+ * @note This API is currently experimental.
+ */
+ virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction);
+
+ /**
+ * Un-maps a dmabuf buffer represented by the file descriptor @a dmabuf_fd for DMA transfers to/from this device, in the direction
+ * @a direction.
+ *
+ * @param[in] dmabuf_fd The file descriptor of the dmabuf to be un-mapped.
+ * @param[in] size The buffer's size in bytes.
+ * @param[in] direction The direction of the mapping.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
+ * @note This API is currently experimental.
+ */
+ virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction);
+
+
virtual hailo_status direct_write_memory(uint32_t address, const void *buffer, uint32_t size);
virtual hailo_status direct_read_memory(uint32_t address, void *buffer, uint32_t size);
hailo_status set_overcurrent_state(bool should_activate);
hailo_status continue_context_switch_breakpoint(uint8_t breakpoint_id);
hailo_status clear_context_switch_breakpoint(uint8_t breakpoint_id);
Expected<uint8_t> get_context_switch_breakpoint_status(uint8_t breakpoint_id);
+ hailo_status init_cache_info(const hailo_cache_info_t &cache_info);
+ Expected<hailo_cache_info_t> get_cache_info();
+ hailo_status update_cache_read_offset(int32_t read_offset_delta);
virtual ~Device() = default;
Device(const Device &) = delete;
HAILO_STATUS__X(59, HAILO_THREAD_NOT_ACTIVATED /*!< The given thread has not been activated */)\
HAILO_STATUS__X(60, HAILO_THREAD_NOT_JOINABLE /*!< The given thread is not joinable */)\
HAILO_STATUS__X(61, HAILO_NOT_FOUND /*!< Could not find element */)\
- HAILO_STATUS__X(62, HAILO_RESERVED_STATUS /*!< Reserved for future use */)\
+ HAILO_STATUS__X(62, HAILO_COMMUNICATION_CLOSED /*!< The communication between endpoints is closed */)\
HAILO_STATUS__X(63, HAILO_STREAM_ABORT /*!< Stream recv/send was aborted */)\
HAILO_STATUS__X(64, HAILO_PCIE_DRIVER_NOT_INSTALLED /*!< Pcie driver is not installed */)\
HAILO_STATUS__X(65, HAILO_NOT_AVAILABLE /*!< Component is not available */)\
HAILO_ARCH_HAILO15H,
HAILO_ARCH_PLUTO,
HAILO_ARCH_HAILO15M,
+ HAILO_ARCH_HAILO10H,
/** Max enum value to maintain ABI Integrity */
HAILO_ARCH_MAX_ENUM = HAILO_MAX_ENUM
// **************************************************************************************** //
/** Hailo buffer flags */
typedef enum {
- HAILO_BUFFER_FLAGS_NONE = 0, /*!< No flags - heap allocated buffer */
- HAILO_BUFFER_FLAGS_DMA = 1 << 0, /*!< Buffer is mapped to DMA (will be page aligned implicitly) */
+ HAILO_BUFFER_FLAGS_NONE = 0, /*!< No flags - heap allocated buffer */
+ HAILO_BUFFER_FLAGS_DMA = 1 << 0, /*!< Buffer is mapped to DMA (will be page aligned implicitly) */
+ HAILO_BUFFER_FLAGS_CONTINUOUS = 1 << 1, /*!< Buffer is physically continuous (will be page aligned implicitly) */
/** Max enum value to maintain ABI Integrity */
HAILO_BUFFER_FLAGS_MAX_ENUM = HAILO_MAX_ENUM
} hailo_latency_measurement_flags_t;
typedef struct {
- /** This parameter is only used in multi-context network_groups.
+ /**
+ * Sets the batch size of the InferModel.
+ * This parameter determines the number of frames that be sent for inference in a single batch.
+ * If a scheduler is enabled, this parameter determines the 'burst size' - the max number of frames after which the scheduler will attempt
+ * to switch to another model.
+ *
* User is advised to modify this (single network parameter) or @a hailo_configure_network_group_params_t batch size parameter. Not both.
* In case user wishes to work with the same batch size for all networks inside a network group, user is advised to set batch_size in @a hailo_configure_network_group_params_t.
- * In case user wished to work with batch size per network, user is advised to use this parameter. Default network batch size is @a HAILO_DEFAULT_BATCH_SIZE */
+ * In case user wished to work with batch size per network, user is advised to use this parameter.
+
+ * note: Default value is @a HAILO_DEFAULT_BATCH_SIZE - means automatic batch determined by hailort.
+ */
uint16_t batch_size;
} hailo_network_parameters_t;
char name[HAILO_MAX_NETWORK_NAME_SIZE];
} hailo_network_info_t;
+/** Notification IDs and structures section start */
+#pragma pack(push, 1)
+
/** Notification IDs, for each notification, one of the ::hailo_notification_message_parameters_t union will be set. */
typedef enum {
/** Matches hailo_notification_message_parameters_t::rx_error_notification. */
HAILO_NOTIFICATION_ID_HW_INFER_MANAGER_INFER_DONE,
/** Matches hailo_notification_message_parameters_t::context_switch_run_time_error */
HAILO_NOTIFICATION_ID_CONTEXT_SWITCH_RUN_TIME_ERROR_EVENT,
+ /** Matched hailo_notification_message_parameters_t::start_update_cache_offset_notification */
+ HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET,
/** Must be last! */
HAILO_NOTIFICATION_ID_COUNT,
uint32_t infer_cycles;
} hailo_hw_infer_manager_infer_done_notification_message_t;
+typedef struct {
+ uint64_t cache_id_bitmask;
+} hailo_start_update_cache_offset_notification_message_t;
+
typedef struct {
uint32_t exit_status;
uint8_t network_group_index;
hailo_hw_infer_manager_infer_done_notification_message_t hw_infer_manager_infer_done_notification;
/** context switch run time error event */
hailo_context_switch_run_time_error_message_t context_switch_run_time_error;
+ /** Start cache offset update notification */
+ hailo_start_update_cache_offset_notification_message_t start_update_cache_offset_notification;
} hailo_notification_message_parameters_t;
/** Notification data that will be passed to the callback passed in ::hailo_notification_callback */
hailo_notification_message_parameters_t body;
} hailo_notification_t;
+#pragma pack(pop)
+/** Notification IDs and structures section end */
+
/**
* A notification callback. See ::hailo_set_notification_callback
*
uint32_t rate;
} hailo_rate_limit_t;
+typedef struct {
+ uint32_t cache_size;
+ uint32_t current_read_offset;
+ int32_t write_offset_delta;
+} hailo_cache_info_t;
+
typedef enum {
HAILO_SENSOR_TYPES_GENERIC = 0,
HAILO_SENSOR_TYPES_ONSEMI_AR0220AT,
HAILORTAPI hailo_status hailo_shutdown_network_group(hailo_configured_network_group network_group);
/**
- * Activates hailo_device inner-resources for context_switch inference.
+ * Activates hailo_device inner-resources for inference.
*
* @param[in] network_group NetworkGroup to be activated.
* @param[in] activation_params Optional parameters for the activation (may be NULL).
hailo_activated_network_group *activated_network_group_out);
/**
- * De-activates hailo_device inner-resources for context_switch inference.
+ * De-activates hailo_device inner-resources for inference.
*
* @param[in] activated_network_group NetworkGroup to deactivate.
* @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
HAILORTAPI hailo_status hailo_vdevice_dma_unmap_buffer(hailo_vdevice vdevice, void *address, size_t size,
hailo_dma_buffer_direction_t direction);
+/**
+ * Maps the dmabuf represented by file descriptor @a dmabuf_fd for DMA transfers to/from the given @a device, in the specified
+ * @a data_direction.
+ * DMA mapping of buffers in advance may improve the performance of async API. This improvement will become
+ * apparent when the buffer is reused multiple times across different async operations.
+ * For low level API (aka ::hailo_input_stream or ::hailo_output_stream), buffers passed to
+ * ::hailo_stream_write_raw_buffer_async and ::hailo_stream_read_raw_buffer_async can be mapped.
+ *
+ * @param[in] device A ::hailo_device object.
+ * @param[in] dmabuf_fd The file decsriptor of the dmabuf to be mapped
+ * @param[in] size The buffer's size in bytes
+ * @param[in] direction The direction of the mapping. For input streams, use `HAILO_DMA_BUFFER_DIRECTION_H2D`
+ * and for output streams, use `HAILO_DMA_BUFFER_DIRECTION_D2H`.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
+ *
+ * @note The DMA mapping will be released upon calling ::hailo_device_dma_unmap_dmabuf with @a dmabuf_fd, @a size and
+ * @a data_direction, or when the @a device object is destroyed.
+ * @note The dmabuf pointed to by @a dmabuf_fd cannot be released until it is unmapped (via
+ * ::hailo_device_dma_map_dmabuf or ::hailo_release_device).
+ * @note This API is currently experimental.
+ */
+HAILORTAPI hailo_status hailo_device_dma_map_dmabuf(hailo_device device, int dmabuf_fd, size_t size,
+ hailo_dma_buffer_direction_t direction);
+
+/**
+ * Un-maps a dmabuf represented by file descriptor @a dmabuf_fd for DMA transfers to/from the given @a device, in the direction
+ * @a direction.
+ *
+ * @param[in] device A ::hailo_device object.
+ * @param[in] dmabuf_fd The file descriptor of the dmabuf to be un-mapped.
+ * @param[in] size The buffer's size in bytes.
+ * @param[in] direction The direction of the mapping.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
+ * @note This API is currently experimental.
+ */
+HAILORTAPI hailo_status hailo_device_dma_unmap_dmabuf(hailo_device device, int dmabuf_fd, size_t size,
+ hailo_dma_buffer_direction_t direction);
+
+/**
+ * Maps the dmabuf represented by the file descriptor @a dmabuf_fd for DMA transfers to/from the given @a vdevice, in the specified
+ * @a data_direction.
+ * DMA mapping of buffers in advance may improve the performance of async API. This improvement will become
+ * apparent when the buffer is reused multiple times across different async operations.
+ * For low level API (aka ::hailo_input_stream or ::hailo_output_stream), buffers passed to
+ * ::hailo_stream_write_raw_buffer_async and ::hailo_stream_read_raw_buffer_async can be mapped.
+ *
+ * @param[in] vdevice A ::hailo_vdevice object.
+ * @param[in] dmabuf_fd The file descriptor of the dmabuf to be mapped
+ * @param[in] size The buffer's size in bytes
+ * @param[in] direction The direction of the mapping. For input streams, use `HAILO_DMA_BUFFER_DIRECTION_H2D`
+ * and for output streams, use `HAILO_DMA_BUFFER_DIRECTION_D2H`.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
+ *
+ * @note The DMA mapping will be released upon calling ::hailo_vdevice_dma_unmap_dmabuf with @a dmabuf_fd, @a size and
+ * @a data_direction, or when the @a vdevice object is destroyed.
+ * @note The dmabuf pointed to by @a dmabuf_fd cannot be released until it is unmapped (via
+ * ::hailo_vdevice_dma_unmap_dmabuf or ::hailo_release_vdevice).
+ * @note This API is currently experimental.
+ */
+HAILORTAPI hailo_status hailo_vdevice_dma_map_dmabuf(hailo_vdevice vdevice, int dmabuf_fd, size_t size,
+ hailo_dma_buffer_direction_t direction);
+
+/**
+ * Un-maps a dmabuf pointed to by @a dmabuf_fd for DMA transfers to/from the given @a vdevice, in the direction
+ * @a direction.
+ *
+ * @param[in] vdevice A ::hailo_vdevice object.
+ * @param[in] dmabuf_fd The file descriptor of the dmabuf to be un-mapped.
+ * @param[in] size The buffer's size in bytes.
+ * @param[in] direction The direction of the mapping.
+ * @note This API is currently experimental.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
+ */
+HAILORTAPI hailo_status hailo_vdevice_dma_unmap_dmabuf(hailo_vdevice vdevice, int dmabuf_fd, size_t size,
+ hailo_dma_buffer_direction_t direction);
+
/** @} */ // end of group_buffer_functions
/** @defgroup group_stream_functions Stream functions
return "PLUTO";
case HAILO_ARCH_HAILO15M:
return "HAILO15M";
+ case HAILO_ARCH_HAILO10H:
+ return "HAILO10H";
default:
return "UNKNOWN ARCHITECTURE";
}
static bool is_hailo1x_device_type(const hailo_device_architecture_t dev_arch)
{
// Compare with HAILO1X device archs
- return (HAILO_ARCH_HAILO15H == dev_arch) || (HAILO_ARCH_HAILO15M == dev_arch) || (HAILO_ARCH_PLUTO == dev_arch);
+ return (HAILO_ARCH_HAILO15H == dev_arch) || (HAILO_ARCH_HAILO15M == dev_arch) || (HAILO_ARCH_PLUTO == dev_arch) ||
+ (HAILO_ARCH_HAILO10H == dev_arch);
}
static Expected<hailo_device_id_t> to_device_id(const std::string &device_id);
static Expected<std::vector<hailo_device_id_t>> to_device_ids_vector(const std::vector<std::string> &device_ids_str);
- static Expected<hailo_pix_buffer_t> as_hailo_pix_buffer(MemoryView &memory_view, hailo_format_order_t order);
+ static Expected<hailo_pix_buffer_t> as_hailo_pix_buffer(MemoryView memory_view, hailo_format_order_t order);
+
+ static bool is_power_measurement_supported(const hailo_device_architecture_t &hw_arch);
+ static bool is_current_measurement_supported(const hailo_device_architecture_t &hw_arch);
+ static bool is_temp_measurement_supported(const hailo_device_architecture_t &hw_arch);
};
#ifndef HAILO_EMULATOR
--- /dev/null
+/**
+ * Copyright (c) 2020-2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file hailort_dma-heap.h
+ * @brief DMABUF Heaps Userspace
+ **/
+
+#ifndef _HAILO_HAILORT_DMAHEAP_H
+#define _HAILO_HAILORT_DMAHEAP_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+/**
+ * DOC: DMABUF Heaps Userspace API
+ */
+
+/* Valid FD_FLAGS are O_CLOEXEC, O_RDONLY, O_WRONLY, O_RDWR */
+#define DMA_HEAP_VALID_FD_FLAGS (O_CLOEXEC | O_ACCMODE)
+
+/* Currently no heap flags */
+#define DMA_HEAP_VALID_HEAP_FLAGS (0)
+
+/**
+ * struct dma_heap_allocation_data - metadata passed from userspace for
+ * allocations
+ * @len: size of the allocation
+ * @fd: will be populated with a fd which provides the
+ * handle to the allocated dma-buf
+ * @fd_flags: file descriptor flags used when allocating
+ * @heap_flags: flags passed to heap
+ *
+ * Provided by userspace as an argument to the ioctl
+ */
+struct dma_heap_allocation_data {
+ __u64 len;
+ __u32 fd;
+ __u32 fd_flags;
+ __u64 heap_flags;
+};
+
+#define DMA_HEAP_IOC_MAGIC 'H'
+
+/**
+ * DOC: DMA_HEAP_IOCTL_ALLOC - allocate memory from pool
+ *
+ * Takes a dma_heap_allocation_data struct and returns it with the fd field
+ * populated with the dmabuf handle of the allocation.
+ */
+#define DMA_HEAP_IOCTL_ALLOC _IOWR(DMA_HEAP_IOC_MAGIC, 0x0,\
+ struct dma_heap_allocation_data)
+
+#endif /* _HAILO_HAILORT_DMAHEAP_H */
friend class ConfiguredNetworkGroupBase;
friend class CoreOp;
friend class VDeviceBase;
- friend class InferModel;
+ friend class InferModelBase;
#ifdef HAILO_SUPPORT_MULTI_PROCESS
friend class HailoRtRpcClient;
#include "hailo/hef.hpp"
#include "hailo/vdevice.hpp"
-#include <condition_variable>
-#include <mutex>
-
/** hailort namespace */
namespace hailort
{
-class ConfiguredInferModelImpl;
+class AsyncInferJobBase;
+class ConfiguredInferModelBase;
class AsyncInferRunnerImpl;
/*! Asynchronous inference job representation is used to manage and control an inference job that is running asynchronously. */
void detach();
private:
- friend class ConfiguredInferModelImpl;
+ friend class AsyncInferJobBase;
- class Impl;
- AsyncInferJob(std::shared_ptr<Impl> pimpl);
- std::shared_ptr<Impl> m_pimpl;
+ AsyncInferJob(std::shared_ptr<AsyncInferJobBase> pimpl);
+ std::shared_ptr<AsyncInferJobBase> m_pimpl;
bool m_should_wait_in_dtor;
};
/**
* Sets the edge's buffer to a new one, of type MemoryView.
*
+ * For best performance and to avoid memory copies, buffers should be aligned to the system PAGE_SIZE.
+ *
+ * On output streams, the actual buffer allocated size must be aligned to PAGE_SIZE as well - otherwise some
+ * memory corruption might occur at the end of the last page. For example, if the buffer size is 4000
+ * bytes, the actual buffer size should be at least 4096 bytes. To fill all requirements, it is recommended
+ * to allocate the buffer with standard page allocation function provided by the os (mmap on linux,
+ * VirtualAlloc in windows).
+ *
* @param[in] view The new buffer to be set.
* @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
*/
/**
* Sets the edge's buffer to a new one, of type hailo_pix_buffer_t.
*
+ * Each plane in the \ref hailo_pix_buffer_t must feet the requirements listed in \ref set_buffer.
+ *
* @param[in] pix_buffer The new buffer to be set.
* @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
* @note Supported only for inputs.
Expected<hailo_dma_buffer_t> get_dma_buffer();
private:
- friend class ConfiguredInferModelImpl;
+ friend class ConfiguredInferModelBase;
friend class AsyncInferRunnerImpl;
class Impl;
Expected<InferStream> output(const std::string &name);
private:
- friend class ConfiguredInferModelImpl;
+ friend class ConfiguredInferModelBase;
Bindings(std::unordered_map<std::string, InferStream> &&inputs,
std::unordered_map<std::string, InferStream> &&outputs);
* The readiness of the model is determined by the ability to push buffers to the asynchronous inference pipeline.
*
* @param[in] timeout Amount of time to wait until the model is ready in milliseconds.
+ * @param[in] frames_count The count of buffers you intent to infer in the next request. Useful for batch inference.
*
* @return Upon success, returns ::HAILO_SUCCESS. Otherwise:
* - If @a timeout has passed and the model is not ready, returns ::HAILO_TIMEOUT.
* - In any other error case, returns ::hailo_status error.
*/
- hailo_status wait_for_async_ready(std::chrono::milliseconds timeout);
+ hailo_status wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count = 1);
/**
- * Activates hailo device inner-resources for context_switch inference.
+ * Activates hailo device inner-resources for inference.
*
* @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns ::hailo_status error.
* @note Calling this function is invalid in case scheduler is enabled.
hailo_status activate();
/**
- * Deactivates hailo device inner-resources for context_switch inference.
+ * Deactivates hailo device inner-resources for inference.
+ * @return Returns ::HAILO_SUCCESS.
* @note Calling this function is invalid in case scheduler is enabled.
*/
- void deactivate();
+ hailo_status deactivate();
/**
* Launches a synchronous inference operation with the provided bindings.
* @param[in] callback The function to be called upon completion of the asynchronous inference operation.
*
* @return Upon success, returns an instance of Expected<AsyncInferJob> representing the launched job.
- * Otherwise, returns Unexpected of ::hailo_status error.
+ * Otherwise, returns Unexpected of ::hailo_status error, and the interface shuts down completly.
* @note @a callback should execute as quickly as possible.
+ * @note The bindings' buffers should be kept intact until the async job is completed
*/
Expected<AsyncInferJob> run_async(Bindings bindings,
std::function<void(const AsyncInferCompletionInfo &)> callback = ASYNC_INFER_EMPTY_CALLBACK);
+ /**
+ * Launches an asynchronous inference operation with the provided bindings.
+ * The completion of the operation is notified through the provided callback function.
+ * Overload for multiple-bindings inference (useful for batch inference).
+ *
+ * @param[in] bindings The bindings for the inputs and outputs of the model.
+ * @param[in] callback The function to be called upon completion of the asynchronous inference operation.
+ *
+ * @return Upon success, returns an instance of Expected<AsyncInferJob> representing the launched job.
+ * Otherwise, returns Unexpected of ::hailo_status error, and the interface shuts down completly.
+ * @note The bindings' buffers should be kept intact until the async job is completed
+ */
+ Expected<AsyncInferJob> run_async(const std::vector<Bindings> &bindings,
+ std::function<void(const AsyncInferCompletionInfo &)> callback = ASYNC_INFER_EMPTY_CALLBACK);
+
/**
* @return Upon success, returns Expected of LatencyMeasurementResult object containing the output latency result.
* Otherwise, returns Unexpected of ::hailo_status error.
/**
* Shuts the inference down. After calling this method, the model is no longer usable.
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error
*/
- void shutdown();
+ hailo_status shutdown();
private:
- friend class InferModel;
+ friend class InferModelBase;
+ friend class ConfiguredInferModelBase;
- ConfiguredInferModel(std::shared_ptr<ConfiguredInferModelImpl> pimpl);
+ ConfiguredInferModel(std::shared_ptr<ConfiguredInferModelBase> pimpl);
- std::shared_ptr<ConfiguredInferModelImpl> m_pimpl;
+ std::shared_ptr<ConfiguredInferModelBase> m_pimpl;
};
/**
* This class is used to set up the model for inference and includes methods for setting and getting the model's parameters.
* By calling the configure function, the user can create a ConfiguredInferModel object, which is used to run inference.
*/
-class HAILORTAPI InferModel final
+class HAILORTAPI InferModel
{
public:
- ~InferModel() = default;
+ virtual ~InferModel() = default;
/**
* Represents the parameters of a stream.
/**
* Set maximum accumulated mask size for all the detections in a frame.
*
- * Note: Used in order to change the output buffer frame size,
+ * @note: Used in order to change the output buffer frame size
* in cases where the output buffer is too small for all the segmentation detections.
*
* @param[in] max_accumulated_mask_size NMS max accumulated mask size.
void set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size);
private:
- friend class InferModel;
- friend class VDevice;
+ friend class InferModelBase;
+ friend class InferModelHrpcClient;
+
+ float32_t nms_score_threshold() const;
+ float32_t nms_iou_threshold() const;
+ uint32_t nms_max_proposals_per_class() const;
+ uint32_t nms_max_accumulated_mask_size() const;
class Impl;
InferStream(std::shared_ptr<Impl> pimpl);
/**
* @return A constant reference to the Hef object associated with this InferModel.
*/
- const Hef &hef() const;
+ virtual const Hef &hef() const = 0;
/**
* Sets the batch size of the InferModel.
+ * This parameter determines the number of frames that be sent for inference in a single batch.
+ * If a scheduler is enabled, this parameter determines the 'burst size' - the max number of frames after which the scheduler will attempt
+ * to switch to another model.
+ *
+ * note: Default value is HAILO_DEFAULT_BATCH_SIZE - means automatic batch determined by hailort.
*
* @param[in] batch_size The new batch size to be set.
*/
- void set_batch_size(uint16_t batch_size);
+ virtual void set_batch_size(uint16_t batch_size) = 0;
/**
* Sets the power mode of the InferModel.
*
* @param[in] power_mode The new power mode to be set.
*/
- void set_power_mode(hailo_power_mode_t power_mode);
+ virtual void set_power_mode(hailo_power_mode_t power_mode) = 0;
/**
* Sets the latency measurement flags of the InferModel.
*
* @param[in] latency The new latency measurement flags to be set.
*/
- void set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency);
+ virtual void set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency) = 0;
/**
* Configures the InferModel object. Also checks the validity of the configuration's formats.
* Otherwise, returns Unexpected of ::hailo_status error.
* @note InferModel can be configured once.
*/
- Expected<ConfiguredInferModel> configure();
+ virtual Expected<ConfiguredInferModel> configure() = 0;
/**
* Returns the single input's InferStream object.
* @note If InferModel has multiple inputs, will return ::HAILO_INVALID_OPERATION.
* In that case - use input(const std::string &name) instead.
*/
- Expected<InferStream> input();
+ virtual Expected<InferStream> input() = 0;
/**
* Returns the single output's InferStream object.
* @note If InferModel has multiple outputs, will return ::HAILO_INVALID_OPERATION.
* In that case - use output(const std::string &name) instead.
*/
- Expected<InferStream> output();
+ virtual Expected<InferStream> output() = 0;
/**
* Gets an input's InferStream object.
* @param[in] name The name of the input edge.
* @return Upon success, returns Expected of the relevant InferStream object. Otherwise, returns a ::hailo_status error.
*/
- Expected<InferStream> input(const std::string &name);
+ virtual Expected<InferStream> input(const std::string &name) = 0;
/**
* Gets an output's InferStream object.
* @param[in] name The name of the output edge.
* @return Upon success, returns Expected of the relevant InferStream object. Otherwise, returns a ::hailo_status error.
*/
- Expected<InferStream> output(const std::string &name);
+ virtual Expected<InferStream> output(const std::string &name) = 0;
/**
* @return A constant reference to the vector of input InferStream objects, each representing an input edge.
*/
- const std::vector<InferStream> &inputs() const;
+ virtual const std::vector<InferStream> &inputs() const = 0;
/**
* @return A constant reference to the vector of output InferStream objects, each representing an output edge.
*/
- const std::vector<InferStream> &outputs() const;
+ virtual const std::vector<InferStream> &outputs() const = 0;
/**
* @return A constant reference to a vector of strings, each representing the name of an input stream.
*/
- const std::vector<std::string> &get_input_names() const;
+ virtual const std::vector<std::string> &get_input_names() const = 0;
/**
* @return A constant reference to a vector of strings, each representing the name of an output stream.
*/
- const std::vector<std::string> &get_output_names() const;
-
- InferModel(InferModel &&);
+ virtual const std::vector<std::string> &get_output_names() const = 0;
- Expected<ConfiguredInferModel> configure_for_ut(std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
+
virtual Expected<ConfiguredInferModel> configure_for_ut(std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
- std::shared_ptr<ConfiguredNetworkGroup> net_group = nullptr);
-
-private:
- friend class VDevice;
-
- InferModel(VDevice &vdevice, Hef &&hef, std::unordered_map<std::string, InferStream> &&inputs,
- std::unordered_map<std::string, InferStream> &&outputs);
-
- std::reference_wrapper<VDevice> m_vdevice;
- Hef m_hef;
- std::unordered_map<std::string, InferStream> m_inputs;
- std::unordered_map<std::string, InferStream> m_outputs;
- std::vector<InferStream> m_inputs_vector;
- std::vector<InferStream> m_outputs_vector;
- std::vector<std::string> m_input_names;
- std::vector<std::string> m_output_names;
- ConfigureNetworkParams m_config_params;
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes = {},
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes = {},
+ std::shared_ptr<ConfiguredNetworkGroup> net_group = nullptr) = 0;
};
} /* namespace hailort */
using PostProcessOpMetadataPtr = std::shared_ptr<OpMetadata>;
}
-using NamedBuffersCallbacks = std::unordered_map<std::string, std::pair<MemoryView, std::function<void(hailo_status)>>>;
+enum class BufferType
+{
+ UNINITIALIZED,
+ VIEW,
+ PIX_BUFFER,
+ DMA_BUFFER,
+};
+
+struct BufferRepresentation {
+ BufferType buffer_type;
+ union {
+ MemoryView view;
+ hailo_dma_buffer_t dma_buffer;
+ };
+};
+
+using NamedBuffersCallbacks = std::unordered_map<std::string, std::pair<BufferRepresentation, std::function<void(hailo_status)>>>;
class InputVStream;
class OutputVStream;
const std::map<std::string, hailo_vstream_params_t> &outputs_params) = 0;
/**
- * Activates hailo device inner-resources for context_switch inference.
+ * Activates hailo device inner-resources for inference.
* @return Upon success, returns Expected of a pointer to ActivatedNetworkGroup object.
* Otherwise, returns Unexpected of ::hailo_status error.
Expected<std::unique_ptr<ActivatedNetworkGroup>> activate();
/**
- * Activates hailo device inner-resources for context_switch inference.
+ * Activates hailo device inner-resources for inference.
*
* @param[in] network_group_params Parameters for the activation.
* @return Upon success, returns Expected of a pointer to ActivatedNetworkGroup object.
virtual hailo_status set_nms_max_bboxes_per_class(const std::string &edge_name, uint32_t max_bboxes_per_class) = 0;
virtual hailo_status set_nms_max_accumulated_mask_size(const std::string &edge_name, uint32_t max_accumulated_mask_size) = 0;
+ virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) = 0;
+ virtual Expected<hailo_cache_info_t> get_cache_info() const = 0;
+ virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) = 0;
+
protected:
ConfiguredNetworkGroup();
*/
hailo_status status;
- const void *buffer_addr; /* Points to the transferred buffer. */
+ union {
+ const void *buffer_addr; /* Points to the transferred buffer. */
+ int dmabuf_fd; /* File descriptor to dmabuf*/
+ };
size_t buffer_size; /* Size of the transferred buffer. */
+
+ CompletionInfo(hailo_status status, const uint8_t *addr, size_t size):
+ status(status), buffer_addr(static_cast<const void*>(addr)), buffer_size(size) {}
+
+ CompletionInfo(hailo_status status, int fd, size_t size):
+ status(status), dmabuf_fd(fd), buffer_size(size) {}
};
/** Async transfer complete callback prototype. */
*/
virtual hailo_status write_async(const void *buffer, size_t size, const TransferDoneCallback &user_callback) = 0;
+ /**
+ * Writes the contents of the dmabuf associated with the fd @a dmabuf_fd to the stream asynchronously, initiating a deferred operation that will be
+ * completed later.
+ * - If the function call succeeds (i.e., write_async() returns ::HAILO_SUCCESS), the deferred operation has been
+ * initiated. Until @a user_callback is called, the user cannot change or delete @a dmabuf_fd.
+ * - If the function call fails (i.e., write_async() returns a status other than ::HAILO_SUCCESS), the deferred
+ * operation will not be initiated and @a user_callback will not be invoked. The user is free to change or delete
+ * @a dmabuf_fd.
+ * - @a user_callback is triggered upon successful completion or failure of the deferred operation. The callback
+ * receives a \ref CompletionInfo object containing a fd representing the transferred buffer (@a dmabuf_fd) and the
+ * transfer status (@a status).
+ *
+ * @param[in] dmabuf_fd The File descriptor to the dmabuf
+ * @param[in] size The size of the given buffer, expected to be get_frame_size().
+ * @param[in] user_callback The callback that will be called when the transfer is complete
+ * or has failed.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise:
+ * - If the stream queue is full, returns ::HAILO_QUEUE_IS_FULL. In this case please wait
+ * until @a user_callback is called on previous writes, or call wait_for_async_ready().
+ * The size of the queue can be determined by calling get_async_max_queue_size().
+ * - In any other error case, returns a ::hailo_status error.
+ *
+ * @note @a user_callback should run as quickly as possible.
+ * @note The dmabuf fd must be a linux-system dmabuf fd - otherwise behavior will fail.
+ * @note This API of write_async is currently experimental.
+ **/
+ virtual hailo_status write_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) = 0;
+
/**
* @returns A ::hailo_stream_info_t object containing the stream's info.
*/
/** Context passed to the \ref TransferDoneCallback after the async operation is done or has failed. */
struct CompletionInfo
{
+ public:
/**
* Status of the async transfer.
* - ::HAILO_SUCCESS - When transfer is complete successfully.
*/
hailo_status status;
- void *buffer_addr; /* Points to the transferred buffer. */
+ union {
+ void *buffer_addr; /* Points to the transferred buffer. */
+ int dmabuf_fd; /* File descriptor to dmabuf*/
+ };
+
size_t buffer_size; /* Size of the transferred buffer. */
+
+ CompletionInfo(hailo_status status, uint8_t *addr, size_t size):
+ status(status), buffer_addr(static_cast<void*>(addr)), buffer_size(size) {}
+
+ CompletionInfo(hailo_status status, int fd, size_t size):
+ status(status), dmabuf_fd(fd), buffer_size(size) {}
};
/** Async transfer complete callback prototype. */
*/
virtual hailo_status read_async(void *buffer, size_t size, const TransferDoneCallback &user_callback) = 0;
+ /**
+ * Reads into dmabuf represented by fd @a dmabuf_fd from the stream asynchronously, initiating a deferred operation that will be completed
+ * later.
+ * - If the function call succeeds (i.e., read_async() returns ::HAILO_SUCCESS), the deferred operation has been
+ * initiated. Until @a user_callback is called, the user cannot change or delete @a dmabuf_fd.
+ * - If the function call fails (i.e., read_async() returns a status other than ::HAILO_SUCCESS), the deferred
+ * operation will not be initiated and @a user_callback will not be invoked. The user is free to change or
+ * delete @a dmabuf_fd.
+ * - @a user_callback is triggered upon successful completion or failure of the deferred operation.
+ * The callback receives a \ref CompletionInfo object containing a fd representing the transferred buffer
+ * (@a dmabuf_fd) and the transfer status (@a status). If the operation has completed successfully, the contents
+ * of the dmabuf will have been updated by the read operation.
+ *
+ * @param[in] dmabuf_fd The File descriptor to the dmabuf
+ * @param[in] size The size of the given buffer, expected to be get_frame_size().
+ * @param[in] user_callback The callback that will be called when the transfer is complete or has failed.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise:
+ * - If the stream queue is full, returns ::HAILO_QUEUE_IS_FULL.
+ * In this case, please wait until @a user_callback is called on previous
+ * reads, or call wait_for_async_ready(). The size of the queue can be
+ * determined by calling get_async_max_queue_size().
+ * - In any other error case, returns a ::hailo_status error.
+ * @note @a user_callback should execute as quickly as possible.
+ * @note The dmabuf fd must be a linux-system dmabuf fd - otherwise behavior will fail.
+ * @note This API of read_async is currently experimental.
+ */
+ virtual hailo_status read_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) = 0;
+
// get_network_group_activated_event is same as this function
virtual EventPtr &get_core_op_activated_event() = 0;
protected:
* @param[in] network_name A string of the network name (optional).
* @return Upon success, returns Expected of a shared pointer of infer model.
* Otherwise, returns Unexpected of ::hailo_status error.
+ * @note the Hef file must be maintained until the completion of the configuration phase.
*/
virtual Expected<std::shared_ptr<InferModel>> create_infer_model(const std::string &hef_path,
const std::string &network_name = "");
+ /**
+ * Creates the infer model from an hef buffer
+ *
+ * @param[in] hef_buffer A pointer to a buffer containing the hef file.
+ * @param[in] network_name A string of the network name (optional).
+ * @return Upon success, returns Expected of a shared pointer of infer model.
+ * Otherwise, returns Unexpected of ::hailo_status error.
+ * @note the Hef buffer must be maintained until the completion of the configuration phase.
+ */
+ virtual Expected<std::shared_ptr<InferModel>> create_infer_model(const MemoryView hef_buffer,
+ const std::string &network_name = "");
+
/**
* Gets the underlying physical devices.
*
*/
virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) = 0;
+ /**
+ * Maps the dmabuf represented by the file descriptor @a dmabuf_fd for DMA transfers to/from this vdevice, in the specified
+ * @a data_direction.
+ * DMA mapping of buffers in advance may improve the performance of async API. This improvement will become
+ * apparent when the buffer is reused multiple times across different async operations.
+ *
+ * For high level API (aka InferModel), buffers bound using ConfiguredInferModel::Bindings::InferStream::set_buffer
+ * can be mapped.
+ *
+ * For low level API (aka InputStream/OutputStream), buffers passed to InputStream::write_async and
+ * OutputStream::read_async can be mapped.
+ *
+ * @param[in] dmabuf_fd The file descriptor of the dmabuf to be mapped.
+ * @param[in] size The buffer's size in bytes.
+ * @param[in] direction The direction of the mapping. For input streams, use `HAILO_DMA_BUFFER_DIRECTION_H2D`
+ * and for output streams, use `HAILO_DMA_BUFFER_DIRECTION_D2H`.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
+ *
+ * @note The DMA mapping will be released upon calling dma_unmap_dmabuf() with @a dmabuf_fd, @a size and @a data_direction, or
+ * when the @a VDevice object is destroyed.
+ */
+ virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) = 0;
+
+ /**
+ * Un-maps a dmabuf buffer represented by the file descriptor @a dmabuf_fd for DMA transfers to/from this vdevice, in the direction
+ * @a direction.
+ *
+ * @param[in] dmabuf_fd The file descriptor of the dmabuf to be un-mapped.
+ * @param[in] size The buffer's size in bytes.
+ * @param[in] direction The direction of the mapping.
+ *
+ * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error.
+ */
+ virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) = 0;
+
virtual hailo_status before_fork();
virtual hailo_status after_fork_in_parent();
virtual hailo_status after_fork_in_child();
VDevice &operator=(VDevice &&other) = delete;
static bool service_over_ip_mode();
+ static bool force_hrpc_client();
protected:
VDevice() = default;
relative_to_absolute_paths(COMMON_C_SOURCES ${COMMON_C_SOURCES})
relative_to_absolute_paths(HAILO_OS_DIR ${HAILO_OS_DIR})
relative_to_absolute_paths(HAILO_FULL_OS_DIR ${HAILO_FULL_OS_DIR})
+relative_to_absolute_paths(DRIVER_OS_DIR ${DRIVER_OS_DIR})
relative_to_absolute_paths(HAILO_DRIVER_SRC_FILES ${HAILO_DRIVER_SRC_FILES})
set(HAILO_OS_DIR ${HAILO_OS_DIR} CACHE INTERNAL "Absolute path of os-dir")
set(HAILO_FULL_OS_DIR ${HAILO_FULL_OS_DIR} CACHE INTERNAL "Absolute Full path of os-dir")
+set(DRIVER_OS_DIR ${DRIVER_OS_DIR} CACHE INTERNAL "Absolute Full path of driver os-dir")
set(HAILO_DRIVER_SRC_FILES ${HAILO_DRIVER_SRC_FILES} CACHE INTERNAL "Absolute Full path of driver src files")
set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} CACHE INTERNAL "Absolute paths of hailort's cpp source files")
set(COMMON_C_SOURCES ${COMMON_C_SOURCES} CACHE INTERNAL "Absolute paths of common source files")
-set(HAILORT_SRCS_ABS ${HAILORT_CPP_SOURCES} ${HAILORT_COMMON_CPP_SOURCES} ${COMMON_C_SOURCES} CACHE INTERNAL "All absolute paths of hailort's source files")
+set(HAILORT_SRCS_ABS ${HAILORT_CPP_SOURCES} ${HAILORT_COMMON_CPP_SOURCES} ${COMMON_C_SOURCES} ${HRPC_CPP_SOURCES} ${HRPC_PROTOCOL_CPP_SOURCES} CACHE INTERNAL "All absolute paths of hailort's source files")
SET_SOURCE_FILES_PROPERTIES(${C_SOURCES} PROPERTIES LANGUAGE CXX)
add_library(libhailort SHARED ${HAILORT_SRCS_ABS})
target_link_libraries(libhailort PRIVATE spdlog::spdlog)
target_link_libraries(libhailort PRIVATE readerwriterqueue)
target_link_libraries(libhailort PRIVATE Eigen3::Eigen)
+target_link_libraries(libhailort PRIVATE rpc_proto)
if(HAILO_BUILD_SERVICE)
target_link_libraries(libhailort PRIVATE grpc++_unsecure)
target_link_libraries(libhailort PRIVATE hailort_rpc_grpc_proto)
$<BUILD_INTERFACE:${COMMON_INC_DIR}>
$<BUILD_INTERFACE:${DRIVER_INC_DIR}>
$<BUILD_INTERFACE:${RPC_DIR}>
+ $<BUILD_INTERFACE:${HRPC_DIR}>
)
target_compile_definitions(libhailort PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/resource_manager_builder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/config_buffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/intermediate_buffer.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/cache_buffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/channel_allocator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/action_list_buffer_builder/control_action_list_buffer_builder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/periph_calculator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/internal_buffer_manager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/internal_buffer_planner.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/cache_manager.cpp
)
set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE)
CoreOp::CoreOp(
const ConfigureNetworkParams &config_params, std::shared_ptr<CoreOpMetadata> metadata,
- ActiveCoreOpHolder &active_core_op_holder, hailo_status &status) :
+ ActiveCoreOpHolder &active_core_op_holder, hailo_status &status, bool is_scheduled) :
m_config_params(config_params),
m_active_core_op_holder(active_core_op_holder),
m_min_configured_batch_size(get_smallest_configured_batch_size(config_params)),
m_metadata(metadata),
m_vdevice_core_op_handle(INVALID_CORE_OP_HANDLE)
{
- auto event = Event::create_shared(Event::State::not_signalled);
- if (!event) {
- LOGGER__ERROR("Failed to create activation event");
- status = event.status();
- return;
+ if (!is_scheduled) {
+ auto event = Event::create_shared(Event::State::not_signalled);
+ if (!event) {
+ LOGGER__ERROR("Failed to create activation event");
+ status = event.status();
+ return;
+ }
+ m_core_op_activated_event = event.release();
}
- m_core_op_activated_event = event.release();
m_activation_time_accumulator = make_shared_nothrow<FullAccumulator<double>>("activation_time");
if (nullptr == m_activation_time_accumulator) {
Expected<std::chrono::nanoseconds> get_latency(LatencyMeterPtr &latency_meter, bool clear)
{
- auto hw_latency = latency_meter->get_latency(clear);
- if (HAILO_NOT_AVAILABLE == hw_latency.status()) {
- return make_unexpected(HAILO_NOT_AVAILABLE);
- }
- CHECK_EXPECTED(hw_latency, "Failed getting latency");
- return hw_latency.release();
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_NOT_AVAILABLE, auto hw_latency,
+ latency_meter->get_latency(clear), "Failed getting latency");
+
+ return hw_latency;
}
/* Network group base functions */
bool clear = ((m_config_params.latency & HAILO_LATENCY_CLEAR_AFTER_GET) == HAILO_LATENCY_CLEAR_AFTER_GET);
LatencyMeasurementResult result = {};
- auto latency_meters_exp = get_latency_meters();
- CHECK_EXPECTED(latency_meters_exp);
- auto latency_meters = latency_meters_exp.release();
+ TRY(auto latency_meters, get_latency_meters());
if (network_name.empty()) {
if (1 != m_input_streams.size()) {
if (HAILO_NOT_AVAILABLE == hw_latency.status()) {
continue;
}
- CHECK_EXPECTED(hw_latency);
+ CHECK_EXPECTED(hw_latency); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
latency_sum += hw_latency.value();
measurements_count++;
}
LOGGER__DEBUG("No latency measurements was found for network {}", network_name);
return make_unexpected(HAILO_NOT_FOUND);
}
- auto hw_latency = get_latency(latency_meters->at(network_name), clear);
- if (HAILO_NOT_AVAILABLE == hw_latency.status()) {
- return make_unexpected(HAILO_NOT_AVAILABLE);
- }
- CHECK_EXPECTED(hw_latency);
- result.avg_hw_latency = hw_latency.value();
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_NOT_AVAILABLE, const auto hw_latency,
+ get_latency(latency_meters->at(network_name), clear));
+
+ result.avg_hw_latency = hw_latency;
}
return result;
}
CHECK(!is_scheduled(), HAILO_INVALID_OPERATION,
"Manually deactivate a core-op is not allowed when the core-op scheduler is active!");
- auto core_op_ref = m_active_core_op_holder.get();
- CHECK_EXPECTED_AS_STATUS(core_op_ref, "Trying to deactivate while no network is running");
+ TRY(auto core_op_ref, m_active_core_op_holder.get(), "Trying to deactivate while no network is running");
- CHECK(this == std::addressof(core_op_ref->get()), HAILO_INTERNAL_FAILURE,
+ CHECK(this == std::addressof(core_op_ref.get()), HAILO_INTERNAL_FAILURE,
"Trying to deactivate different core-op");
m_active_core_op_holder.clear();
return make_unexpected(HAILO_NOT_SUPPORTED);
}
+Expected<Buffer> CoreOp::get_cache_buffer(uint32_t)
+{
+ LOGGER__ERROR("Getting cache buffer is not supported for this core op");
+ return make_unexpected(HAILO_NOT_SUPPORTED);
+}
+
+Expected<std::map<uint32_t, Buffer>> CoreOp::get_cache_buffers()
+{
+ LOGGER__ERROR("Getting cache buffers is not supported for this core op");
+ return make_unexpected(HAILO_NOT_SUPPORTED);
+}
+
hailo_status CoreOp::wrap_streams_for_remote_process()
{
for (auto &input_stream_pair : m_input_streams) {
auto base_stream = input_stream_pair.second;
-
- auto remote_proc_stream = RemoteProcessInputStream::create(base_stream);
- CHECK_EXPECTED_AS_STATUS(remote_proc_stream);
-
- input_stream_pair.second = remote_proc_stream.release();
+ TRY(input_stream_pair.second, RemoteProcessInputStream::create(base_stream));
}
for (auto &output_stream_pair : m_output_streams) {
auto base_stream = output_stream_pair.second;
-
- auto remote_proc_stream = RemoteProcessOutputStream::create(base_stream);
- CHECK_EXPECTED_AS_STATUS(remote_proc_stream);
-
- output_stream_pair.second = remote_proc_stream.release();
+ TRY(output_stream_pair.second, RemoteProcessOutputStream::create(base_stream));
}
return HAILO_SUCCESS;
size_t queue_size = std::numeric_limits<size_t>::max();
for (const auto &input : m_input_streams) {
- auto stream_queue_size = input.second->get_async_max_queue_size();
- CHECK_EXPECTED(stream_queue_size);
- queue_size = std::min(queue_size, *stream_queue_size);
+ TRY(auto stream_queue_size, input.second->get_async_max_queue_size());
+ queue_size = std::min(queue_size, stream_queue_size);
}
for (const auto &output : m_output_streams) {
- auto stream_queue_size = output.second->get_async_max_queue_size();
- CHECK_EXPECTED(stream_queue_size);
- queue_size = std::min(queue_size, *stream_queue_size);
+ TRY(auto stream_queue_size, output.second->get_async_max_queue_size());
+ queue_size = std::min(queue_size, stream_queue_size);
}
return queue_size;
Expected<std::shared_ptr<InputStreamBase>> CoreOp::create_input_stream_from_config_params(Device &device,
const hailo_stream_parameters_t &stream_params, const std::string &stream_name)
{
- auto layer_info = get_layer_info(stream_name);
- CHECK_EXPECTED(layer_info);
-
+ TRY(const auto layer_info, get_layer_info(stream_name));
CHECK_AS_EXPECTED(device.is_stream_interface_supported(stream_params.stream_interface), HAILO_INVALID_OPERATION,
"Device does not supports the given stream interface streams. Please update input_stream_params for stream {}.",
stream_name);
// Fallthrough
case HAILO_STREAM_INTERFACE_INTEGRATED:
{
- auto input_stream_exp = create_vdma_input_stream(device, stream_name, layer_info.value(), stream_params);
- CHECK_EXPECTED(input_stream_exp);
- input_stream = input_stream_exp.release();
+ TRY(input_stream, create_vdma_input_stream(device, stream_name, layer_info, stream_params));
break;
}
case HAILO_STREAM_INTERFACE_ETH:
{
- auto input_stream_exp = EthernetInputStream::create(device,
- layer_info.value(), stream_params.eth_input_params, m_core_op_activated_event);
- CHECK_EXPECTED(input_stream_exp);
- input_stream = input_stream_exp.release();
+ TRY(input_stream, EthernetInputStream::create(device,
+ layer_info, stream_params.eth_input_params, m_core_op_activated_event));
break;
}
case HAILO_STREAM_INTERFACE_MIPI:
{
- auto input_stream_exp = MipiInputStream::create(device,
- layer_info.value(), stream_params.mipi_input_params, m_core_op_activated_event);
- CHECK_EXPECTED(input_stream_exp);
- input_stream = input_stream_exp.release();
+ TRY(input_stream, MipiInputStream::create(device,
+ layer_info, stream_params.mipi_input_params, m_core_op_activated_event));
break;
}
HAILO_INTERNAL_FAILURE, "Invalid device type");
VdmaDevice *vdma_device = reinterpret_cast<VdmaDevice*>(&device);
- auto vdma_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(stream_name);
- CHECK_EXPECTED(vdma_channel_ptr_exp, "Failed to get vdma channel for output stream {}", stream_name);
+ TRY(auto vdma_channel, get_boundary_vdma_channel_by_stream_name(stream_name),
+ "Failed to get vdma channel for output stream {}", stream_name);
- return VdmaInputStream::create(stream_params.stream_interface, *vdma_device, vdma_channel_ptr_exp.value(),
+ return VdmaInputStream::create(stream_params.stream_interface, *vdma_device, vdma_channel,
layer_info, m_core_op_activated_event);
}
Expected<std::shared_ptr<OutputStreamBase>> CoreOp::create_output_stream_from_config_params(Device &device,
const hailo_stream_parameters_t &stream_params, const std::string &stream_name)
{
- auto layer_info = get_layer_info(stream_name);
- CHECK_EXPECTED(layer_info);
-
+ TRY(const auto layer_info, get_layer_info(stream_name));
CHECK_AS_EXPECTED(device.is_stream_interface_supported(stream_params.stream_interface), HAILO_INVALID_OPERATION,
"Device does not supports the given stream interface streams. Please update input_stream_params for stream {}.",
stream_name);
// Fallthrough
case HAILO_STREAM_INTERFACE_INTEGRATED:
{
- auto output_stream_exp = create_vdma_output_stream(device, stream_name, layer_info.value(), stream_params);
- CHECK_EXPECTED(output_stream_exp);
- output_stream = output_stream_exp.release();
+ TRY(output_stream, create_vdma_output_stream(device, stream_name, layer_info, stream_params));
break;
}
case HAILO_STREAM_INTERFACE_ETH:
{
- auto output_stream_exp = EthernetOutputStream::create(device,
- layer_info.value(), stream_params.eth_output_params,
- m_core_op_activated_event);
- CHECK_EXPECTED(output_stream_exp);
- output_stream = output_stream_exp.release();
+ TRY(output_stream, EthernetOutputStream::create(device,
+ layer_info, stream_params.eth_output_params, m_core_op_activated_event));
break;
}
return make_unexpected(HAILO_NOT_IMPLEMENTED);
}
- if (HAILO_FORMAT_ORDER_HAILO_NMS == layer_info->format.order) {
+ if (HAILO_FORMAT_ORDER_HAILO_NMS == layer_info.format.order) {
// In NMS we create some new stream object that wraps the original stream (and converts
// bbox/burst reads into frame reads).
// After HRT-10553 is implemented, we won't need this wrapper anymore.
const auto batch_size = get_smallest_configured_batch_size(m_config_params);
const auto max_queue_size = batch_size * MAX_ACTIVE_TRANSFERS_SCALE;
- auto nms_stream = NmsOutputStream::create(base_stream, layer_info.value(), max_queue_size,
- m_core_op_activated_event, stream_params.stream_interface);
- CHECK_EXPECTED(nms_stream);
- output_stream = nms_stream.release();
+ TRY(output_stream, NmsOutputStream::create(base_stream, layer_info, max_queue_size,
+ m_core_op_activated_event, stream_params.stream_interface));
}
return output_stream;
HAILO_INTERNAL_FAILURE, "Invalid device type");
VdmaDevice *vdma_device = reinterpret_cast<VdmaDevice*>(&device);
- auto batch_size_exp = get_stream_batch_size(stream_name);
- CHECK_EXPECTED(batch_size_exp);
+ TRY(auto vdma_channel, get_boundary_vdma_channel_by_stream_name(stream_name),
+ "Failed to get vdma channel for output stream {}", stream_name);
- auto vdma_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(stream_name);
- CHECK_EXPECTED(vdma_channel_ptr_exp, "Failed to get vdma channel for output stream {}", stream_name);
+ TRY(auto result, VdmaOutputStream::create(stream_params.stream_interface, *vdma_device, vdma_channel,
+ layer_info, m_core_op_activated_event));
- return VdmaOutputStream::create(stream_params.stream_interface, *vdma_device, vdma_channel_ptr_exp.value(),
- layer_info, m_core_op_activated_event);
+ return Expected<std::shared_ptr<OutputStreamBase>>(result);
}
hailo_status CoreOp::create_streams_from_config_params(Device &device)
switch (stream_parameters_pair.second.direction) {
case HAILO_H2D_STREAM:
{
- auto stream = create_input_stream_from_config_params(device,
- stream_parameters_pair.second,
- stream_parameters_pair.first);
- CHECK_EXPECTED_AS_STATUS(stream);
+ TRY(auto stream,
+ create_input_stream_from_config_params(device,
+ stream_parameters_pair.second,
+ stream_parameters_pair.first));
- auto status = add_input_stream(stream.release(), stream_parameters_pair.second);
+ auto status = add_input_stream(std::move(stream), stream_parameters_pair.second);
CHECK_SUCCESS(status);
}
break;
case HAILO_D2H_STREAM:
{
- auto stream = create_output_stream_from_config_params(device,
- stream_parameters_pair.second,
- stream_parameters_pair.first);
- CHECK_EXPECTED_AS_STATUS(stream);
+ TRY(auto stream,
+ create_output_stream_from_config_params(device,
+ stream_parameters_pair.second,
+ stream_parameters_pair.first));
- auto status = add_output_stream(stream.release(), stream_parameters_pair.second);
+ auto status = add_output_stream(std::move(stream), stream_parameters_pair.second);
CHECK_SUCCESS(status);
}
break;
Expected<InputStreamRefVector> CoreOp::get_input_streams_by_network(const std::string &network_name)
{
- auto input_stream_infos = m_metadata->get_input_stream_infos(network_name);
- CHECK_EXPECTED(input_stream_infos);
+ TRY(auto input_stream_infos, m_metadata->get_input_stream_infos(network_name));
InputStreamRefVector result;
- for (auto &stream_info : input_stream_infos.value()) {
- auto stream_ref = get_input_stream_by_name(stream_info.name);
- CHECK_EXPECTED(stream_ref);
- result.emplace_back(stream_ref.release());
+ for (auto &stream_info : input_stream_infos) {
+ TRY(auto stream_ref, get_input_stream_by_name(stream_info.name));
+ result.emplace_back(stream_ref);
}
return result;
}
Expected<OutputStreamRefVector> CoreOp::get_output_streams_by_network(const std::string &network_name)
{
- auto output_stream_infos = m_metadata->get_output_stream_infos(network_name);
- CHECK_EXPECTED(output_stream_infos);
+ TRY(auto output_stream_infos, m_metadata->get_output_stream_infos(network_name));
OutputStreamRefVector result;
- for (auto &stream_info : output_stream_infos.value()) {
- auto stream_ref = get_output_stream_by_name(stream_info.name);
- CHECK_EXPECTED(stream_ref);
- result.emplace_back(stream_ref.release());
+ for (auto &stream_info : output_stream_infos) {
+ TRY(auto stream_ref, get_output_stream_by_name(stream_info.name));
+ result.emplace_back(stream_ref);
}
return result;
}
bool is_default_batch_size() const;
virtual Expected<Buffer> get_intermediate_buffer(const IntermediateBufferKey &key);
+ virtual Expected<Buffer> get_cache_buffer(uint32_t cache_id);
+ virtual Expected<std::map<uint32_t, Buffer>> get_cache_buffers();
hailo_status wrap_streams_for_remote_process();
*/
hailo_status infer_async(InferRequest &&request);
+ virtual bool has_caches() const = 0;
+ virtual Expected<uint32_t> get_cache_read_size() const = 0;
+ virtual Expected<uint32_t> get_cache_write_size() const = 0;
+ virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) = 0;
+ virtual Expected<hailo_cache_info_t> get_cache_info() const = 0;
+ virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) = 0;
+
std::map<std::string, std::shared_ptr<InputStreamBase>> m_input_streams;
std::map<std::string, std::shared_ptr<OutputStreamBase>> m_output_streams;
protected:
CoreOp(const ConfigureNetworkParams &config_params, std::shared_ptr<CoreOpMetadata> metadata,
- ActiveCoreOpHolder &active_core_op_holder, hailo_status &status);
+ ActiveCoreOpHolder &active_core_op_holder, hailo_status &status, bool is_scheduled = false);
Expected<std::shared_ptr<OutputStreamBase>> create_output_stream_from_config_params(Device &device,
const hailo_stream_parameters_t &stream_params, const std::string &stream_name);
**/
#include "ddr_action_list_buffer_builder.hpp"
+#include "common/os_utils.hpp"
+#include "vdma/integrated/integrated_device.hpp"
namespace hailort
{
// Like the dsp etc...need to check with them before doing so. For now - this should almost always retirn in the mapped area and we will verify
// to double check
-DDRActionListBufferBuilder::DDRActionListBufferBuilder(vdma::ContinuousBuffer &&buffer) :
+DDRActionListBufferBuilder::DDRActionListBufferBuilder(void* user_address, uint64_t dma_address) :
ActionListBufferBuilder(ActionListBufferBuilder::Type::DDR),
- m_action_list_buffer(std::move(buffer)),
+ m_user_address(user_address),
+ m_dma_address(dma_address),
m_write_offset(0),
m_current_context_info{}
{}
}
Expected<std::shared_ptr<DDRActionListBufferBuilder>> DDRActionListBufferBuilder::create(size_t num_contexts,
- HailoRTDriver &driver)
+ VdmaDevice &vdma_device)
{
- // Try to allocate continous buffer for action list in DDR
- auto continous_alloc = vdma::ContinuousBuffer::create(num_contexts *
- sizeof(CONTROL_PROTOCOL__context_switch_context_info_chunk_t), driver);
-
- // TODO HRT-12512 - Add fallback to Control if continous buffer allocation fails
- CHECK_EXPECTED(continous_alloc);
- // Verify that continous buffer is in allocated region
- CHECK_AS_EXPECTED(verify_dma_addr(continous_alloc.value()), HAILO_INTERNAL_FAILURE,
- "Failed to allocate continous buffer in M4 mapped memory region");
- return make_shared_nothrow<DDRActionListBufferBuilder>(continous_alloc.release());
+ auto integrated_device = dynamic_cast<IntegratedDevice*>(&vdma_device);
+
+ size_t size_of_contexts = HailoRTCommon::align_to(num_contexts *
+ sizeof(CONTROL_PROTOCOL__context_switch_context_info_chunk_t), OsUtils::get_page_size());
+
+ TRY(auto addr_pair, integrated_device->allocate_infinite_action_list_buffer(size_of_contexts));
+
+ auto ddr_action_list_buiffer_builder = make_shared_nothrow<DDRActionListBufferBuilder>(
+ addr_pair.first, addr_pair.second);
+ CHECK_NOT_NULL_AS_EXPECTED(ddr_action_list_buiffer_builder, HAILO_OUT_OF_HOST_MEMORY);
+
+ return ddr_action_list_buiffer_builder;
}
hailo_status DDRActionListBufferBuilder::write_action(MemoryView action,
if (is_last_action_in_context) {
const auto write_size = sizeof(CONTROL_PROTOCOL__context_switch_context_info_chunk_t);
- auto status = m_action_list_buffer.write(&m_current_context_info, write_size, m_write_offset);
- CHECK_SUCCESS(status);
+ memcpy(static_cast<void*>(reinterpret_cast<uint8_t*>(m_user_address) + m_write_offset), &m_current_context_info,
+ write_size);
m_write_offset += write_size;
}
uint64_t DDRActionListBufferBuilder::get_mapped_buffer_dma_address() const
{
- return m_action_list_buffer.dma_address();
+ return m_dma_address;
}
} /* namespace hailort */
\ No newline at end of file
#include "context_switch_defs.h"
#include "core_op/resource_manager/action_list_buffer_builder/action_list_buffer_builder.hpp"
#include "vdma/memory/continuous_buffer.hpp"
+#include "vdma/vdma_device.hpp"
#define DDR_ACTION_LIST_ENV_VAR ("HAILO_DDR_ACTION_LIST")
#define DDR_ACTION_LIST_ENV_VAR_VALUE ("1")
class DDRActionListBufferBuilder : public ActionListBufferBuilder {
public:
- DDRActionListBufferBuilder(vdma::ContinuousBuffer &&buffer);
+ DDRActionListBufferBuilder(void* user_address, uint64_t dma_address);
virtual ~DDRActionListBufferBuilder() = default;
- static Expected<std::shared_ptr<DDRActionListBufferBuilder>> create(size_t num_contexts, HailoRTDriver &driver);
+ static Expected<std::shared_ptr<DDRActionListBufferBuilder>> create(size_t num_contexts, VdmaDevice &vdma_device);
virtual hailo_status write_action(MemoryView action, CONTROL_PROTOCOL__context_switch_context_type_t context_type,
bool is_new_context, bool last_action_buffer_in_context) override;
virtual uint64_t get_mapped_buffer_dma_address() const override;
-private:
- vdma::ContinuousBuffer m_action_list_buffer;
+
// TODO: HRT-12512 : Can remove this check when / if continuous buffer comes from designated region
static bool verify_dma_addr(vdma::ContinuousBuffer &buffer);
+private:
+ // vdma::ContinuousBuffer m_action_list_buffer;
+ void* m_user_address;
+ uint64_t m_dma_address;
size_t m_write_offset;
CONTROL_PROTOCOL__context_switch_context_info_chunk_t m_current_context_info;
};
--- /dev/null
+\r
+/**\r
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.\r
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)\r
+**/\r
+/**\r
+ * @file cache_buffer.cpp\r
+ * @brief Wrapper for intermediate buffers used as caches\r
+ **/\r
+\r
+#include "cache_buffer.hpp"\r
+#include "hailo/hailort.h"\r
+#include "vdma/memory/sg_buffer.hpp"\r
+\r
+namespace hailort\r
+{\r
+\r
+Expected<CacheBuffer> CacheBuffer::create(HailoRTDriver &driver, uint32_t cache_size,\r
+ uint32_t input_size, uint32_t output_size)\r
+{\r
+ CHECK(cache_size > 0, HAILO_INVALID_ARGUMENT);\r
+ CHECK((input_size > 0) && (input_size < cache_size), HAILO_INVALID_ARGUMENT,\r
+ "Invalid cache input size: {} (cache size: {})", input_size, cache_size);\r
+ CHECK((output_size > 0) && (output_size < cache_size), HAILO_INVALID_ARGUMENT,\r
+ "Invalid cache output size: {} (cache size: {})", output_size, cache_size);\r
+\r
+ // Cache buffers are by sg buffers\r
+ TRY(auto buffer, vdma::SgBuffer::create(driver, cache_size, HailoRTDriver::DmaDirection::BOTH));\r
+ auto buffer_ptr = make_shared_nothrow<vdma::SgBuffer>(std::move(buffer));\r
+ CHECK_NOT_NULL(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY);\r
+ return CacheBuffer(cache_size, input_size, output_size, buffer_ptr);\r
+}\r
+\r
+CacheBuffer::CacheBuffer(uint32_t cache_size, uint32_t input_size, uint32_t output_size,\r
+ std::shared_ptr<vdma::VdmaBuffer> backing_buffer) :\r
+ m_cache_size(cache_size),\r
+ m_input_size(input_size),\r
+ m_output_size(output_size),\r
+ m_backing_buffer(backing_buffer)\r
+{}\r
+\r
+ExpectedRef<IntermediateBuffer> CacheBuffer::set_input_channel(HailoRTDriver &driver, vdma::ChannelId channel_id)\r
+{\r
+ if (m_cache_input) {\r
+ return std::ref(*m_cache_input);\r
+ }\r
+\r
+ static const auto SINGLE_BATCH = 1;\r
+ static const auto BUFFER_START = 0;\r
+ TRY(auto intermediate_buffer, IntermediateBuffer::create_shared(driver, m_input_size, SINGLE_BATCH, channel_id,\r
+ IntermediateBuffer::StreamingType::BURST, m_backing_buffer, BUFFER_START));\r
+ m_cache_input = intermediate_buffer;\r
+ return std::ref(*m_cache_input);\r
+}\r
+\r
+ExpectedRef<IntermediateBuffer> CacheBuffer::set_output_channel(HailoRTDriver &driver, vdma::ChannelId channel_id)\r
+{\r
+ if (m_cache_output) {\r
+ return std::ref(*m_cache_output);\r
+ }\r
+\r
+ static const auto SINGLE_BATCH = 1;\r
+ static const auto BUFFER_START = 0;\r
+ TRY(auto intermediate_buffer, IntermediateBuffer::create_shared(driver, m_output_size, SINGLE_BATCH, channel_id,\r
+ IntermediateBuffer::StreamingType::BURST, m_backing_buffer, BUFFER_START));\r
+ m_cache_output = intermediate_buffer;\r
+ return std::ref(*m_cache_output);\r
+}\r
+\r
+ExpectedRef<IntermediateBuffer> CacheBuffer::get_input()\r
+{\r
+ CHECK(m_cache_input, HAILO_INTERNAL_FAILURE, "Input not set");\r
+ return std::ref(*m_cache_input);\r
+}\r
+\r
+ExpectedRef<IntermediateBuffer> CacheBuffer::get_output()\r
+{\r
+ CHECK(m_cache_output, HAILO_INTERNAL_FAILURE, "Output not set");\r
+ return std::ref(*m_cache_output);\r
+}\r
+\r
+Expected<Buffer> CacheBuffer::read_entire_cache()\r
+{\r
+ CHECK(m_cache_input && m_cache_output, HAILO_INTERNAL_FAILURE, "Input or output not set");\r
+\r
+ return m_cache_input->read(m_cache_size);\r
+}\r
+\r
+uint32_t CacheBuffer::cache_size() const\r
+{\r
+ return m_cache_size;\r
+}\r
+\r
+uint32_t CacheBuffer::input_size() const\r
+{\r
+ return m_input_size;\r
+}\r
+\r
+uint32_t CacheBuffer::output_size() const\r
+{\r
+ return m_output_size;\r
+}\r
+\r
+bool CacheBuffer::is_configured() const\r
+{\r
+ return m_cache_input && m_cache_output;\r
+}\r
+\r
+} /* namespace hailort */\r
--- /dev/null
+/**\r
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.\r
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)\r
+**/\r
+/**\r
+ * @file cache_buffer.hpp\r
+ * @brief Wrapper for intermediate buffers used as caches\r
+ **/\r
+\r
+#ifndef _HAILO_CACHE_BUFFER_HPP_\r
+#define _HAILO_CACHE_BUFFER_HPP_\r
+\r
+#include "hailo/hailort.h"\r
+#include "core_op/resource_manager/intermediate_buffer.hpp"\r
+\r
+namespace hailort\r
+{\r
+\r
+class CacheBuffer final\r
+{\r
+public:\r
+ static Expected<CacheBuffer> create(HailoRTDriver &driver, uint32_t cache_size,\r
+ uint32_t input_size, uint32_t output_size);\r
+\r
+ CacheBuffer(CacheBuffer &&) = default;\r
+ CacheBuffer(const CacheBuffer &) = delete;\r
+ CacheBuffer &operator=(CacheBuffer &&) = delete;\r
+ CacheBuffer &operator=(const CacheBuffer &) = delete;\r
+ ~CacheBuffer() = default;\r
+\r
+ // Set input/output channels to/from the cache. Will only be set once for each direction.\r
+ // (subsequent calls will return the same IntermediateBuffer.)\r
+ ExpectedRef<IntermediateBuffer> set_input_channel(HailoRTDriver &driver, vdma::ChannelId channel_id);\r
+ ExpectedRef<IntermediateBuffer> set_output_channel(HailoRTDriver &driver, vdma::ChannelId channel_id);\r
+ ExpectedRef<IntermediateBuffer> get_input();\r
+ ExpectedRef<IntermediateBuffer> get_output();\r
+ Expected<Buffer> read_entire_cache();\r
+ uint32_t cache_size() const;\r
+ uint32_t input_size() const;\r
+ uint32_t output_size() const;\r
+ // Returns true if both input and output channels are set.\r
+ bool is_configured() const;\r
+\r
+private:\r
+ CacheBuffer(uint32_t cache_size, uint32_t input_size, uint32_t output_size,\r
+ std::shared_ptr<vdma::VdmaBuffer> backing_buffer);\r
+\r
+ const uint32_t m_cache_size;\r
+ const uint32_t m_input_size;\r
+ const uint32_t m_output_size;\r
+ const std::shared_ptr<vdma::VdmaBuffer> m_backing_buffer;\r
+ // Each cache buffer has an input and output IntermediateBuffer -\r
+ // * They both share the same backing buffer.\r
+ // * They each have separate descriptor lists that will be programmed separately.\r
+ // * This way we can read/write/reprogram the cache buffer without affecting the other direction.\r
+ std::shared_ptr<IntermediateBuffer> m_cache_input;\r
+ std::shared_ptr<IntermediateBuffer> m_cache_output;\r
+};\r
+\r
+} /* namespace hailort */\r
+\r
+#endif /* _HAILO_CACHE_BUFFER_HPP_ */\r
--- /dev/null
+/**\r
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.\r
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)\r
+**/\r
+/**\r
+ * @file cache_manager.cpp\r
+ * @brief Manges creation and configuration of cache buffers\r
+ **/\r
+\r
+#include "cache_manager.hpp"\r
+#include "hailo/hailort.h"\r
+\r
+namespace hailort\r
+{\r
+\r
+Expected<CacheManagerPtr> CacheManager::create_shared(HailoRTDriver &driver)\r
+{\r
+ auto cache_manager = make_shared_nothrow<CacheManager>(driver);\r
+ CHECK_NOT_NULL(cache_manager, HAILO_OUT_OF_HOST_MEMORY);\r
+ \r
+ return cache_manager;\r
+}\r
+\r
+CacheManager::CacheManager(HailoRTDriver &driver) :\r
+ m_driver(driver),\r
+ m_caches_created(false),\r
+ m_initialized(false),\r
+ m_cache_input_size(0),\r
+ m_cache_output_size(0),\r
+ m_cache_size(0),\r
+ m_read_offset_bytes(0),\r
+ m_write_offset_bytes_delta(0),\r
+ m_cache_buffers(),\r
+ m_uninitialized_caches()\r
+{\r
+}\r
+\r
+hailo_status CacheManager::create_caches_from_core_op(std::shared_ptr<CoreOpMetadata> core_op_metadata)\r
+{\r
+ if (m_caches_created) {\r
+ // Already created caches, nothing to do\r
+ // In debug, validate that the cache sizes + ids are the same as the ones we already have\r
+ assert(m_cache_input_size == get_cache_input_size(core_op_metadata));\r
+ assert(m_cache_output_size == get_cache_output_size(core_op_metadata));\r
+ assert(validate_cache_ids(core_op_metadata, m_cache_buffers));\r
+\r
+ return HAILO_SUCCESS;\r
+ }\r
+\r
+ if (!core_op_has_caches(core_op_metadata)) {\r
+ // No cache layers found, nothing to do\r
+ return HAILO_SUCCESS;\r
+ }\r
+\r
+ m_cache_input_size = get_cache_input_size(core_op_metadata);\r
+ m_cache_output_size = get_cache_output_size(core_op_metadata);\r
+ // TODO: cache size should be a param of the hef (via sdk)\r
+ // that way we can also immediately know if caches are used or not\r
+ // it should be a param that appears once in the hef, and is used by all caches (HRT-13584)\r
+ m_cache_size = m_cache_input_size + m_cache_output_size;\r
+ m_write_offset_bytes_delta = m_cache_input_size;\r
+\r
+ assert(validate_cache_edge_layers(core_op_metadata, m_cache_input_size, m_cache_output_size));\r
+ auto status = allocate_cache_buffers(core_op_metadata);\r
+ CHECK_SUCCESS(status, "Failed to allocate cache buffers");\r
+\r
+ m_caches_created = true;\r
+\r
+ return HAILO_SUCCESS;\r
+\r
+}\r
+\r
+bool CacheManager::core_op_has_caches(std::shared_ptr<CoreOpMetadata> core_op_metadata)\r
+{\r
+ for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) {\r
+ if (!context_metadata.get_cache_input_layers().empty() || !context_metadata.get_cache_output_layers().empty()) {\r
+ return true;\r
+ }\r
+ }\r
+\r
+ return false;\r
+}\r
+\r
+bool CacheManager::validate_cache_edge_layers(std::shared_ptr<CoreOpMetadata> core_op_metadata,\r
+ uint32_t cache_input_size, uint32_t cache_output_size)\r
+{\r
+ std::unordered_map<uint32_t, uint32_t> cache_id_count;\r
+ for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) {\r
+ for (const auto &layer_info : context_metadata.get_cache_input_layers()) {\r
+ cache_id_count[layer_info.cache_info.id]++;\r
+ if (cache_input_size != LayerInfoUtils::get_layer_transfer_size(layer_info)) {\r
+ return false;\r
+ }\r
+ }\r
+\r
+ for (const auto &layer_info : context_metadata.get_cache_output_layers()) {\r
+ cache_id_count[layer_info.cache_info.id]++;\r
+ if (cache_output_size != LayerInfoUtils::get_layer_transfer_size(layer_info)) {\r
+ return false;\r
+ }\r
+ }\r
+ }\r
+\r
+ static const uint32_t EXPECTED_CACHE_ID_COUNT = 2; // Each cache has 2 layers (input and output)\r
+ for (const auto &cache_id : cache_id_count) {\r
+ if (cache_id.second != EXPECTED_CACHE_ID_COUNT) {\r
+ return false;\r
+ }\r
+ }\r
+\r
+ return true;\r
+}\r
+\r
+uint32_t CacheManager::get_cache_input_size(std::shared_ptr<CoreOpMetadata> core_op_metadata)\r
+{\r
+ // All cache layers have the same input size (this will be asserted in debug)\r
+ for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) {\r
+ for (const auto &layer_info : context_metadata.get_cache_input_layers()) {\r
+ return LayerInfoUtils::get_layer_transfer_size(layer_info);\r
+ }\r
+ }\r
+\r
+ // No cache layers found\r
+ return 0;\r
+}\r
+\r
+uint32_t CacheManager::get_cache_output_size(std::shared_ptr<CoreOpMetadata> core_op_metadata)\r
+{\r
+ // All cache layers have the same output size (this will be asserted in debug)\r
+ for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) {\r
+ for (const auto &layer_info : context_metadata.get_cache_output_layers()) {\r
+ return LayerInfoUtils::get_layer_transfer_size(layer_info);\r
+ }\r
+ }\r
+\r
+ // No cache layers found\r
+ return 0;\r
+}\r
+\r
+bool CacheManager::validate_cache_ids(std::shared_ptr<CoreOpMetadata> core_op_metadata,\r
+ const std::unordered_map<uint32_t, CacheBuffer> ¤t_cache_buffers)\r
+{\r
+ std::unordered_set<uint32_t> cache_ids;\r
+ for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) {\r
+ for (const auto &layer_info : context_metadata.get_cache_input_layers()) {\r
+ cache_ids.insert(layer_info.cache_info.id);\r
+ }\r
+\r
+ for (const auto &layer_info : context_metadata.get_cache_output_layers()) {\r
+ cache_ids.insert(layer_info.cache_info.id);\r
+ }\r
+ }\r
+\r
+ if (cache_ids.size() != current_cache_buffers.size()) {\r
+ return false;\r
+ }\r
+\r
+ for (const auto &cache_id : cache_ids) {\r
+ if (std::end(current_cache_buffers) == current_cache_buffers.find(cache_id)) {\r
+ return false;\r
+ }\r
+ }\r
+\r
+ return true;\r
+}\r
+\r
+ExpectedRef<CacheBuffer> CacheManager::get_cache_buffer(uint32_t cache_id)\r
+{\r
+ const auto cache_buffer_it = m_cache_buffers.find(cache_id);\r
+ if (std::end(m_cache_buffers) != cache_buffer_it) {\r
+ return std::ref(cache_buffer_it->second);\r
+ }\r
+\r
+ return make_unexpected(HAILO_NOT_FOUND);\r
+}\r
+\r
+hailo_status CacheManager::allocate_cache_buffers(std::shared_ptr<CoreOpMetadata> core_op_metadata)\r
+{\r
+ // It's enough to go over cache_output_layers, as each cache has both input and output layers (that share the same buffer)\r
+ for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) {\r
+ for (const auto &layer_info : context_metadata.get_cache_output_layers()) {\r
+ const auto cache_id = layer_info.cache_info.id;\r
+ TRY(auto cache_buffer, CacheBuffer::create(m_driver, m_cache_size, m_cache_input_size, m_cache_output_size));\r
+ auto emplace_res = m_cache_buffers.emplace(cache_id, std::move(cache_buffer));\r
+ CHECK(emplace_res.second, HAILO_INTERNAL_FAILURE);\r
+\r
+ // The cache buffer is yet to be initialized (will be initalized when it is configured with input/output channels)\r
+ m_uninitialized_caches.insert(cache_id);\r
+ }\r
+ }\r
+\r
+ return HAILO_SUCCESS;\r
+}\r
+\r
+hailo_status CacheManager::program_cache_buffers()\r
+{\r
+ // Set the cache to the initial configuration (program the descriptors to the initial offset)\r
+ static const auto INITIAL_CONFIGURATION_OFFSET = 0;\r
+ return update_cache_offset(INITIAL_CONFIGURATION_OFFSET);\r
+}\r
+\r
+hailo_status CacheManager::try_complete_cache_initialization()\r
+{\r
+ // If all caches are now initialized, program their desc list and set the CacheManager as initialized\r
+ if (m_uninitialized_caches.empty() && !m_initialized) {\r
+ m_initialized = true;\r
+\r
+ auto status = program_cache_buffers();\r
+ CHECK_SUCCESS(status, "Failed to program cache buffers");\r
+ }\r
+\r
+ return HAILO_SUCCESS;\r
+}\r
+\r
+ExpectedRef<IntermediateBuffer> CacheManager::set_cache_input_channel(uint32_t cache_id, uint16_t batch_size,\r
+ vdma::ChannelId channel_id)\r
+{\r
+\r
+ // TODO: Support non-1 batches? (HRT-13628)\r
+ CHECK(1 == batch_size, HAILO_INVALID_ARGUMENT, "Cache input batch size must be 1");\r
+ TRY(auto cache_buffer, get_cache_buffer(cache_id));\r
+ if (m_initialized) {\r
+ // Cache is already initialized, return the input channel\r
+ return cache_buffer.get().get_input();\r
+ }\r
+ TRY(auto result, cache_buffer.get().set_input_channel(m_driver, channel_id));\r
+\r
+ // If the cache is now fully configured, remove it from the uninitialized set\r
+ if (cache_buffer.get().is_configured()) {\r
+ m_uninitialized_caches.erase(cache_id);\r
+ auto status = try_complete_cache_initialization();\r
+ CHECK_SUCCESS(status);\r
+ }\r
+\r
+ return result;\r
+}\r
+\r
+ExpectedRef<IntermediateBuffer> CacheManager::set_cache_output_channel(uint32_t cache_id, uint16_t batch_size,\r
+ vdma::ChannelId channel_id)\r
+{\r
+ // TODO: Support non-1 batches? (HRT-13628)\r
+ CHECK(1 == batch_size, HAILO_INVALID_ARGUMENT, "Cache output batch size must be 1");\r
+ TRY(auto cache_buffer, get_cache_buffer(cache_id));\r
+ if (m_initialized) {\r
+ // Cache is already initialized, return the output channel\r
+ return cache_buffer.get().get_output();\r
+ }\r
+ TRY(auto result, cache_buffer.get().set_output_channel(m_driver, channel_id));\r
+\r
+ // If the cache is now fully configured, remove it from the uninitialized set\r
+ if (cache_buffer.get().is_configured()) {\r
+ m_uninitialized_caches.erase(cache_id);\r
+ auto status = try_complete_cache_initialization();\r
+ CHECK_SUCCESS(status);\r
+ }\r
+\r
+ return result;\r
+}\r
+\r
+std::unordered_map<uint32_t, CacheBuffer> &CacheManager::get_cache_buffers()\r
+{\r
+ return m_cache_buffers;\r
+}\r
+\r
+hailo_status CacheManager::init_caches(uint32_t initial_read_offset_bytes, int32_t write_offset_bytes_delta)\r
+{\r
+ if (!m_caches_created) {\r
+ // No cache layers found, nothing to do\r
+ LOGGER__WARNING("No cache layers found, but init_cache was called");\r
+ return HAILO_SUCCESS;\r
+ }\r
+\r
+ CHECK(initial_read_offset_bytes < m_cache_size, HAILO_INVALID_ARGUMENT);\r
+ CHECK(write_offset_bytes_delta != 0, HAILO_INVALID_ARGUMENT);\r
+\r
+ m_read_offset_bytes = initial_read_offset_bytes;\r
+ m_write_offset_bytes_delta = write_offset_bytes_delta;\r
+\r
+ LOGGER__WARNING("Initializing caches [read_offset={}, write_offset_delta={}]",\r
+ m_read_offset_bytes, m_write_offset_bytes_delta);\r
+\r
+ return program_cache_buffers();\r
+}\r
+\r
+hailo_status CacheManager::update_cache_offset(int32_t offset_delta_bytes)\r
+{\r
+ if (!m_caches_created) {\r
+ // No cache layers found, nothing to do\r
+ LOGGER__WARNING("No cache layers found, but update_cache_offset was called");\r
+ return HAILO_SUCCESS;\r
+ }\r
+\r
+ CHECK(m_initialized, HAILO_INVALID_OPERATION, "CacheManager not initialized");\r
+\r
+ auto status = HAILO_UNINITIALIZED;\r
+ auto new_read_offset = (m_read_offset_bytes + offset_delta_bytes) % m_cache_size;\r
+ auto new_write_offset = (m_read_offset_bytes + offset_delta_bytes + m_write_offset_bytes_delta) % m_cache_size;\r
+\r
+ for (auto &cache_buffer : m_cache_buffers) {\r
+ TRY(auto cache_input, cache_buffer.second.get_input());\r
+ status = cache_input.get().reprogram_descriptors(new_read_offset);\r
+ CHECK_SUCCESS(status, "Failed to reprogram read cache descriptors to offset 0x{:x} (cache_id {})",\r
+ new_read_offset, cache_buffer.first);\r
+\r
+ TRY(auto cache_output, cache_buffer.second.get_output());\r
+ status = cache_output.get().reprogram_descriptors(new_write_offset);\r
+ CHECK_SUCCESS(status, "Failed to reprogram write cache descriptors to offset 0x{:x} (cache_id {})",\r
+ new_write_offset, cache_buffer.first);\r
+ }\r
+\r
+ m_read_offset_bytes = new_read_offset;\r
+\r
+ return HAILO_SUCCESS;\r
+}\r
+\r
+uint32_t CacheManager::get_cache_size() const\r
+{\r
+ return m_cache_size;\r
+}\r
+\r
+uint32_t CacheManager::get_read_offset_bytes() const\r
+{\r
+ return m_read_offset_bytes;\r
+}\r
+\r
+int32_t CacheManager::get_write_offset_bytes_delta() const\r
+{\r
+ return m_write_offset_bytes_delta;\r
+}\r
+\r
+} /* namespace hailort */\r
--- /dev/null
+/**\r
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.\r
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)\r
+**/\r
+/**\r
+ * @file cache_manager.hpp\r
+ * @brief Manges creation and configuration of cache buffers\r
+ **/\r
+\r
+#ifndef _HAILO_CACHE_MANAGER_HPP_\r
+#define _HAILO_CACHE_MANAGER_HPP_\r
+\r
+#include "hailo/hailort.h"\r
+#include "hailo/expected.hpp"\r
+#include "core_op/resource_manager/cache_buffer.hpp"\r
+#include "hef/core_op_metadata.hpp"\r
+\r
+#include <unordered_map>\r
+#include <unordered_set>\r
+\r
+namespace hailort\r
+{\r
+\r
+class CacheManager;\r
+using CacheManagerPtr = std::shared_ptr<CacheManager>;\r
+class CacheManager final\r
+{\r
+public:\r
+ // TODO: Support getting initial_read_offset_bytes + write_offset_bytes_delta from configured_network_params\r
+ // s.t. the CacheManager can be created with the correct offsets, and init_caches won't be needed at the start.\r
+ // Currently, the CacheManager is created with the m_read_offset_bytes=0 and\r
+ // m_write_offset_bytes_delta=m_cache_input_size (i.e. right after where data was read from) (HRT-14288)\r
+ static Expected<CacheManagerPtr> create_shared(HailoRTDriver &driver);\r
+\r
+ CacheManager(HailoRTDriver &driver);\r
+ CacheManager(CacheManager &&) = default;\r
+ CacheManager(const CacheManager &) = delete;\r
+ CacheManager &operator=(CacheManager &&) = delete;\r
+ CacheManager &operator=(const CacheManager &) = delete;\r
+ ~CacheManager() = default;\r
+\r
+ hailo_status create_caches_from_core_op(std::shared_ptr<CoreOpMetadata> core_op_metadata);\r
+ ExpectedRef<IntermediateBuffer> set_cache_input_channel(uint32_t cache_id, uint16_t batch_size, vdma::ChannelId channel_id);\r
+ ExpectedRef<IntermediateBuffer> set_cache_output_channel(uint32_t cache_id, uint16_t batch_size, vdma::ChannelId channel_id);\r
+ std::unordered_map<uint32_t, CacheBuffer> &get_cache_buffers();\r
+\r
+ // Note: These functions are not thread-safe!\r
+ // Programs the CacheManager instance with the given offsets, overriding the current offsets.\r
+ hailo_status init_caches(uint32_t initial_read_offset_bytes, int32_t write_offset_bytes_delta);\r
+ // Updates the read offset by the given delta\r
+ hailo_status update_cache_offset(int32_t offset_delta_bytes);\r
+\r
+ uint32_t get_cache_size() const;\r
+ uint32_t get_read_offset_bytes() const;\r
+ int32_t get_write_offset_bytes_delta() const;\r
+\r
+private:\r
+ static bool core_op_has_caches(std::shared_ptr<CoreOpMetadata> core_op_metadata);\r
+ static bool validate_cache_edge_layers(std::shared_ptr<CoreOpMetadata> core_op_metadata,\r
+ uint32_t cache_input_size, uint32_t cache_output_size);\r
+ static uint32_t get_cache_input_size(std::shared_ptr<CoreOpMetadata> core_op_metadata);\r
+ static uint32_t get_cache_output_size(std::shared_ptr<CoreOpMetadata> core_op_metadata);\r
+ static bool validate_cache_ids(std::shared_ptr<CoreOpMetadata> core_op_metadata,\r
+ const std::unordered_map<uint32_t, CacheBuffer> ¤t_cache_buffers);\r
+ ExpectedRef<CacheBuffer> get_cache_buffer(uint32_t cache_id);\r
+ hailo_status allocate_cache_buffers(std::shared_ptr<CoreOpMetadata> core_op_metadata);\r
+ hailo_status program_cache_buffers();\r
+ hailo_status try_complete_cache_initialization();\r
+\r
+ HailoRTDriver &m_driver;\r
+ bool m_caches_created;\r
+ // This class is initialized (and the member is set to true) when all caches are allocated and configured with\r
+ // input/output channels. This is done in two steps: (1) cache allocation; (2) channel configuration\r
+ // Two steps are necessary because this class allocates the buffers, however the input/output channels are assigned\r
+ // by the resource manager\r
+ bool m_initialized;\r
+ uint32_t m_cache_input_size;\r
+ uint32_t m_cache_output_size;\r
+ uint32_t m_cache_size;\r
+ uint32_t m_read_offset_bytes;\r
+ int32_t m_write_offset_bytes_delta;\r
+ std::unordered_map<uint32_t, CacheBuffer> m_cache_buffers;\r
+ std::unordered_set<uint32_t> m_uninitialized_caches;\r
+};\r
+\r
+} /* namespace hailort */\r
+\r
+#endif /* _HAILO_CACHE_MANAGER_HPP_ */\r
namespace hailort {
Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> ConfigBuffer::create_buffer(HailoRTDriver &driver, vdma::ChannelId channel_id,
- const std::vector<uint32_t> &cfg_sizes, const uint32_t buffer_size)
+ const std::vector<uint32_t> &bursts_sizes, const uint32_t buffer_size)
{
auto buffer_ptr = should_use_ccb(driver) ?
create_ccb_buffer(driver, buffer_size) :
- create_sg_buffer(driver, channel_id, cfg_sizes);
+ create_sg_buffer(driver, channel_id, bursts_sizes);
if (should_use_ccb(driver) && (HAILO_OUT_OF_HOST_CMA_MEMORY == buffer_ptr.status())) {
/* Try to use sg buffer instead */
- return create_sg_buffer(driver, channel_id, cfg_sizes);
+ return create_sg_buffer(driver, channel_id, bursts_sizes);
} else {
return buffer_ptr;
}
}
Expected<ConfigBuffer> ConfigBuffer::create(HailoRTDriver &driver, vdma::ChannelId channel_id,
- const std::vector<uint32_t> &cfg_sizes)
+ const std::vector<uint32_t> &bursts_sizes)
{
- const auto buffer_size = std::accumulate(cfg_sizes.begin(), cfg_sizes.end(), 0);
+ const auto buffer_size = std::accumulate(bursts_sizes.begin(), bursts_sizes.end(), 0);
CHECK_AS_EXPECTED(IS_FIT_IN_UINT32(buffer_size), HAILO_INTERNAL_FAILURE, "config buffer size exceeded UINT32 range limit");
- auto buffer_ptr = create_buffer(driver, channel_id, cfg_sizes, static_cast<uint32_t>(buffer_size));
- CHECK_EXPECTED(buffer_ptr);
+ TRY(auto buffer_ptr, create_buffer(driver, channel_id, bursts_sizes, static_cast<uint32_t>(buffer_size)));
- return ConfigBuffer(buffer_ptr.release(), channel_id, buffer_size);
+ return ConfigBuffer(std::move(buffer_ptr), channel_id, buffer_size);
}
ConfigBuffer::ConfigBuffer(std::unique_ptr<vdma::VdmaEdgeLayer> &&buffer,
Expected<uint32_t> ConfigBuffer::program_descriptors()
{
// TODO HRT-9657: remove DEVICE interrupts
- auto descriptors_count =
- m_buffer->program_descriptors(m_acc_buffer_offset, InterruptsDomain::DEVICE, m_acc_desc_count);
- CHECK_EXPECTED(descriptors_count);
+ TRY(auto descriptors_count,
+ m_buffer->program_descriptors(m_acc_buffer_offset, InterruptsDomain::DEVICE, m_acc_desc_count));
- m_acc_desc_count += descriptors_count.value();
+ m_acc_desc_count += descriptors_count;
m_acc_buffer_offset = 0;
return descriptors_count;
}
Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> ConfigBuffer::create_sg_buffer(HailoRTDriver &driver,
- vdma::ChannelId channel_id, const std::vector<uint32_t> &cfg_sizes)
+ vdma::ChannelId channel_id, const std::vector<uint32_t> &bursts_sizes)
{
static const auto NOT_CIRCULAR = false;
// For config channels (In Hailo15), the page size must be a multiplication of host default page size.
// Therefore we use the flag force_default_page_size for those types of buffers.
static const auto FORCE_DEFAULT_PAGE_SIZE = true;
static const auto FORCE_BATCH_SIZE = true;
- auto buffer_size_requirements = vdma::BufferSizesRequirements::get_buffer_requirements_multiple_transfers(
- vdma::VdmaBuffer::Type::SCATTER_GATHER, driver.desc_max_page_size(), 1, cfg_sizes, NOT_CIRCULAR,
- FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE);
- CHECK_EXPECTED(buffer_size_requirements);
- const auto page_size = buffer_size_requirements->desc_page_size();
- const auto descs_count = buffer_size_requirements->descs_count();
- const auto buffer_size = buffer_size_requirements->buffer_size();
-
- auto buffer = vdma::SgBuffer::create(driver, buffer_size, HailoRTDriver::DmaDirection::H2D);
- CHECK_EXPECTED(buffer);
-
- auto buffer_ptr = make_shared_nothrow<vdma::SgBuffer>(buffer.release());
+ TRY(const auto buffer_size_requirements, vdma::BufferSizesRequirements::get_buffer_requirements_multiple_transfers(
+ vdma::VdmaBuffer::Type::SCATTER_GATHER, driver.desc_max_page_size(), 1, bursts_sizes, NOT_CIRCULAR,
+ FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE));
+ const auto page_size = buffer_size_requirements.desc_page_size();
+ const auto descs_count = buffer_size_requirements.descs_count();
+ const auto buffer_size = buffer_size_requirements.buffer_size();
+
+ TRY(auto buffer, vdma::SgBuffer::create(driver, buffer_size, HailoRTDriver::DmaDirection::H2D));
+
+ auto buffer_ptr = make_shared_nothrow<vdma::SgBuffer>(std::move(buffer));
CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY);
static const auto DEFAULT_OFFSET = 0;
- auto edge_layer = vdma::SgEdgeLayer::create(std::move(buffer_ptr), buffer_size, DEFAULT_OFFSET, driver, descs_count,
- page_size, NOT_CIRCULAR, channel_id);
- CHECK_EXPECTED(edge_layer);
+ TRY(auto edge_layer, vdma::SgEdgeLayer::create(std::move(buffer_ptr), buffer_size, DEFAULT_OFFSET, driver, descs_count,
+ page_size, NOT_CIRCULAR, channel_id));
- auto edge_layer_ptr = make_unique_nothrow<vdma::SgEdgeLayer>(edge_layer.release());
+ auto edge_layer_ptr = make_unique_nothrow<vdma::SgEdgeLayer>(std::move(edge_layer));
CHECK_NOT_NULL_AS_EXPECTED(edge_layer_ptr, HAILO_OUT_OF_HOST_MEMORY);
return std::unique_ptr<vdma::VdmaEdgeLayer>(std::move(edge_layer_ptr));
static const auto FORCE_BATCH_SIZE = true;
static const auto DEFAULT_BATCH_SIZE = 1;
static const auto IS_VDMA_ALIGNED_BUFFER = true;
- auto buffer_size_requirements = vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer(
+ TRY(const auto buffer_size_requirements, vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer(
vdma::VdmaBuffer::Type::CONTINUOUS, driver.desc_max_page_size(), DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE,
- buffer_size, NOT_CIRCULAR, FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER);
- CHECK_EXPECTED(buffer_size_requirements);
-
- const auto page_size = buffer_size_requirements->desc_page_size();
- const auto descs_count = buffer_size_requirements->descs_count();
- auto buffer = vdma::ContinuousBuffer::create(buffer_size_requirements->buffer_size(), driver);
- /* Don't print error here since this might be expected error that the libhailoRT can recover from
- (out of host memory). If it's not the case, there is a print in hailort_driver.cpp file */
- if (HAILO_OUT_OF_HOST_CMA_MEMORY == buffer.status()) {
- return make_unexpected(buffer.status());
- } else {
- CHECK_EXPECTED(buffer);
- }
+ buffer_size, NOT_CIRCULAR, FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER));
+
+ const auto page_size = buffer_size_requirements.desc_page_size();
+ const auto descs_count = buffer_size_requirements.descs_count();
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_OUT_OF_HOST_CMA_MEMORY, auto buffer,
+ vdma::ContinuousBuffer::create(buffer_size_requirements.buffer_size(), driver));
- auto buffer_ptr = make_shared_nothrow<vdma::ContinuousBuffer>(buffer.release());
+ auto buffer_ptr = make_shared_nothrow<vdma::ContinuousBuffer>(std::move(buffer));
CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY);
static const auto DEFAULT_OFFSET = 0;
- auto edge_layer = vdma::ContinuousEdgeLayer::create(std::move(buffer_ptr), buffer_size, DEFAULT_OFFSET, page_size, descs_count);
- CHECK_EXPECTED(edge_layer);
+ TRY(auto edge_layer, vdma::ContinuousEdgeLayer::create(std::move(buffer_ptr), buffer_size, DEFAULT_OFFSET, page_size, descs_count));
- auto edge_layer_ptr = make_unique_nothrow<vdma::ContinuousEdgeLayer>(edge_layer.release());
+ auto edge_layer_ptr = make_unique_nothrow<vdma::ContinuousEdgeLayer>(std::move(edge_layer));
CHECK_NOT_NULL_AS_EXPECTED(edge_layer_ptr, HAILO_OUT_OF_HOST_MEMORY);
return std::unique_ptr<vdma::VdmaEdgeLayer>(std::move(edge_layer_ptr));
bool ConfigBuffer::should_use_ccb(HailoRTDriver &driver)
{
- switch (driver.dma_type()) {
- case HailoRTDriver::DmaType::PCIE:
- return false;
- case HailoRTDriver::DmaType::DRAM:
- if (nullptr != std::getenv("HAILO_FORCE_CONF_CHANNEL_OVER_DESC")) {
- LOGGER__WARNING("Using desc instead of CCB for config channel is not optimal for performance.\n");
- return false;
- }
- else {
- return true;
- }
+ if (driver.dma_type() != HailoRTDriver::DmaType::DRAM) {
+ return false; // not supported
}
- // Shouldn't reach here
- assert(false);
- return false;
+ if (nullptr != std::getenv("HAILO_FORCE_CONF_CHANNEL_OVER_DESC")) {
+ LOGGER__WARNING("Using desc instead of CCB for config channel is not optimal for performance.\n");
+ return false;
+ }
+ else {
+ return true;
+ }
}
} /* hailort */
\ No newline at end of file
{
public:
static Expected<ConfigBuffer> create(HailoRTDriver &driver, vdma::ChannelId channel_id,
- const std::vector<uint32_t> &cfg_sizes);
+ const std::vector<uint32_t> &bursts_sizes);
// Write data to config channel
hailo_status write(const MemoryView &data);
std::unique_ptr<vdma::VdmaEdgeLayer> m_buffer;
vdma::ChannelId m_channel_id;
- const size_t m_total_buffer_size;
+ const size_t m_total_buffer_size;
size_t m_acc_buffer_offset;
uint32_t m_acc_desc_count;
size_t m_current_buffer_size;
namespace hailort
{
Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> IntermediateBuffer::create_edge_layer(
- std::shared_ptr<vdma::VdmaBuffer> &&buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size,
+ std::shared_ptr<vdma::VdmaBuffer> buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size,
uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type)
{
const bool is_circular = (streaming_type == StreamingType::CIRCULAR_CONTINUOS);
auto buffer_exp = (vdma::VdmaBuffer::Type::CONTINUOUS == buffer->type()) ?
- create_ccb_edge_layer(std::move(buffer), buffer_offset, driver, transfer_size, max_batch_size, is_circular) :
- create_sg_edge_layer(std::move(buffer), buffer_offset, driver, transfer_size, max_batch_size, d2h_channel_id, is_circular);
+ create_ccb_edge_layer(buffer, buffer_offset, driver, transfer_size, max_batch_size, is_circular) :
+ create_sg_edge_layer(buffer, buffer_offset, driver, transfer_size, max_batch_size, d2h_channel_id, is_circular);
return buffer_exp;
}
Expected<IntermediateBuffer> IntermediateBuffer::create(HailoRTDriver &driver, uint32_t transfer_size,
uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type,
- std::shared_ptr<vdma::VdmaBuffer> &&buffer, size_t buffer_offset)
+ std::shared_ptr<vdma::VdmaBuffer> buffer, size_t buffer_offset)
{
- auto edge_layer_exp = create_edge_layer(std::move(buffer), buffer_offset, driver, transfer_size, max_batch_size,
- d2h_channel_id, streaming_type);
- CHECK_EXPECTED(edge_layer_exp);
- auto edge_layer_ptr = edge_layer_exp.release();
+ LOGGER__TRACE("Creating IntermediateBuffer: transfer_size = {}, max_batch_size = {}, d2h_channel_id = {}, "
+ "streaming_type = {}, buffer = 0x{:X}, buffer_offset = {}",
+ transfer_size, max_batch_size, d2h_channel_id, streaming_type, (uintptr_t)buffer.get(), buffer_offset);
+
+ TRY(auto edge_layer_ptr, create_edge_layer(buffer, buffer_offset, driver, transfer_size, max_batch_size,
+ d2h_channel_id, streaming_type));
if (streaming_type == StreamingType::BURST) {
// We have max_batch_size transfers, so we program them one by one. The last transfer should report interrupt
// to the device.
- size_t acc_offset = 0;
+ size_t desc_acc_offset = 0;
+ size_t buffer_acc_offset = 0;
for (uint16_t i = 0; i < max_batch_size; i++) {
const auto last_desc_interrupts_domain = ((max_batch_size - 1) == i) ?
InterruptsDomain::DEVICE : InterruptsDomain::NONE;
- auto desc_count_local = edge_layer_ptr->program_descriptors(transfer_size, last_desc_interrupts_domain, acc_offset);
- CHECK_EXPECTED(desc_count_local, "Failed to program descs for inter context channels. Given max_batch_size is too big.");
- acc_offset += desc_count_local.value();
+ TRY(const auto desc_count_local, edge_layer_ptr->program_descriptors(transfer_size,
+ last_desc_interrupts_domain, desc_acc_offset, buffer_acc_offset),
+ "Failed to program descs for inter context channels. Given max_batch_size is too big.");
+ desc_acc_offset += desc_count_local;
+ buffer_acc_offset += (desc_count_local * edge_layer_ptr->desc_page_size());
}
} else {
// Program all descriptors, no need for interrupt.
const auto interrupts_domain = InterruptsDomain::NONE;
const auto total_size = edge_layer_ptr->descs_count() * edge_layer_ptr->desc_page_size();
- auto desc_count_local = edge_layer_ptr->program_descriptors(total_size, interrupts_domain, 0);
- CHECK_EXPECTED(desc_count_local);
+ TRY(const auto desc_count_local, edge_layer_ptr->program_descriptors(total_size, interrupts_domain, 0));
+ (void)desc_count_local;
}
- return IntermediateBuffer(std::move(edge_layer_ptr), transfer_size, max_batch_size);
+ return IntermediateBuffer(std::move(edge_layer_ptr), transfer_size, streaming_type, max_batch_size);
+}
+
+Expected<std::shared_ptr<IntermediateBuffer>> IntermediateBuffer::create_shared(HailoRTDriver &driver,
+ uint32_t transfer_size, uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type,
+ std::shared_ptr<vdma::VdmaBuffer> buffer, size_t buffer_offset)
+{
+ TRY(auto intermediate_buffer, create(driver, transfer_size, max_batch_size, d2h_channel_id, streaming_type,
+ buffer, buffer_offset));
+
+ auto intermediate_buffer_ptr = make_shared_nothrow<IntermediateBuffer>(std::move(intermediate_buffer));
+ CHECK_NOT_NULL_AS_EXPECTED(intermediate_buffer_ptr, HAILO_OUT_OF_HOST_MEMORY);
+ return intermediate_buffer_ptr;
}
-Expected<Buffer> IntermediateBuffer::read()
+Expected<Buffer> IntermediateBuffer::read(size_t size)
{
- const auto size = m_transfer_size * m_dynamic_batch_size;
- assert(size <= m_edge_layer->size());
+ if (size == 0) {
+ size = m_transfer_size * m_dynamic_batch_size;
+ }
+ CHECK_AS_EXPECTED(size <= m_edge_layer->backing_buffer_size(), HAILO_INTERNAL_FAILURE,
+ "Requested size {} is bigger than buffer size {}", size, m_edge_layer->backing_buffer_size());
- auto res = Buffer::create(size);
- CHECK_EXPECTED(res);
+ TRY(auto res, Buffer::create(size));
- auto status = m_edge_layer->read(res->data(), size, 0);
+ auto status = m_edge_layer->read(res.data(), size, 0);
CHECK_SUCCESS_AS_EXPECTED(status);
- return res.release();
+ return res;
}
CONTROL_PROTOCOL__host_buffer_info_t IntermediateBuffer::get_host_buffer_info() const
return m_edge_layer->get_host_buffer_info(m_transfer_size);
}
+hailo_status IntermediateBuffer::reprogram_descriptors(size_t buffer_offset)
+{
+ CHECK(m_streaming_type == StreamingType::BURST, HAILO_INTERNAL_FAILURE,
+ "Reprogramming descriptors is only supported for burst streaming type");
+
+ CHECK(buffer_offset % m_edge_layer->desc_page_size() == 0, HAILO_INTERNAL_FAILURE,
+ "Buffer offset must be aligned to descriptor page size");
+
+ assert(m_edge_layer->backing_buffer_size() >= buffer_offset);
+ const auto size_to_end = m_edge_layer->backing_buffer_size() - buffer_offset;
+ const auto first_chunk_size = std::min(size_to_end, static_cast<size_t>(m_transfer_size));
+
+ // Program the first chunk of descriptors - from the buffer offset to the end of the buffer
+ static const auto BIND = true;
+ static const auto DESC_LIST_START = 0;
+ TRY(const uint32_t first_chunk_desc_count, m_edge_layer->program_descriptors(first_chunk_size,
+ InterruptsDomain::NONE, DESC_LIST_START, buffer_offset, BIND));
+
+ uint32_t second_chunk_desc_count = 0;
+ if (first_chunk_size < m_transfer_size) {
+ // Program the second chunk of descriptors - from the start of the buffer till the end of the remaining size
+ static const auto BUFFER_START = 0;
+ const auto second_chunk_size = m_transfer_size - first_chunk_size;
+ TRY(second_chunk_desc_count, m_edge_layer->program_descriptors(second_chunk_size, InterruptsDomain::NONE,
+ first_chunk_desc_count, BUFFER_START, BIND));
+ }
+
+ const auto expected_desc_count = m_edge_layer->descs_count() - 1;
+ CHECK(first_chunk_desc_count + second_chunk_desc_count == expected_desc_count, HAILO_INTERNAL_FAILURE,
+ "Expected {} descriptors, got {}", expected_desc_count, first_chunk_desc_count + second_chunk_desc_count);
+
+ return HAILO_SUCCESS;
+}
+
+uint32_t IntermediateBuffer::transfer_size() const
+{
+ return m_transfer_size;
+}
+
IntermediateBuffer::IntermediateBuffer(std::unique_ptr<vdma::VdmaEdgeLayer> &&edge_layer, uint32_t transfer_size,
- uint16_t batch_size) :
+ StreamingType streaming_type, uint16_t batch_size) :
m_edge_layer(std::move(edge_layer)),
m_transfer_size(transfer_size),
+ m_streaming_type(streaming_type),
m_dynamic_batch_size(batch_size)
{}
Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> IntermediateBuffer::create_sg_edge_layer(
- std::shared_ptr<vdma::VdmaBuffer> &&buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size,
+ std::shared_ptr<vdma::VdmaBuffer> buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size,
uint16_t batch_size, vdma::ChannelId d2h_channel_id, bool is_circular)
{
static const auto DONT_FORCE_DEFAULT_PAGE_SIZE = false;
static const auto FORCE_BATCH_SIZE = true;
static const auto IS_VDMA_ALIGNED_BUFFER = true;
- auto buffer_requirements = vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer(
+ TRY(const auto buffer_requirements, vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer(
vdma::VdmaBuffer::Type::SCATTER_GATHER, driver.desc_max_page_size(), batch_size, batch_size, transfer_size,
- is_circular, DONT_FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER);
- CHECK_EXPECTED(buffer_requirements);
- const auto desc_page_size = buffer_requirements->desc_page_size();
- const auto descs_count = buffer_requirements->descs_count();
- const auto buffer_size = buffer_requirements->buffer_size();
+ is_circular, DONT_FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER));
+ const auto desc_page_size = buffer_requirements.desc_page_size();
+ const auto descs_count = buffer_requirements.descs_count();
+ const auto buffer_size = buffer_requirements.buffer_size();
- auto edge_layer = vdma::SgEdgeLayer::create(std::dynamic_pointer_cast<vdma::SgBuffer>(buffer), buffer_size,
- buffer_offset, driver, descs_count, desc_page_size, is_circular, d2h_channel_id);
- CHECK_EXPECTED(edge_layer);
+ TRY(auto edge_layer, vdma::SgEdgeLayer::create(std::dynamic_pointer_cast<vdma::SgBuffer>(buffer), buffer_size,
+ buffer_offset, driver, descs_count, desc_page_size, is_circular, d2h_channel_id));
- auto edge_layer_ptr = make_unique_nothrow<vdma::SgEdgeLayer>(edge_layer.release());
+ auto edge_layer_ptr = make_unique_nothrow<vdma::SgEdgeLayer>(std::move(edge_layer));
CHECK_NOT_NULL_AS_EXPECTED(edge_layer_ptr, HAILO_OUT_OF_HOST_MEMORY);
return std::unique_ptr<vdma::VdmaEdgeLayer>(std::move(edge_layer_ptr));
}
-Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> IntermediateBuffer::create_ccb_edge_layer(std::shared_ptr<vdma::VdmaBuffer> &&buffer,
+Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> IntermediateBuffer::create_ccb_edge_layer(std::shared_ptr<vdma::VdmaBuffer> buffer,
size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t batch_size, bool is_circular)
{
static const auto DONT_FORCE_DEFAULT_PAGE_SIZE = false;
static const auto FORCE_BATCH_SIZE = true;
static const auto IS_VDMA_ALIGNED_BUFFER = true;
- auto buffer_size_requirements = vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer(
+ TRY(const auto buffer_size_requirements, vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer(
vdma::VdmaBuffer::Type::CONTINUOUS, driver.desc_max_page_size(), batch_size, batch_size, transfer_size,
- is_circular, DONT_FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER);
- CHECK_EXPECTED(buffer_size_requirements);
+ is_circular, DONT_FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER));
- const auto page_size = buffer_size_requirements->desc_page_size();
- const auto descs_count = buffer_size_requirements->descs_count();
- const auto buffer_size = buffer_size_requirements->buffer_size();
+ const auto page_size = buffer_size_requirements.desc_page_size();
+ const auto descs_count = buffer_size_requirements.descs_count();
+ const auto buffer_size = buffer_size_requirements.buffer_size();
- auto edge_layer = vdma::ContinuousEdgeLayer::create(std::dynamic_pointer_cast<vdma::ContinuousBuffer>(buffer),
- buffer_size, buffer_offset, page_size, descs_count);
- CHECK_EXPECTED(edge_layer);
+ TRY(auto edge_layer, vdma::ContinuousEdgeLayer::create(std::dynamic_pointer_cast<vdma::ContinuousBuffer>(buffer),
+ buffer_size, buffer_offset, page_size, descs_count));
- auto edge_layer_ptr = make_unique_nothrow<vdma::ContinuousEdgeLayer>(edge_layer.release());
+ auto edge_layer_ptr = make_unique_nothrow<vdma::ContinuousEdgeLayer>(std::move(edge_layer));
CHECK_NOT_NULL_AS_EXPECTED(edge_layer_ptr, HAILO_OUT_OF_HOST_MEMORY);
return std::unique_ptr<vdma::VdmaEdgeLayer>(std::move(edge_layer_ptr));
static Expected<IntermediateBuffer> create(HailoRTDriver &driver, uint32_t transfer_size,
uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type,
- std::shared_ptr<vdma::VdmaBuffer> &&buffer, size_t buffer_offset);
+ std::shared_ptr<vdma::VdmaBuffer> buffer, size_t buffer_offset);
+ static Expected<std::shared_ptr<IntermediateBuffer>> create_shared(HailoRTDriver &driver, uint32_t transfer_size,
+ uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type,
+ std::shared_ptr<vdma::VdmaBuffer> buffer, size_t buffer_offset);
- Expected<Buffer> read();
+ // If size is 0, the entire buffer is read (based on the transfer size passed in the create function)
+ Expected<Buffer> read(size_t size=0);
CONTROL_PROTOCOL__host_buffer_info_t get_host_buffer_info() const;
+ hailo_status reprogram_descriptors(size_t buffer_offset);
+ uint32_t transfer_size() const;
private:
- IntermediateBuffer(std::unique_ptr<vdma::VdmaEdgeLayer> &&buffer, uint32_t transfer_size, uint16_t batch_size);
+ IntermediateBuffer(std::unique_ptr<vdma::VdmaEdgeLayer> &&buffer, uint32_t transfer_size,
+ StreamingType streaming_type, uint16_t batch_size);
- static Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> create_sg_edge_layer(std::shared_ptr<vdma::VdmaBuffer> &&buffer,
+ static Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> create_sg_edge_layer(std::shared_ptr<vdma::VdmaBuffer> buffer,
size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t batch_size,
vdma::ChannelId d2h_channel_id, bool is_circular);
- static Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> create_ccb_edge_layer(std::shared_ptr<vdma::VdmaBuffer> &&buffer,
+ static Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> create_ccb_edge_layer(std::shared_ptr<vdma::VdmaBuffer> buffer,
size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t batch_size, bool is_circular);
- static Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> create_edge_layer(std::shared_ptr<vdma::VdmaBuffer> &&buffer,
+ static Expected<std::unique_ptr<vdma::VdmaEdgeLayer>> create_edge_layer(std::shared_ptr<vdma::VdmaBuffer> buffer,
size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t max_batch_size,
vdma::ChannelId d2h_channel_id, StreamingType streaming_type);
std::unique_ptr<vdma::VdmaEdgeLayer> m_edge_layer;
const uint32_t m_transfer_size;
+ const StreamingType m_streaming_type;
uint16_t m_dynamic_batch_size;
};
namespace hailort
{
-// Macros that check status. If status is HAILO_CANT_MEET_BUFFER_REQUIREMENTS, return without printing error to the prompt.
-#define CHECK_EXPECTED_OUT_OF_CMA_MEMORY(type) if (HAILO_OUT_OF_HOST_CMA_MEMORY == (type).status()) {return make_unexpected(HAILO_OUT_OF_HOST_CMA_MEMORY);} CHECK_SUCCESS(type);
-
Expected<std::shared_ptr<InternalBufferManager>> InternalBufferManager::create(HailoRTDriver &driver,
const ConfigureNetworkParams &config_params)
{
Expected<std::shared_ptr<vdma::VdmaBuffer>> InternalBufferManager::create_intermediate_sg_buffer(
const size_t buffer_size)
{
- auto buffer = vdma::SgBuffer::create(m_driver, buffer_size, HailoRTDriver::DmaDirection::BOTH);
- CHECK_EXPECTED(buffer);
+ TRY(auto buffer, vdma::SgBuffer::create(m_driver, buffer_size, HailoRTDriver::DmaDirection::BOTH));
- auto buffer_ptr = make_shared_nothrow<vdma::SgBuffer>(buffer.release());
+ auto buffer_ptr = make_shared_nothrow<vdma::SgBuffer>(std::move(buffer));
CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY);
return std::shared_ptr<vdma::VdmaBuffer>(std::move(buffer_ptr));
Expected<std::shared_ptr<vdma::VdmaBuffer>> InternalBufferManager::create_intermediate_ccb_buffer(
const size_t buffer_size)
{
- auto buffer = vdma::ContinuousBuffer::create(buffer_size, m_driver);
- CHECK_EXPECTED_OUT_OF_CMA_MEMORY(buffer);
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_OUT_OF_HOST_CMA_MEMORY, auto buffer,
+ vdma::ContinuousBuffer::create(buffer_size, m_driver));
- auto buffer_ptr = make_shared_nothrow<vdma::ContinuousBuffer>(buffer.release());
+ auto buffer_ptr = make_shared_nothrow<vdma::ContinuousBuffer>(std::move(buffer));
CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY);
return std::shared_ptr<vdma::VdmaBuffer>(std::move(buffer_ptr));
for (const auto &edge_layer_offset : buffer_plan.edge_layer_offsets) {
m_edge_layer_to_buffer_map.emplace(
edge_layer_offset.first,
- EdgeLayerToBufferMap{buffer_ptr.value(), edge_layer_offset.second});
+ EdgeLayerBuffer{buffer_ptr.value(), edge_layer_offset.second});
}
// Add edge layers to executed list
for (const auto &edge_layer_info : buffer_plan.edge_layer_infos) {
return execution_status;
}
-Expected<EdgeLayerToBufferMap> InternalBufferManager::get_intermediate_buffer(const EdgeLayerKey &key)
+ExpectedRef<EdgeLayerInfo> InternalBufferManager::get_layer_buffer_info(const EdgeLayerKey &key)
+{
+ const auto buffer_it = m_edge_layer_infos.find(key);
+ if (std::end(m_edge_layer_infos) == buffer_it) {
+ return make_unexpected(HAILO_NOT_FOUND);
+ }
+
+ return ExpectedRef<EdgeLayerInfo>(buffer_it->second);
+}
+
+Expected<EdgeLayerBuffer> InternalBufferManager::get_intermediate_buffer(const EdgeLayerKey &key)
{
const auto buffer_it = m_edge_layer_to_buffer_map.find(key);
if (std::end(m_edge_layer_to_buffer_map) == buffer_it) {
return make_unexpected(HAILO_NOT_FOUND);
}
- return Expected<EdgeLayerToBufferMap>(buffer_it->second);
+ return Expected<EdgeLayerBuffer>(buffer_it->second);
}
} /* namespace hailort */
hailo_status add_config_buffer_info(const uint16_t context_index, const size_t config_stream_index,
const std::vector<uint32_t> &cfg_sizes);
hailo_status add_layer_buffer_info(const LayerInfo &layer_info);
- Expected<EdgeLayerToBufferMap> get_intermediate_buffer(const EdgeLayerKey &key);
+ ExpectedRef<EdgeLayerInfo> get_layer_buffer_info(const EdgeLayerKey &key);
+ Expected<EdgeLayerBuffer> get_intermediate_buffer(const EdgeLayerKey &key);
hailo_status plan_and_execute(InternalBufferPlanner::Type default_planner_type, const size_t number_of_contexts);
+
private:
+ InternalBufferManager(HailoRTDriver &driver, const ConfigureNetworkParams &config_params);
// Add buffer info phase functions
void add_buffer_info(const EdgeLayerKey &edge_layer_key, const EdgeLayerInfo &buffer_info);
const ConfigureNetworkParams &m_config_params;
// m_edge_layer_infos is filled by add_buffer_info API
std::map<EdgeLayerKey, EdgeLayerInfo> m_edge_layer_infos;
-
- std::map<EdgeLayerKey, EdgeLayerToBufferMap> m_edge_layer_to_buffer_map;
-
- InternalBufferManager(HailoRTDriver &driver, const ConfigureNetworkParams &config_params);
+ std::map<EdgeLayerKey, EdgeLayerBuffer> m_edge_layer_to_buffer_map;
};
} /* namespace hailort */
constexpr size_t NAIVE_PLANNING_EDGE_LAYER_OFFSET = 0;
-// Macros that check status. If status is HAILO_CANT_MEET_BUFFER_REQUIREMENTS, return without printing error to the prompt.
-#define CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(type) if (HAILO_CANT_MEET_BUFFER_REQUIREMENTS == type.status()) {return make_unexpected(HAILO_CANT_MEET_BUFFER_REQUIREMENTS);} CHECK_SUCCESS(type);
+// Macro that check status. If status is HAILO_CANT_MEET_BUFFER_REQUIREMENTS, return without printing error to the prompt.
#define CHECK_STATUS_CANT_MEET_REQUIREMENTS(status) if (HAILO_CANT_MEET_BUFFER_REQUIREMENTS == status) {return make_unexpected(status);} CHECK_SUCCESS(status);
namespace hailort
else {
return true;
}
+ case LayerType::CACHE:
+ // Cache layers are always sg
+ return false;
default:
// Shouldn't reach here
assert(false);
edge_layer_offsets.emplace_back(edge_layer_info.first, NAIVE_PLANNING_EDGE_LAYER_OFFSET);
vdma::VdmaBuffer::Type buffer_type = should_edge_layer_use_ccb(edge_layer_info.second.type, dma_type, force_sg_buffer_type) ?
vdma::VdmaBuffer::Type::CONTINUOUS : vdma::VdmaBuffer::Type::SCATTER_GATHER;
- const auto buffer_requirements = return_buffer_requirements(edge_layer_info.second, buffer_type, max_page_size);
- CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(buffer_requirements);
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, const auto buffer_requirements,
+ return_buffer_requirements(edge_layer_info.second, buffer_type, max_page_size));
buffer_planning.emplace_back(
BufferPlan{
buffer_type,
- buffer_requirements->buffer_size(),
- buffer_requirements->buffer_size(),
+ buffer_requirements.buffer_size(),
+ buffer_requirements.buffer_size(),
edge_layer_offsets,
plan_edge_layer_infos});
}
std::vector<std::vector<BufferUsageSegment>> &context_buffer_usage_vector, BufferPlan &buffer_plan,
const vdma::VdmaBuffer::Type buffer_type, uint16_t max_page_size)
{
- const auto buffer_requirements = return_buffer_requirements(edge_layer.second, buffer_type, max_page_size);
- CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(buffer_requirements);
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, const auto buffer_requirements,
+ return_buffer_requirements(edge_layer.second, buffer_type, max_page_size));
// Check if there is enough space in the current context buffer.
const auto start_context = edge_layer.second.start_context;
const auto end_context = edge_layer.second.end_context;
const auto buffer_map = build_availibility_map(context_buffer_usage_vector, start_context, end_context);
- const auto edge_layer_size = buffer_requirements->buffer_size();
- const auto buffer_offset_alignment = buffer_requirements->desc_page_size();
+ const auto edge_layer_size = buffer_requirements.buffer_size();
+ const auto buffer_offset_alignment = buffer_requirements.desc_page_size();
const auto buffer_offset = find_new_buffer_offset(buffer_map, edge_layer_size, buffer_offset_alignment);
auto end_of_edge_layer_offset = buffer_offset + edge_layer_size;
InternalBufferPlanning buffer_planning;
// Second - create buffer planning for each buffer type
if (!ccb_edge_layers.empty()) {
- auto ccb_buffer_planning =
- create_single_buffer_planning(ccb_edge_layers, number_of_contexts, vdma::VdmaBuffer::Type::CONTINUOUS, max_page_size);
- CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(ccb_buffer_planning);
- buffer_planning.insert(buffer_planning.end(), ccb_buffer_planning->begin(), ccb_buffer_planning->end());
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, const auto ccb_buffer_planning,
+ create_single_buffer_planning(ccb_edge_layers, number_of_contexts, vdma::VdmaBuffer::Type::CONTINUOUS, max_page_size));
+ buffer_planning.insert(buffer_planning.end(), ccb_buffer_planning.begin(), ccb_buffer_planning.end());
}
if (!sg_edge_layers.empty()) {
- auto sg_buffer_planning =
- create_single_buffer_planning(sg_edge_layers, number_of_contexts, vdma::VdmaBuffer::Type::SCATTER_GATHER, max_page_size);
- CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(sg_buffer_planning);
- buffer_planning.insert(buffer_planning.end(), sg_buffer_planning->begin(), sg_buffer_planning->end());
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, auto sg_buffer_planning,
+ create_single_buffer_planning(sg_edge_layers, number_of_contexts, vdma::VdmaBuffer::Type::SCATTER_GATHER, max_page_size));
+ buffer_planning.insert(buffer_planning.end(), sg_buffer_planning.begin(), sg_buffer_planning.end());
}
return buffer_planning;
{
TRY(auto edge_layer_info, get_edge_info_from_buffer_plan(buffer_planning, edge_layer_key));
for (auto &buffer_plan : buffer_planning) {
- const auto buffer_requirements = return_buffer_requirements(edge_layer_info, buffer_plan.buffer_type, max_page_size);
- CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(buffer_requirements);
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, const auto buffer_requirements,
+ return_buffer_requirements(edge_layer_info, buffer_plan.buffer_type, max_page_size));
for (auto &edge_layer_offset : buffer_plan.edge_layer_offsets) {
if (edge_layer_offset.first == edge_layer_key) {
edge_layer_offset.second = new_offset;
- if (edge_layer_offset.second + buffer_requirements->buffer_size() > buffer_plan.buffer_size) {
- buffer_plan.buffer_size = edge_layer_offset.second + buffer_requirements->buffer_size();
+ if (edge_layer_offset.second + buffer_requirements.buffer_size() > buffer_plan.buffer_size) {
+ buffer_plan.buffer_size = edge_layer_offset.second + buffer_requirements.buffer_size();
}
return HAILO_SUCCESS;
}
bool reuse_buffer;
};
-struct EdgeLayerToBufferMap {
+struct EdgeLayerBuffer {
std::shared_ptr<vdma::VdmaBuffer> buffer;
size_t offset;
};
// confgured periph registers will add / removed the extra padding
if (is_core_hw_padding_config_in_dfc) {
if (0 != (periph_frame_size % PERIPH_FRAME_ALIGNMENT)) {
- auto max_periph_padding_payload = HefConfigurator::max_periph_padding_payload_value(
- DeviceBase::hef_arch_to_device_arch(hw_arch));
- CHECK_EXPECTED(max_periph_padding_payload);
+ TRY(const auto max_periph_padding_payload, HefConfigurator::max_periph_padding_payload_value(
+ DeviceBase::hef_arch_to_device_arch(hw_arch)));
// Currently case of payload larger than max periph padding payload value - not supported
- CHECK_AS_EXPECTED(max_periph_padding_payload.value() > periph_frame_size, HAILO_INVALID_HEF,
- "Error, padded frame size larger than {} Currently not supported", max_periph_padding_payload.value());
+ CHECK_AS_EXPECTED(max_periph_padding_payload > periph_frame_size, HAILO_INVALID_HEF,
+ "Error, padded frame size larger than {} Currently not supported", max_periph_padding_payload);
const auto padded_periph_frame_size = HailoRTCommon::align_to(periph_frame_size,
static_cast<uint32_t>(PERIPH_FRAME_ALIGNMENT));
const uint32_t desc_page_size, const bool is_periph_calculated_in_hailort, const HEFHwArch &hw_arch,
const bool is_core_hw_padding_config_in_dfc)
{
- auto max_periph_bytes_from_hef = HefConfigurator::max_periph_bytes_value(DeviceBase::hef_arch_to_device_arch(hw_arch));
- CHECK_EXPECTED(max_periph_bytes_from_hef);
- const auto max_periph_bytes = std::min(max_periph_bytes_from_hef.value(), layer_info.max_shmifo_size);
+ TRY(const auto max_periph_bytes_from_hef, HefConfigurator::max_periph_bytes_value(DeviceBase::hef_arch_to_device_arch(hw_arch)));
+ const auto max_periph_bytes = std::min(max_periph_bytes_from_hef, layer_info.max_shmifo_size);
// If extension for calculating periph values in hailort is false and core hw padding is not supported - copy values from
// Core registers, otherwise calculate them according to shape and other layer information
const bool hw_padding_supported = HefConfigurator::is_core_hw_padding_supported(layer_info, max_periph_bytes,
std::vector<ConfigBuffer> config_buffers;
config_buffers.reserve(config_buffer_infos.size());
for (uint8_t config_stream_index = 0; config_stream_index < config_buffer_infos.size(); config_stream_index++) {
- auto buffer_resource = ConfigBuffer::create(driver, config_channels_ids[config_stream_index],
- config_buffer_infos.at(config_stream_index));
- CHECK_EXPECTED(buffer_resource);
- config_buffers.emplace_back(buffer_resource.release());
+ TRY(auto buffer_resource, ConfigBuffer::create(driver, config_channels_ids[config_stream_index],
+ config_buffer_infos.at(config_stream_index).bursts_sizes));
+ config_buffers.emplace_back(std::move(buffer_resource));
- internal_buffer_manager->add_config_buffer_info(context_index, config_stream_index, config_buffer_infos.at(config_stream_index));
+ internal_buffer_manager->add_config_buffer_info(context_index, config_stream_index,
+ config_buffer_infos.at(config_stream_index).bursts_sizes);
}
return ContextResources(driver, context_type, std::move(config_buffers), internal_buffer_manager);
// Best effort for starting latency meter.
auto networks_names = core_op_metadata->get_network_names();
for (auto &network_name : networks_names) {
- auto layer_infos = core_op_metadata->get_all_layer_infos(network_name);
- CHECK_EXPECTED(layer_infos);
- auto latency_meter = create_hw_latency_meter(layer_infos.value());
- if (latency_meter) {
- latency_meters_map.emplace(network_name, latency_meter.release());
- LOGGER__DEBUG("Starting hw latency measurement for network {}", network_name);
- }
+ TRY(const auto layer_infos, core_op_metadata->get_all_layer_infos(network_name));
+ TRY(auto latency_meter, create_hw_latency_meter(layer_infos));
+ latency_meters_map.emplace(network_name, latency_meter);
+ LOGGER__DEBUG("Starting hw latency measurement for network {}", network_name);
}
}
}
Expected<ResourcesManager> ResourcesManager::create(VdmaDevice &vdma_device, HailoRTDriver &driver,
- const ConfigureNetworkParams &config_params, std::shared_ptr<CoreOpMetadata> core_op_metadata,
- uint8_t core_op_index)
+ const ConfigureNetworkParams &config_params, CacheManagerPtr cache_manager,
+ std::shared_ptr<CoreOpMetadata> core_op_metadata, uint8_t core_op_index)
{
// Allocate config channels. In order to use the same channel ids for config channels in all contexts,
// we allocate all of them here, and use in preliminary/dynamic context.
for (uint8_t cfg_index = 0; cfg_index < config_channels_info.size(); cfg_index++) {
const auto layer_identifier = std::make_tuple(LayerType::CFG, HAILO_H2D_STREAM, "", cfg_index);
const auto engine_index = config_channels_info[cfg_index].engine_index;
- auto channel_id = allocator.get_available_channel_id(layer_identifier, HailoRTDriver::DmaDirection::H2D, engine_index);
- CHECK_EXPECTED(channel_id);
- config_channels_ids.push_back(channel_id.release());
+ TRY(const auto channel_id,
+ allocator.get_available_channel_id(layer_identifier, HailoRTDriver::DmaDirection::H2D, engine_index));
+ config_channels_ids.push_back(channel_id);
}
- auto internal_buffer_manager = InternalBufferManager::create(driver, config_params);
- CHECK_EXPECTED(internal_buffer_manager);
-
- auto action_list_buffer_builder = create_action_list_buffer_builder(core_op_metadata->dynamic_contexts().size(),
- driver);
- CHECK_EXPECTED(action_list_buffer_builder);
-
+ TRY(auto internal_buffer_manager, InternalBufferManager::create(driver, config_params));
+ TRY_V(auto action_list_buffer_builder, create_action_list_buffer_builder(core_op_metadata->dynamic_contexts().size(),
+ vdma_device));
+ TRY(auto latency_meters, create_latency_meters_from_config_params(config_params, core_op_metadata));
auto network_index_map = core_op_metadata->get_network_names();
- auto latency_meters = create_latency_meters_from_config_params(config_params, core_op_metadata);
- CHECK_EXPECTED(latency_meters);
- ResourcesManager resources_manager(vdma_device, driver, std::move(allocator), config_params,
- std::move(core_op_metadata), core_op_index, std::move(network_index_map), latency_meters.release(),
- std::move(config_channels_ids), internal_buffer_manager.release(), action_list_buffer_builder.release());
+ ResourcesManager resources_manager(vdma_device, driver, std::move(allocator), config_params, cache_manager,
+ std::move(core_op_metadata), core_op_index, std::move(network_index_map), std::move(latency_meters),
+ std::move(config_channels_ids), internal_buffer_manager, std::move(action_list_buffer_builder));
return resources_manager;
}
ResourcesManager::ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &driver,
ChannelAllocator &&channel_allocator, const ConfigureNetworkParams config_params,
- std::shared_ptr<CoreOpMetadata> &&core_op_metadata,
+ CacheManagerPtr cache_manager, std::shared_ptr<CoreOpMetadata> &&core_op_metadata,
uint8_t core_op_index, const std::vector<std::string> &&network_index_map,
LatencyMetersMap &&latency_meters,
std::vector<vdma::ChannelId> &&config_channels_ids,
m_vdma_device(vdma_device),
m_driver(driver),
m_config_params(config_params),
+ m_cache_manager(cache_manager),
m_intermediate_buffers(),
m_core_op_metadata(std::move(core_op_metadata)),
m_core_op_index(core_op_index),
m_total_context_count(0),
m_network_index_map(std::move(network_index_map)),
m_latency_meters(std::move(latency_meters)),
- m_boundary_channels(),
m_is_configured(false),
m_is_activated(false),
m_config_channels_ids(std::move(config_channels_ids)),
m_vdma_device(other.m_vdma_device),
m_driver(other.m_driver),
m_config_params(other.m_config_params),
+ m_cache_manager(std::move(other.m_cache_manager)),
m_intermediate_buffers(std::move(other.m_intermediate_buffers)),
m_core_op_metadata(std::move(other.m_core_op_metadata)),
m_core_op_index(other.m_core_op_index),
for (network_index = 0; network_index < m_network_index_map.size(); network_index++) {
auto const network_name_from_map = m_network_index_map[network_index];
if (network_name_from_map == network_name_from_params) {
- auto batch_size = get_network_batch_size(network_name_from_params);
- CHECK_EXPECTED_AS_STATUS(batch_size);
- app_header.batch_size[network_index] = batch_size.value();
+ TRY(const auto batch_size, get_network_batch_size(network_name_from_params));
+ app_header.batch_size[network_index] = batch_size;
break;
}
}
return HAILO_SUCCESS;
}
-void ResourcesManager::process_interrupts(IrqData &&irq_data)
-{
- assert(irq_data.channels_count <= ARRAY_ENTRIES(irq_data.channels_irq_data));
- for (uint8_t irq_index = 0; irq_index < irq_data.channels_count; irq_index++) {
- const auto &channel_irq_data = irq_data.channels_irq_data[irq_index];
- auto boundary_channel = m_boundary_channels.find(channel_irq_data.channel_id);
- if (std::end(m_boundary_channels) == boundary_channel) {
- LOGGER__ERROR("Got interrupt for channel {}, but there is no such boundary channel", channel_irq_data.channel_id);
- continue;
- }
-
- if ((channel_irq_data.host_error != 0) || (channel_irq_data.device_error != 0)) {
- LOGGER__CRITICAL("Got error on channel {} host_error=0x{:x} device_error=0x{:x}",
- channel_irq_data.channel_id, channel_irq_data.host_error, channel_irq_data.device_error);
- continue;
- }
-
- if (!channel_irq_data.validation_success) {
- LOGGER__CRITICAL("Got validation error on channel {}", channel_irq_data.channel_id);
- continue;
- }
-
- if (!channel_irq_data.is_active) {
- LOGGER__CRITICAL("Channel {} was aborted by external source", channel_irq_data.channel_id);
- continue;
- }
-
- auto status = boundary_channel->second->trigger_channel_completion(channel_irq_data.desc_num_processed);
- if ((status != HAILO_SUCCESS) &&
- (status != HAILO_STREAM_ABORT) &&
- (status != HAILO_STREAM_NOT_ACTIVATED)) {
- // Log error and continue gracefully to process other interrupts
- LOGGER__ERROR("Trigger channel completion failed on channel {} with status {}", channel_irq_data.channel_id, status);
- }
- }
-}
-
Expected<uint16_t> ResourcesManager::get_batch_size() const
{
uint16_t batch_size = UINT16_MAX;
}
+std::pair<size_t, size_t> ResourcesManager::calculate_transfer_queue_sizes(const vdma::DescriptorList &desc_list,
+ uint32_t transfer_size, uint32_t max_active_trans, bool use_latency_meter)
+{
+ // Calculate m_ongoing_transfers capacity - transfers that are already bound to the descriptor list
+ // Add desc for boundary channel because might need extra for non aligned async API
+ // We don't use get_max_aligned_transfers_in_desc_list because we want to include the option of a bounce buffer
+ static const auto INCLUDE_BOUNCE_BUFFER = true;
+ const size_t max_transfers_in_desc_list = desc_list.max_transfers(transfer_size, INCLUDE_BOUNCE_BUFFER);
+
+ // Max capacity due to driver constraints (see HAILO_VDMA_MAX_ONGOING_TRANSFERS)
+ const size_t max_ongoing_transfers_capacity = (use_latency_meter ?
+ (ONGOING_TRANSFERS_SIZE / 2) : ONGOING_TRANSFERS_SIZE) - 1;
+
+ const auto ongoing_transfers = std::min(max_transfers_in_desc_list, max_ongoing_transfers_capacity);
+
+ // We want to allow max_active_trans transfers in m_pending_transfers
+ // * If the transfers can all fit in m_ongoing_transfers, we don't need to use m_pending_transfers so we set it
+ // to 0. In this case, all transfers will be handled via m_ongoing_transfers and each time launch_transfer is
+ // called, the transfer will be launched immediately.
+ // * Otherwise, we set it to max_active_trans. In this case, we will use m_pending_transfers to queue up
+ // transfers that can't fit in m_ongoing_transfers. We will then launch them as soon as there is room in
+ // m_ongoing_transfers, via the transfer launcher.
+ // TODO: Bring back commented out impl bellow (HRT-13644)
+ // Setting pending_transfers to zero, s.t. the pending transfer queue won't be used.
+ (void)max_active_trans;
+ const auto pending_transfers = 0;
+ // const auto pending_transfers = (max_active_trans > ongoing_transfers) ? max_active_trans : 0;
+
+ return std::make_pair(ongoing_transfers, pending_transfers);
+}
+
hailo_status ResourcesManager::create_boundary_vdma_channel(const LayerInfo &layer_info)
{
// TODO: put in layer info
const auto channel_direction = layer_info.direction == HAILO_H2D_STREAM ? HailoRTDriver::DmaDirection::H2D :
HailoRTDriver::DmaDirection::D2H;
- const auto channel_id = get_available_channel_id(to_layer_identifier(layer_info),
- channel_direction, layer_info.dma_engine_index);
- CHECK_EXPECTED_AS_STATUS(channel_id);
-
- const auto network_batch_size = get_network_batch_size(layer_info.network_name);
- CHECK_EXPECTED_AS_STATUS(network_batch_size);
+ TRY(const auto channel_id, get_available_channel_id(to_layer_identifier(layer_info),
+ channel_direction, layer_info.dma_engine_index));
+ TRY(const auto network_batch_size, get_network_batch_size(layer_info.network_name));
const auto nms_max_detections_per_frame =
- layer_info.nms_info.number_of_classes * layer_info.nms_info.max_bboxes_per_class * layer_info.nms_info.chunks_per_frame;
+ layer_info.nms_info.number_of_classes * layer_info.nms_info.max_bboxes_per_class * layer_info.nms_info.chunks_per_frame;
const auto max_active_transfers_scale = (layer_info.format.order == HAILO_FORMAT_ORDER_HAILO_NMS) ?
(nms_max_detections_per_frame * MAX_ACTIVE_TRANSFERS_SCALE) : MAX_ACTIVE_TRANSFERS_SCALE;
- auto device_arch = m_vdma_device.get_architecture();
- CHECK_EXPECTED_AS_STATUS(device_arch);
+ TRY(const auto device_arch, m_vdma_device.get_architecture());
/* Add error in configure phase for invalid NMS parameters */
- if (layer_info.format.order == HAILO_FORMAT_ORDER_HAILO_NMS &&
- (device_arch.value() == HAILO_ARCH_HAILO15H || device_arch.value() == HAILO_ARCH_HAILO15M || device_arch.value() == HAILO_ARCH_PLUTO)) {
- CHECK(layer_info.nms_info.number_of_classes * layer_info.nms_info.chunks_per_frame * network_batch_size.value() < HAILO15H_NMS_MAX_CLASSES,
- HAILO_INVALID_ARGUMENT, "Invalid NMS parameters. Number of classes ({}) * division factor ({}) * batch size ({}) must be under {}",
- layer_info.nms_info.number_of_classes, layer_info.nms_info.chunks_per_frame, network_batch_size.value(), HAILO15H_NMS_MAX_CLASSES);
+ if ((layer_info.format.order == HAILO_FORMAT_ORDER_HAILO_NMS) && (HailoRTCommon::is_hailo1x_device_type(device_arch))) {
+ CHECK(layer_info.nms_info.number_of_classes * layer_info.nms_info.chunks_per_frame * network_batch_size < HAILO15H_NMS_MAX_CLASSES,
+ HAILO_INVALID_ARGUMENT, "Invalid NMS parameters. Number of classes ({}) * division factor ({}) * batch size ({}) must be under {}",
+ layer_info.nms_info.number_of_classes, layer_info.nms_info.chunks_per_frame, network_batch_size, HAILO15H_NMS_MAX_CLASSES);
}
- const auto min_active_trans = MIN_ACTIVE_TRANSFERS_SCALE * network_batch_size.value();
+ const auto min_active_trans = MIN_ACTIVE_TRANSFERS_SCALE * network_batch_size;
const auto max_active_trans = (layer_info.format.order == HAILO_FORMAT_ORDER_HAILO_NMS) ?
/* NMS Case - Value be be higher than UINT16_MAX. in this case we only limit to UART16_MAX with no error */
- std::min(static_cast<uint32_t>(UINT16_MAX), max_active_transfers_scale * network_batch_size.value()) :
- max_active_transfers_scale * network_batch_size.value();
+ std::min(static_cast<uint32_t>(UINT16_MAX), max_active_transfers_scale * network_batch_size) :
+ max_active_transfers_scale * network_batch_size;
CHECK(IS_FIT_IN_UINT16(min_active_trans), HAILO_INVALID_ARGUMENT,
"calculated min_active_trans for vdma descriptor list is out of UINT16 range");
DONT_FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER);
if (HAILO_CANT_MEET_BUFFER_REQUIREMENTS == buffer_sizes_requirements.status()) {
LOGGER__ERROR("Network shapes and batch size exceeds driver descriptors capabilities. "
- "(A common cause for this error could be the batch size - which is {}).", network_batch_size.value());
+ "(A common cause for this error could be the batch size - which is {}).", network_batch_size);
}
- CHECK_EXPECTED_AS_STATUS(buffer_sizes_requirements);
+ CHECK_EXPECTED_AS_STATUS(buffer_sizes_requirements); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
const auto page_size = buffer_sizes_requirements->desc_page_size();
const auto descs_count = (nullptr != std::getenv("HAILO_CONFIGURE_FOR_HW_INFER")) ?
MAX_SG_DESCS_COUNT : buffer_sizes_requirements->descs_count();
- auto channel = vdma::BoundaryChannel::create(channel_id.value(), channel_direction, m_driver, descs_count,
- page_size, layer_info.name, latency_meter);
- CHECK_EXPECTED_AS_STATUS(channel);
+ const bool CIRCULAR = true;
+ TRY(auto desc_list, vdma::DescriptorList::create(descs_count, page_size, CIRCULAR, m_driver));
- m_boundary_channels.emplace(channel_id.value(), channel.release());
+ size_t pending_transfers = 0, ongoing_transfers = 0;
+ std::tie(ongoing_transfers, pending_transfers) = calculate_transfer_queue_sizes(desc_list, transfer_size,
+ max_active_trans, (latency_meter != nullptr));
+
+ TRY(auto vdma_transfer_launcher, m_vdma_device.get_vdma_transfer_launcher());
+ TRY(auto channel, vdma::BoundaryChannel::create(m_driver, channel_id, channel_direction, std::move(desc_list),
+ vdma_transfer_launcher.get(), ongoing_transfers, pending_transfers, layer_info.name, latency_meter));
+
+ m_boundary_channels.add_channel(std::move(channel));
return HAILO_SUCCESS;
}
Expected<vdma::BoundaryChannelPtr> ResourcesManager::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name)
{
- for (const auto &boundary_channel : m_boundary_channels) {
- if (boundary_channel.second->stream_name() == stream_name) {
- return vdma::BoundaryChannelPtr(boundary_channel.second);
- }
- }
-
- return make_unexpected(HAILO_NOT_FOUND);
+ return m_boundary_channels.get_by_name(stream_name);
}
Expected<std::shared_ptr<const vdma::BoundaryChannel>> ResourcesManager::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) const
{
- for (const auto &boundary_channel : m_boundary_channels) {
- if (boundary_channel.second->stream_name() == stream_name) {
- return std::shared_ptr<const vdma::BoundaryChannel>(boundary_channel.second);
- }
- }
-
- return make_unexpected(HAILO_NOT_FOUND);
+ return const_cast<vdma::ChannelsGroup &>(m_boundary_channels).get_by_name(stream_name);
}
hailo_power_mode_t ResourcesManager::get_power_mode() const
auto edge_layer_key = std::make_pair(src_context_index, src_stream_index);
TRY(auto buffer_info, m_internal_buffer_manager->get_intermediate_buffer(edge_layer_key));
- auto intermediate_buffer = IntermediateBuffer::create(m_driver, transfer_size, batch_size, d2h_channel_id,
- streaming_type, std::move(buffer_info.buffer), buffer_info.offset);
- CHECK_EXPECTED(intermediate_buffer);
+ TRY(auto intermediate_buffer, IntermediateBuffer::create(m_driver, transfer_size, batch_size, d2h_channel_id,
+ streaming_type, buffer_info.buffer, buffer_info.offset));
const auto key = std::make_pair(src_context_index, src_stream_index);
- auto emplace_res = m_intermediate_buffers.emplace(key, intermediate_buffer.release());
+ auto emplace_res = m_intermediate_buffers.emplace(key, std::move(intermediate_buffer));
return std::ref(emplace_res.first->second);
}
return std::ref(buffer_it->second);
}
+ExpectedRef<IntermediateBuffer> ResourcesManager::set_cache_input_channel(uint32_t cache_id, uint16_t batch_size,
+ vdma::ChannelId channel_id)
+{
+ return m_cache_manager->set_cache_input_channel(cache_id, batch_size, channel_id);
+}
+
+ExpectedRef<IntermediateBuffer> ResourcesManager::set_cache_output_channel(uint32_t cache_id, uint16_t batch_size,
+ vdma::ChannelId channel_id)
+{
+ return m_cache_manager->set_cache_output_channel(cache_id, batch_size, channel_id);
+}
+
+std::unordered_map<uint32_t, CacheBuffer> &ResourcesManager::get_cache_buffers()
+{
+ return m_cache_manager->get_cache_buffers();
+}
+
Expected<CONTROL_PROTOCOL__application_header_t> ResourcesManager::get_control_core_op_header()
{
CONTROL_PROTOCOL__application_header_t app_header{};
{
CHECK_AS_EXPECTED(m_total_context_count < std::numeric_limits<uint16_t>::max(), HAILO_INVALID_CONTEXT_COUNT);
- auto context_resources = ContextResources::create(m_driver, context_type, context_index,
- m_config_channels_ids, config_info, m_internal_buffer_manager);
- CHECK_EXPECTED(context_resources);
-
- m_contexts_resources.emplace_back(context_resources.release());
+ TRY(auto context_resources, ContextResources::create(m_driver, context_type, context_index,
+ m_config_channels_ids, config_info, m_internal_buffer_manager));
+ m_contexts_resources.emplace_back(std::move(context_resources));
m_total_context_count++;
if (CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC == context_type) {
m_dynamic_context_count++;
return intermediate_buffer_it->second.read();
}
+Expected<Buffer> ResourcesManager::read_cache_buffer(uint32_t cache_id)
+{
+ auto &cache_buffers_map = m_cache_manager->get_cache_buffers();
+ auto cache_buffer_it = cache_buffers_map.find(cache_id);
+ CHECK_AS_EXPECTED(std::end(cache_buffers_map) != cache_buffer_it, HAILO_NOT_FOUND,
+ "Failed to find cache buffer for cache_id {}", cache_id);
+ return cache_buffer_it->second.read_entire_cache();
+}
+
+Expected<std::map<uint32_t, Buffer>> ResourcesManager::read_cache_buffers()
+{
+ std::map<uint32_t, Buffer> result;
+ for (auto &cache_buffer : m_cache_manager->get_cache_buffers()) {
+ TRY(auto buffer, cache_buffer.second.read_entire_cache());
+ result.emplace(cache_buffer.first, std::move(buffer));
+ }
+
+ return result;
+}
+
hailo_status ResourcesManager::configure()
{
CHECK(!m_is_configured, HAILO_INTERNAL_FAILURE, "Can't configure the same core-op twice");
m_is_configured = true;
- auto core_op_header = get_control_core_op_header();
- CHECK_EXPECTED_AS_STATUS(core_op_header);
-
- auto status = Control::context_switch_set_network_group_header(m_vdma_device, core_op_header.release());
+ TRY(const auto core_op_header, get_control_core_op_header());
+ auto status = Control::context_switch_set_network_group_header(m_vdma_device, core_op_header);
CHECK_SUCCESS(status);
// Only send controls to FW in case of control action list builder
{
CHECK(get_is_activated(), HAILO_INTERNAL_FAILURE, "Cannot call start_vdma_interrupts_dispatcher when core-op already deactivated");
- auto interrupts_dispatcher = m_vdma_device.get_vdma_interrupts_dispatcher();
- CHECK_EXPECTED_AS_STATUS(interrupts_dispatcher);
-
- ChannelsBitmap channels_bitmap{};
- for (const auto &boundary_channel : m_boundary_channels) {
- const auto channel_id = boundary_channel.first;
- channels_bitmap[channel_id.engine_index] |= (1 << channel_id.channel_index);
- }
-
- const bool enable_timestamp_measure = !m_latency_meters.empty();
- return interrupts_dispatcher->get().start(channels_bitmap, enable_timestamp_measure, [this](IrqData &&irq_data){
- process_interrupts(std::move(irq_data));
- });
+ TRY(auto interrupts_dispatcher, m_vdma_device.get_vdma_interrupts_dispatcher());
+ return interrupts_dispatcher.get().start(m_boundary_channels);
}
hailo_status ResourcesManager::stop_vdma_interrupts_dispatcher()
return HAILO_SUCCESS;
}
-Expected<uint16_t> ResourcesManager::program_desc_for_hw_only_flow(std::shared_ptr<vdma::DescriptorList> desc_list,
+Expected<uint16_t> ResourcesManager::program_desc_for_hw_only_flow(vdma::DescriptorList &desc_list,
+ vdma::MappedBuffer &mapped_buffer, vdma::ChannelId channel_id,
const uint32_t single_transfer_size, const uint16_t dynamic_batch_size, const uint16_t batch_count)
{
size_t acc_desc_offset = 0;
for (uint16_t transfer_index = 0; transfer_index < dynamic_batch_size; transfer_index++) {
const auto last_desc_interrupts_domain = ((dynamic_batch_size - 1) == transfer_index) ?
InterruptsDomain::DEVICE : InterruptsDomain::NONE;
- auto desc_count_local = desc_list->program_last_descriptor(single_transfer_size,
- last_desc_interrupts_domain, acc_desc_offset);
- CHECK_EXPECTED(desc_count_local, "Failed to program descs for inter context channels. Given max_batch_size is too big.");
- acc_desc_offset += desc_count_local.value();
+ const bool should_bind = false;
+ CHECK_SUCCESS(desc_list.program(mapped_buffer, single_transfer_size,
+ (acc_desc_offset * desc_list.desc_page_size()), channel_id, static_cast<uint32_t>(acc_desc_offset),
+ should_bind, last_desc_interrupts_domain));
+ acc_desc_offset += desc_list.descriptors_in_buffer(single_transfer_size);
}
}
CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(acc_desc_offset), HAILO_INTERNAL_FAILURE,
{
const auto total_frames_per_run = dynamic_batch_size * batch_count;
- auto desc_list = boundary_channel_ptr->get_desc_list();
- const auto descs_per_transfer = desc_list->descriptors_in_buffer(single_transfer_size);
+ auto &desc_list = boundary_channel_ptr->get_desc_list();
+ const auto descs_per_transfer = desc_list.descriptors_in_buffer(single_transfer_size);
const auto total_desc_count = total_frames_per_run * descs_per_transfer;
CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(total_desc_count), HAILO_INVALID_ARGUMENT,
"calculated total_desc_count for vdma descriptor list is out of UINT16 range");
- auto mapped_buffer = vdma::MappedBuffer::create_shared_by_allocation(
- total_desc_count * desc_list->desc_page_size(), m_driver, direction);
- CHECK_EXPECTED(mapped_buffer);
- m_hw_only_boundary_buffers.emplace_back(mapped_buffer.release());
+ TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared_by_allocation(
+ total_desc_count * desc_list.desc_page_size(), m_driver, direction));
+ m_hw_only_boundary_buffers.emplace_back(std::move(mapped_buffer));
static const auto DEFAULT_BUFFER_OFFSET = 0;
- auto status = desc_list->configure_to_use_buffer(*m_hw_only_boundary_buffers.back(),
+ auto status = desc_list.program(*m_hw_only_boundary_buffers.back(),
m_hw_only_boundary_buffers.back()->size(), DEFAULT_BUFFER_OFFSET, boundary_channel_ptr->get_channel_id());
CHECK_SUCCESS_AS_EXPECTED(status);
- auto desc_programed = program_desc_for_hw_only_flow(desc_list, single_transfer_size, dynamic_batch_size, batch_count);
- CHECK_EXPECTED(desc_programed);
- assert(static_cast<uint16_t>(total_desc_count) == desc_programed.value());
+ TRY(auto desc_programed,
+ program_desc_for_hw_only_flow(desc_list, *m_hw_only_boundary_buffers.back(), boundary_channel_ptr->get_channel_id(), single_transfer_size, dynamic_batch_size, batch_count));
+ assert(static_cast<uint16_t>(total_desc_count) == desc_programed);
- auto channel_info_pair = std::make_pair(boundary_channel_ptr->get_channel_id(), desc_programed.release());
+ auto channel_info_pair = std::make_pair(boundary_channel_ptr->get_channel_id(), desc_programed);
return channel_info_pair;
}
const auto &stream_infos = LayerInfoUtils::get_stream_infos_from_layer_info(layer_info);
for (auto &stream_info : stream_infos) {
uint32_t single_transfer_size = LayerInfoUtils::get_stream_transfer_size(stream_info, layer_info);
- auto boundary_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(layer_info.name);
- CHECK_EXPECTED(boundary_channel_ptr_exp);
- auto boundary_channel_ptr = boundary_channel_ptr_exp.release();
- const auto max_batch_transfers = boundary_channel_ptr->get_desc_list()->max_transfers(single_transfer_size * dynamic_batch_size);
+ TRY(auto boundary_channel_ptr, get_boundary_vdma_channel_by_stream_name(layer_info.name));
+ const auto max_batch_transfers = boundary_channel_ptr->get_desc_list().max_transfers(single_transfer_size * dynamic_batch_size);
// infer batch count is the lowest number of "Max transfers" per descriptor list that for all given boundary channels.
batch_count = MIN(batch_count, max_batch_transfers);
}
channels_info.channel_count = 0;
static constexpr auto INFER_TIMEOUT = std::chrono::milliseconds(120000);
- auto batch_size = get_batch_size();
- CHECK_EXPECTED(batch_size);
-
- auto batch_count = calc_hw_infer_batch_count(*batch_size);
- CHECK_EXPECTED(batch_count);
+ TRY(const auto batch_size, get_batch_size());
+ TRY(const auto batch_count, calc_hw_infer_batch_count(batch_size));
for (const auto &layer_info : m_core_op_metadata->get_all_layer_infos()) {
- auto boundary_channel_ptr = get_boundary_vdma_channel_by_stream_name(layer_info.name);
- CHECK_EXPECTED(boundary_channel_ptr);
+ TRY(auto boundary_channel_ptr, get_boundary_vdma_channel_by_stream_name(layer_info.name));
const auto &stream_infos = LayerInfoUtils::get_stream_infos_from_layer_info(layer_info);
for (auto &stream_info : stream_infos) {
auto single_transfer_size = (HAILO_FORMAT_ORDER_HAILO_NMS == stream_info.format.order) ?
const auto direction = (layer_info.direction == HAILO_H2D_STREAM) ?
HailoRTDriver::DmaDirection::H2D : HailoRTDriver::DmaDirection::D2H;
- auto channel_info_pair = create_mapped_buffer_for_hw_only_infer(boundary_channel_ptr.release(), direction,
- single_transfer_size, *batch_size, batch_count.value());
- CHECK_EXPECTED(channel_info_pair);
-
- add_channel_to_hw_infer_channel_info(channel_info_pair.release(), channels_info);
+ TRY(const auto channel_info_pair, create_mapped_buffer_for_hw_only_infer(std::move(boundary_channel_ptr), direction,
+ single_transfer_size, batch_size, batch_count));
+ add_channel_to_hw_infer_channel_info(std::move(channel_info_pair), channels_info);
}
}
std::mutex mutex;
std::unique_lock<std::mutex> lock(mutex);
- status = Control::start_hw_only_infer(m_vdma_device, m_core_op_index, *batch_size,
- batch_count.value(), &channels_info);
+ status = Control::start_hw_only_infer(m_vdma_device, m_core_op_index, batch_size,
+ batch_count, &channels_info);
CHECK_SUCCESS_AS_EXPECTED(status);
infer_done_cond.wait_for(lock, INFER_TIMEOUT);
status = Control::stop_hw_only_infer(m_vdma_device, &fw_infer_results);
CHECK_SUCCESS_AS_EXPECTED(status);
- auto single_frame_transfer_size = m_core_op_metadata->get_total_transfer_size();
- CHECK_EXPECTED(single_frame_transfer_size);
+ TRY(const auto single_frame_transfer_size, m_core_op_metadata->get_total_transfer_size());
- return hw_infer_calc_stats(batch_count.value(), *batch_size, single_frame_transfer_size.release(),
+ return hw_infer_calc_stats(batch_count, batch_size, single_frame_transfer_size,
fw_infer_results.infer_cycles);
}
+
hailo_status ResourcesManager::fill_internal_buffers_info()
{
for (const auto &context_metadata : m_core_op_metadata->dynamic_contexts()) {
}
Expected<std::shared_ptr<ActionListBufferBuilder>> ResourcesManager::create_action_list_buffer_builder(
- size_t num_dynamic_contexts, HailoRTDriver &driver)
+ size_t num_dynamic_contexts, VdmaDevice &vdma_device)
{
static const auto total_num_contexts = CONTROL_PROTOCOL__CONTEXT_SWITCH_NUMBER_OF_NON_DYNAMIC_CONTEXTS +
num_dynamic_contexts;
- if (should_use_ddr_action_list(total_num_contexts, driver.dma_type())) {
- auto ddr_action_list_buffer_builder = DDRActionListBufferBuilder::create(total_num_contexts, driver);
- CHECK_EXPECTED(ddr_action_list_buffer_builder);
- return std::static_pointer_cast<ActionListBufferBuilder>(ddr_action_list_buffer_builder.release());
+ if (should_use_ddr_action_list(total_num_contexts, vdma_device.get_driver().dma_type())) {
+ TRY(auto ddr_action_list_buffer_builder, DDRActionListBufferBuilder::create(total_num_contexts, vdma_device));
+ return std::static_pointer_cast<ActionListBufferBuilder>(std::move(ddr_action_list_buffer_builder));
} else {
- auto control_action_list_buffer_builder = ControlActionListBufferBuilder::create();
- CHECK_EXPECTED(control_action_list_buffer_builder);
- return std::static_pointer_cast<ActionListBufferBuilder>(control_action_list_buffer_builder.release());
+ TRY(auto control_action_list_buffer_builder, ControlActionListBufferBuilder::create());
+ return std::static_pointer_cast<ActionListBufferBuilder>(std::move(control_action_list_buffer_builder));
}
}
#include "hailo/hailort.h"
#include "core_op/resource_manager/intermediate_buffer.hpp"
+#include "core_op/resource_manager/cache_buffer.hpp"
+#include "core_op/resource_manager/cache_manager.hpp"
#include "core_op/resource_manager/config_buffer.hpp"
#include "core_op/resource_manager/channel_allocator.hpp"
#include "core_op/resource_manager/action_list_buffer_builder/control_action_list_buffer_builder.hpp"
{
public:
static Expected<ResourcesManager> create(VdmaDevice &vdma_device, HailoRTDriver &driver,
- const ConfigureNetworkParams &config_params, std::shared_ptr<CoreOpMetadata> core_op_metadata,
- uint8_t core_op_index);
+ const ConfigureNetworkParams &config_params, CacheManagerPtr cache_manager,
+ std::shared_ptr<CoreOpMetadata> core_op_metadata, uint8_t core_op_index);
// TODO: HRT-9432 needs to call stop_vdma_interrupts_dispatcher and any other resource on dtor.
~ResourcesManager() = default;
uint32_t transfer_size, uint16_t batch_size, uint8_t src_stream_index, uint16_t src_context_index,
vdma::ChannelId d2h_channel_id, IntermediateBuffer::StreamingType streaming_type);
ExpectedRef<IntermediateBuffer> get_intermediate_buffer(const IntermediateBufferKey &key);
+ ExpectedRef<IntermediateBuffer> set_cache_input_channel(uint32_t cache_id, uint16_t batch_size, vdma::ChannelId channel_id);
+ ExpectedRef<IntermediateBuffer> set_cache_output_channel(uint32_t cache_id, uint16_t batch_size, vdma::ChannelId channel_id);
+ std::unordered_map<uint32_t, CacheBuffer> &get_cache_buffers();
hailo_status create_boundary_vdma_channel(const LayerInfo &layer_info);
Expected<CONTROL_PROTOCOL__application_header_t> get_control_core_op_header();
return m_latency_meters;
}
- std::map<vdma::ChannelId, vdma::BoundaryChannelPtr> get_boundary_vdma_channels() const
- {
- return m_boundary_channels;
- }
-
std::shared_ptr<ActionListBufferBuilder>& get_action_list_buffer_builder()
{
return m_action_list_buffer_builder;
Expected<hailo_stream_interface_t> get_default_streams_interface();
Expected<Buffer> read_intermediate_buffer(const IntermediateBufferKey &key);
+ Expected<Buffer> read_cache_buffer(uint32_t cache_id);
+ Expected<std::map<uint32_t, Buffer>> read_cache_buffers();
hailo_status configure();
hailo_status enable_state_machine(uint16_t dynamic_batch_size,
Expected<vdma::BoundaryChannelPtr> get_boundary_vdma_channel_by_stream_name(const std::string &stream_name);
Expected<std::shared_ptr<const vdma::BoundaryChannel>> get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) const;
hailo_power_mode_t get_power_mode() const;
- Expected<uint16_t> program_desc_for_hw_only_flow(std::shared_ptr<vdma::DescriptorList> desc_list,
+ Expected<uint16_t> program_desc_for_hw_only_flow(vdma::DescriptorList &desc_list,
+ vdma::MappedBuffer &mapped_buffer, vdma::ChannelId channel_id,
const uint32_t single_transfer_size, const uint16_t dynamic_batch_size, const uint16_t batch_count);
Expected<std::pair<vdma::ChannelId, uint16_t>> create_mapped_buffer_for_hw_only_infer(
vdma::BoundaryChannelPtr boundary_channel_ptr, const HailoRTDriver::DmaDirection direction,
hailo_status fill_internal_buffers_info();
static bool should_use_ddr_action_list(size_t num_contexts, HailoRTDriver::DmaType dma_type);
static Expected<std::shared_ptr<ActionListBufferBuilder>> create_action_list_buffer_builder(
- size_t num_dynamic_contexts, HailoRTDriver &driver);
+ size_t num_dynamic_contexts, VdmaDevice &vdma_device);
bool get_can_fast_batch_switch()
{
return m_core_op_metadata->get_can_fast_batch_switch();
hailo_status fill_validation_features(CONTROL_PROTOCOL__application_header_t &app_header);
hailo_status fill_network_batch_size(CONTROL_PROTOCOL__application_header_t &app_header);
hailo_status fill_csm_buffer_size(CONTROL_PROTOCOL__application_header_t &app_header);
- void process_interrupts(IrqData &&irq_data);
Expected<uint16_t> get_batch_size() const;
+ // <ongoing_transfers, pending_transfers>
+ static std::pair<size_t, size_t> calculate_transfer_queue_sizes(const vdma::DescriptorList &desc_list,
+ uint32_t transfer_size, uint32_t max_active_trans, bool use_latency_meter);
+
std::vector<ContextResources> m_contexts_resources;
ChannelAllocator m_channel_allocator;
VdmaDevice &m_vdma_device;
HailoRTDriver &m_driver;
const ConfigureNetworkParams m_config_params;
+ CacheManagerPtr m_cache_manager;
std::map<IntermediateBufferKey, IntermediateBuffer> m_intermediate_buffers;
std::shared_ptr<CoreOpMetadata> m_core_op_metadata;
uint8_t m_core_op_index;
uint16_t m_total_context_count;
const std::vector<std::string> m_network_index_map;
LatencyMetersMap m_latency_meters; // Latency meter per network
- // TODO: HRT-9429 - fast access to channel by id, using array, using engine_index and channel_index.
- std::map<vdma::ChannelId, vdma::BoundaryChannelPtr> m_boundary_channels;
+ vdma::ChannelsGroup m_boundary_channels;
bool m_is_configured;
bool m_is_activated;
// Config channels ids are shared between all context. The following vector contains the channel id for each
ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &driver,
ChannelAllocator &&channel_allocator, const ConfigureNetworkParams config_params,
+ CacheManagerPtr cache_manager,
std::shared_ptr<CoreOpMetadata> &&core_op_metadata, uint8_t core_op_index,
const std::vector<std::string> &&network_index_map, LatencyMetersMap &&latency_meters,
std::vector<vdma::ChannelId> &&config_channels_ids,
#include "device_common/control.hpp"
#include "periph_calculator.hpp"
#include "hef/hef_internal.hpp"
+#include "common/file_utils.hpp"
namespace hailort
{
local_layer_info.nn_stream_config.is_periph_calculated_in_hailort = is_periph_calculated_in_hailort;
local_layer_info.nn_stream_config.is_core_hw_padding_config_in_dfc = is_core_hw_padding_config_in_dfc;
- auto updated_periph_layer_info = PeriphCalculator::calculate_periph_registers(local_layer_info,
- buffer_info.desc_page_size, is_periph_calculated_in_hailort, hw_arch, is_core_hw_padding_config_in_dfc);
- CHECK_EXPECTED(updated_periph_layer_info);
+ TRY(const auto updated_periph_layer_info, PeriphCalculator::calculate_periph_registers(local_layer_info,
+ buffer_info.desc_page_size, is_periph_calculated_in_hailort, hw_arch, is_core_hw_padding_config_in_dfc));
- auto updated_local_layer_info = calculate_credit_params(hw_consts, buffer_info.desc_page_size, should_optimize_credits,
- updated_periph_layer_info.release());
- CHECK_EXPECTED(updated_local_layer_info);
+ TRY(auto updated_local_layer_info, calculate_credit_params(hw_consts, buffer_info.desc_page_size, should_optimize_credits,
+ updated_periph_layer_info));
return updated_local_layer_info;
}
{
const auto transfer_size = LayerInfoUtils::get_layer_transfer_size(layer_info);
- auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name);
- CHECK_EXPECTED_AS_STATUS(vdma_channel);
+ TRY(const auto vdma_channel, resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name));
- const auto buffer_info = vdma_channel.value()->get_boundary_buffer_info(transfer_size);
+ const auto buffer_info = vdma_channel->get_boundary_buffer_info(transfer_size);
const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort;
const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc;
- auto local_layer_info = update_layer_info(layer_info, buffer_info, hw_consts, hw_arch, should_optimize_credits,
- is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc);
- CHECK_EXPECTED_AS_STATUS(local_layer_info);
+ TRY(auto local_layer_info, update_layer_info(layer_info, buffer_info, hw_consts, hw_arch, should_optimize_credits,
+ is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc));
- const auto channel_id = vdma_channel.value()->get_channel_id();
- auto status = context_resources.add_edge_layer(local_layer_info.value(), channel_id, buffer_info,
+ const auto channel_id = vdma_channel->get_channel_id();
+ auto status = context_resources.add_edge_layer(local_layer_info, channel_id, buffer_info,
resources_manager.get_supported_features());
CHECK_SUCCESS(status);
ResourcesManager &resources_manager, const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts,
const HEFHwArch &hw_arch, bool should_optimize_credits)
{
- const auto channel_id = resources_manager.get_available_channel_id(to_layer_identifier(layer_info),
- HailoRTDriver::DmaDirection::H2D, layer_info.dma_engine_index);
- CHECK_EXPECTED_AS_STATUS(channel_id);
+ TRY(const auto channel_id, resources_manager.get_available_channel_id(to_layer_identifier(layer_info),
+ HailoRTDriver::DmaDirection::H2D, layer_info.dma_engine_index));
/* Get inter context buffer previously created */
const auto &connected_context = layer_info.connected_context_info;
auto intermediate_buffer_key = std::make_pair(connected_context.context_index, connected_context.stream_index);
- auto inter_context_buffer_exp = resources_manager.get_intermediate_buffer(intermediate_buffer_key);
- CHECK_EXPECTED_AS_STATUS(inter_context_buffer_exp, "Failed to find inter context buffer for src context {}, src_stream_index {}",
+ TRY(auto inter_context_buffer, resources_manager.get_intermediate_buffer(intermediate_buffer_key),
+ "Failed to find inter context buffer for src context {}, src_stream_index {}",
connected_context.context_index, connected_context.stream_index);
- auto &inter_context_buffer = inter_context_buffer_exp->get();
const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort;
const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc;
- auto local_layer_info = update_layer_info(layer_info, inter_context_buffer.get_host_buffer_info(), hw_consts,
- hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc);
- CHECK_EXPECTED_AS_STATUS(local_layer_info);
+ TRY(auto local_layer_info, update_layer_info(layer_info, inter_context_buffer.get().get_host_buffer_info(), hw_consts,
+ hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc));
- auto status = context_resources.add_edge_layer(local_layer_info.value(), channel_id.value(),
- inter_context_buffer.get_host_buffer_info(), resources_manager.get_supported_features());
+ auto status = context_resources.add_edge_layer(local_layer_info, channel_id,
+ inter_context_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features());
CHECK_SUCCESS(status);
LOGGER__DEBUG("Intermediate edge key: {}:{} src_context:{}, dst_context: {}, h2d_channel {}.",
connected_context.context_index, connected_context.stream_index,
- layer_info.connected_context_info.context_index, layer_info.context_index, channel_id.value());
+ layer_info.connected_context_info.context_index, layer_info.context_index, channel_id);
return HAILO_SUCCESS;
}
{
const auto transfer_size = LayerInfoUtils::get_layer_transfer_size(layer_info);
- auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name);
- CHECK_EXPECTED_AS_STATUS(vdma_channel);
+ TRY(const auto vdma_channel, resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name));
- const auto buffer_info = vdma_channel.value()->get_boundary_buffer_info(transfer_size);
+ const auto buffer_info = vdma_channel->get_boundary_buffer_info(transfer_size);
const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort;
const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc;
- auto local_layer_info = update_layer_info(layer_info, buffer_info, hw_consts, hw_arch, should_optimize_credits,
- is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc);
- CHECK_EXPECTED_AS_STATUS(local_layer_info);
+ TRY(auto local_layer_info, update_layer_info(layer_info, buffer_info, hw_consts, hw_arch, should_optimize_credits,
+ is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc));
- const auto channel_id = vdma_channel.value()->get_channel_id();
- auto status = context_resources.add_edge_layer(local_layer_info.value(), channel_id, buffer_info,
+ const auto channel_id = vdma_channel->get_channel_id();
+ auto status = context_resources.add_edge_layer(local_layer_info, channel_id, buffer_info,
resources_manager.get_supported_features());
CHECK_SUCCESS(status);
ResourcesManager &resources_manager, const LayerInfo &layer_info,
const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch, bool should_optimize_credits)
{
- const auto channel_id = resources_manager.get_available_channel_id(to_layer_identifier(layer_info),
- HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index);
- CHECK_EXPECTED_AS_STATUS(channel_id);
+ TRY(const auto channel_id, resources_manager.get_available_channel_id(to_layer_identifier(layer_info),
+ HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index));
const auto frame_credits_in_bytes = LayerInfoUtils::get_layer_transfer_size(layer_info);
- auto network_batch_size = resources_manager.get_network_batch_size(layer_info.network_name);
- CHECK_EXPECTED_AS_STATUS(network_batch_size);
+ TRY(const auto network_batch_size, resources_manager.get_network_batch_size(layer_info.network_name));
- auto inter_context_buffer_exp = resources_manager.create_intermediate_buffer(frame_credits_in_bytes,
- network_batch_size.value(), layer_info.stream_index, layer_info.context_index,
- channel_id.value(), IntermediateBuffer::StreamingType::BURST);
- CHECK_EXPECTED_AS_STATUS(inter_context_buffer_exp);
- auto &inter_context_buffer = inter_context_buffer_exp->get();
+ TRY(auto inter_context_buffer, resources_manager.create_intermediate_buffer(frame_credits_in_bytes,
+ network_batch_size, layer_info.stream_index, layer_info.context_index,
+ channel_id, IntermediateBuffer::StreamingType::BURST));
const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort;
const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc;
- auto local_layer_info = update_layer_info(layer_info, inter_context_buffer.get_host_buffer_info(), hw_consts,
- hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc);
- CHECK_EXPECTED_AS_STATUS(local_layer_info);
+ TRY(auto local_layer_info, update_layer_info(layer_info, inter_context_buffer.get().get_host_buffer_info(), hw_consts,
+ hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc));
- auto status = context_resources.add_edge_layer(local_layer_info.value(), channel_id.value(),
- inter_context_buffer.get_host_buffer_info(), resources_manager.get_supported_features());
+ auto status = context_resources.add_edge_layer(local_layer_info, channel_id,
+ inter_context_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features());
CHECK_SUCCESS(status);
LOGGER__DEBUG("Inter-context output stream {}, src_context:{}, d2h_channel {}.",
- layer_info.stream_index, layer_info.context_index, channel_id.value());
+ layer_info.stream_index, layer_info.context_index, channel_id);
return HAILO_SUCCESS;
}
const auto h2d_stream_index = layer_info.connected_context_info.stream_index;
const auto h2d_layer_identifier = std::make_tuple(LayerType::DDR, HAILO_H2D_STREAM,
layer_info.name, h2d_stream_index);
- const auto h2d_channel_id = resources_manager.get_available_channel_id(h2d_layer_identifier,
- HailoRTDriver::DmaDirection::H2D, layer_info.connected_context_info.dma_engine_index);
- CHECK_EXPECTED_AS_STATUS(h2d_channel_id);
+ TRY(const auto h2d_channel_id, resources_manager.get_available_channel_id(h2d_layer_identifier,
+ HailoRTDriver::DmaDirection::H2D, layer_info.connected_context_info.dma_engine_index));
const auto d2h_stream_index = layer_info.stream_index;
const auto d2h_layer_identifier = std::make_tuple(LayerType::DDR, HAILO_D2H_STREAM,
layer_info.name, d2h_stream_index);
- const auto d2h_channel_id = resources_manager.get_available_channel_id(d2h_layer_identifier,
- HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index);
- CHECK_EXPECTED_AS_STATUS(d2h_channel_id);
+ TRY(const auto d2h_channel_id, resources_manager.get_available_channel_id(d2h_layer_identifier,
+ HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index));
// In DDR - always use core bytes per buffer as row size
const auto row_size = static_cast<uint16_t>(layer_info.nn_stream_config.core_bytes_per_buffer);
const auto min_buffered_rows = layer_info.ddr_info.min_buffered_rows;
// Allocate the ddr buffer
- auto ddr_buffer = resources_manager.create_intermediate_buffer(row_size, min_buffered_rows,
- d2h_stream_index, layer_info.context_index, d2h_channel_id.value(),
- IntermediateBuffer::StreamingType::CIRCULAR_CONTINUOS);
- CHECK_EXPECTED_AS_STATUS(ddr_buffer);
+ TRY(auto ddr_buffer, resources_manager.create_intermediate_buffer(row_size, min_buffered_rows,
+ d2h_stream_index, layer_info.context_index, d2h_channel_id,
+ IntermediateBuffer::StreamingType::CIRCULAR_CONTINUOS));
DdrChannelsInfo ddr_pair_info{};
ddr_pair_info.h2d_stream_index = h2d_stream_index;
ddr_pair_info.d2h_stream_index = d2h_stream_index;
ddr_pair_info.network_index = layer_info.network_index;
- ddr_pair_info.h2d_channel_id = h2d_channel_id.value();
- ddr_pair_info.d2h_channel_id = d2h_channel_id.value();
+ ddr_pair_info.h2d_channel_id = h2d_channel_id;
+ ddr_pair_info.d2h_channel_id = d2h_channel_id;
ddr_pair_info.row_size = row_size;
ddr_pair_info.min_buffered_rows = min_buffered_rows;
ddr_pair_info.total_buffers_per_frame = layer_info.ddr_info.total_buffers_per_frame;
- ddr_pair_info.host_buffer_info = ddr_buffer->get().get_host_buffer_info();
+ ddr_pair_info.host_buffer_info = ddr_buffer.get().get_host_buffer_info();
context_resources.add_ddr_channels_info(ddr_pair_info);
// On ddr layers, we assume the periph credit size is aligned to the size of descriptor, so we don't want to
const bool should_optimize_credits = false;
const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort;
const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc;
- auto local_layer_info = update_layer_info(layer_info, ddr_buffer->get().get_host_buffer_info(), hw_consts,
- hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc);
- CHECK_EXPECTED_AS_STATUS(local_layer_info);
+ TRY(auto local_layer_info, update_layer_info(layer_info, ddr_buffer.get().get_host_buffer_info(), hw_consts,
+ hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc));
- auto status = context_resources.add_edge_layer(local_layer_info.value(), ddr_pair_info.d2h_channel_id,
- ddr_buffer->get().get_host_buffer_info(), resources_manager.get_supported_features());
+ auto status = context_resources.add_edge_layer(local_layer_info, ddr_pair_info.d2h_channel_id,
+ ddr_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features());
CHECK_SUCCESS(status);
return HAILO_SUCCESS;
const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch)
{
auto connected_stream_index = layer_info.connected_context_info.stream_index;
- auto ddr_info = context_resources.get_ddr_channels_info(connected_stream_index);
- CHECK_EXPECTED_AS_STATUS(ddr_info, "Matching DDR layer as not found for context {} src stream {}",
- layer_info.context_index, connected_stream_index);
+ TRY(const auto ddr_info, context_resources.get_ddr_channels_info(connected_stream_index),
+ "Matching DDR layer as not found for context {} src stream {}", layer_info.context_index, connected_stream_index);
LOGGER__DEBUG("DDR layer: input stream_index: {}, output stream_index: {}, h2d_channel {}, d2h_channel: {}.",
- ddr_info->h2d_stream_index, ddr_info->d2h_stream_index, ddr_info->h2d_channel_id, ddr_info->d2h_channel_id);
+ ddr_info.h2d_stream_index, ddr_info.d2h_stream_index, ddr_info.h2d_channel_id, ddr_info.d2h_channel_id);
- CHECK(layer_info.stream_index == ddr_info->h2d_stream_index, HAILO_INVALID_HEF, "DDR channel pair mismatch in h2d channel");
- CHECK(layer_info.connected_context_info.stream_index == ddr_info->d2h_stream_index, HAILO_INVALID_HEF, "DDR channel pair mismatch in d2h channel");
- CHECK(layer_info.network_index == ddr_info->network_index, HAILO_INVALID_HEF, "DDR channel pair mismatch network_index");
+ CHECK(layer_info.stream_index == ddr_info.h2d_stream_index, HAILO_INVALID_HEF, "DDR channel pair mismatch in h2d channel");
+ CHECK(layer_info.connected_context_info.stream_index == ddr_info.d2h_stream_index, HAILO_INVALID_HEF, "DDR channel pair mismatch in d2h channel");
+ CHECK(layer_info.network_index == ddr_info.network_index, HAILO_INVALID_HEF, "DDR channel pair mismatch network_index");
// On ddr layers, we assume the periph credit size is aligned to the size of descriptor, so we don't want to
// optimize the credits.
const bool should_optimize_credits = false;
const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort;
const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc;
- auto local_layer_info = update_layer_info(layer_info, ddr_info->host_buffer_info, hw_consts,
- hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc);
- CHECK_EXPECTED_AS_STATUS(local_layer_info);
+ TRY(auto local_layer_info, update_layer_info(layer_info, ddr_info.host_buffer_info, hw_consts,
+ hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc));
- auto status = context_resources.add_edge_layer(local_layer_info.value(), ddr_info->h2d_channel_id,
- ddr_info->host_buffer_info, resources_manager.get_supported_features());
+ auto status = context_resources.add_edge_layer(local_layer_info, ddr_info.h2d_channel_id,
+ ddr_info.host_buffer_info, resources_manager.get_supported_features());
CHECK_SUCCESS(status);
return HAILO_SUCCESS;
}
+static hailo_status fill_cache_output_layer(ContextResources &context_resources, ResourcesManager &resources_manager,
+ const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch,
+ bool should_optimize_credits)
+{
+ TRY(const auto channel_id, resources_manager.get_available_channel_id(to_layer_identifier(layer_info),
+ HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index));
+
+ TRY(const auto network_batch_size, resources_manager.get_network_batch_size(layer_info.network_name));
+ TRY(auto cache_buffer, resources_manager.set_cache_output_channel(layer_info.cache_info.id,
+ network_batch_size, channel_id));
+
+ const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort;
+ const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc;
+ TRY(auto local_layer_info, update_layer_info(layer_info, cache_buffer.get().get_host_buffer_info(), hw_consts,
+ hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc));
+
+ auto status = context_resources.add_edge_layer(local_layer_info, channel_id,
+ cache_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features());
+ CHECK_SUCCESS(status);
+
+ LOGGER__DEBUG("Cache id {}: output stream {}, d2h_channel {}, context {}",
+ layer_info.cache_info.id, layer_info.stream_index, channel_id, layer_info.context_index);
+ return HAILO_SUCCESS;
+}
+
+static hailo_status fill_cache_input_layer(ContextResources &context_resources, ResourcesManager &resources_manager,
+ const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch, bool should_optimize_credits)
+{
+ TRY(const auto channel_id, resources_manager.get_available_channel_id(to_layer_identifier(layer_info),
+ HailoRTDriver::DmaDirection::H2D, layer_info.dma_engine_index));
+
+ TRY(const auto network_batch_size, resources_manager.get_network_batch_size(layer_info.network_name));
+ TRY(auto cache_buffer, resources_manager.set_cache_input_channel(layer_info.cache_info.id, network_batch_size, channel_id));
+
+ const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort;
+ const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc;
+ TRY(auto local_layer_info, update_layer_info(layer_info, cache_buffer.get().get_host_buffer_info(), hw_consts,
+ hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc));
+
+ auto status = context_resources.add_edge_layer(local_layer_info, channel_id,
+ cache_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features());
+ CHECK_SUCCESS(status);
+
+ LOGGER__DEBUG("Cache id {}: input stream {}, h2d_channel {}, context {}",
+ layer_info.cache_info.id, layer_info.stream_index, channel_id, layer_info.context_index);
+
+ return HAILO_SUCCESS;
+}
+
static hailo_status add_ddr_buffers_info(std::vector<ContextSwitchConfigActionPtr> &configuration_actions,
const ContextResources &context_resources)
{
bool start_fw_ddr_buffer_task = false;
for (const auto &ddr_info : context_resources.get_ddr_channels_infos()) {
if (ddr_info.need_manual_credit_management()) {
- auto ddr_pair_action = DdrPairInfoAction::create(ddr_info.h2d_channel_id, ddr_info.d2h_channel_id,
- ddr_info.network_index, ddr_info.descriptors_per_frame(), ddr_info.descs_count());
- CHECK_EXPECTED_AS_STATUS(ddr_pair_action);
- configuration_actions.emplace_back(ddr_pair_action.release());
+ TRY(auto ddr_pair_action, DdrPairInfoAction::create(ddr_info.h2d_channel_id, ddr_info.d2h_channel_id,
+ ddr_info.network_index, ddr_info.descriptors_per_frame(), ddr_info.descs_count()));
+ configuration_actions.emplace_back(std::move(ddr_pair_action));
start_fw_ddr_buffer_task = true;
}
}
if (start_fw_ddr_buffer_task) {
- auto start_ddr_buffering_action = StartDdrBufferingTaskAction::create();
- CHECK_EXPECTED_AS_STATUS(start_ddr_buffering_action);
- configuration_actions.emplace_back(start_ddr_buffering_action.release());
+ TRY(auto start_ddr_buffering_action, StartDdrBufferingTaskAction::create());
+ configuration_actions.emplace_back(std::move(start_ddr_buffering_action));
}
return HAILO_SUCCESS;
{
hailo_status status = HAILO_UNINITIALIZED;
- auto hw_consts = Control::get_hw_consts(resources_manager.get_device());
- CHECK_EXPECTED_AS_STATUS(hw_consts);
- const bool should_optimize_credits = hw_consts->should_optimize_credits &&
+ TRY(const auto hw_consts, Control::get_hw_consts(resources_manager.get_device()));
+ const bool should_optimize_credits = hw_consts.should_optimize_credits &&
(HAILO_POWER_MODE_PERFORMANCE == resources_manager.get_power_mode());
// Parse the edge layer by order - first output edge layers, then ddr inputs and only then the input edge layers
// We parse ddr inputs before boundary/inter-context because otherwise on C2C mode we may lose some credit.
for (const auto &output_layer_info : context_metadata.get_ddr_output_layers()) {
- status = fill_ddr_output_layer(context_resources, resources_manager, output_layer_info, *hw_consts, hw_arch);
+ status = fill_ddr_output_layer(context_resources, resources_manager, output_layer_info, hw_consts, hw_arch);
CHECK_SUCCESS(status);
}
for (const auto &output_layer_info : context_metadata.get_boundary_output_layers()) {
status = fill_boundary_output_layer(context_resources, resources_manager, output_layer_info,
- *hw_consts, hw_arch, should_optimize_credits);
+ hw_consts, hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
for (const auto &output_layer_info : context_metadata.get_inter_context_output_layers()) {
status = fill_inter_context_output_layer(context_resources, resources_manager, output_layer_info,
- *hw_consts, hw_arch, should_optimize_credits);
+ hw_consts, hw_arch, should_optimize_credits);
+ CHECK_SUCCESS(status);
+ }
+
+ for (const auto &cache_layer_info : context_metadata.get_cache_output_layers()) {
+ status = fill_cache_output_layer(context_resources, resources_manager, cache_layer_info, hw_consts,
+ hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
for (const auto &input_layer_info : context_metadata.get_ddr_input_layers()) {
- status = fill_ddr_input_layer(context_resources, resources_manager, input_layer_info, *hw_consts, hw_arch);
+ status = fill_ddr_input_layer(context_resources, resources_manager, input_layer_info, hw_consts, hw_arch);
CHECK_SUCCESS(status);
}
for (const auto &input_layer_info : context_metadata.get_boundary_input_layers()) {
status = fill_boundary_input_layer(context_resources, resources_manager, input_layer_info,
- *hw_consts, hw_arch, should_optimize_credits);
+ hw_consts, hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
for (const auto &input_layer_info : context_metadata.get_inter_context_input_layers()) {
status = fill_inter_context_input_layer(context_resources, resources_manager, input_layer_info,
- *hw_consts, hw_arch, should_optimize_credits);
+ hw_consts, hw_arch, should_optimize_credits);
+ CHECK_SUCCESS(status);
+ }
+
+ for (const auto &cache_layer_info : context_metadata.get_cache_input_layers()) {
+ status = fill_cache_input_layer(context_resources, resources_manager, cache_layer_info, hw_consts,
+ hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
- /* UN-Lock resources at the end of the context -
- h2d inter-context, d2h inter-context and DDR buffer channels */
+ // Unlock resources at the end of the context -
+ // Inter-context channels, DDR buffer channels and cache channels
for (const auto &input_layer_info : context_metadata.get_inter_context_input_layers()) {
status = resources_manager.free_channel_index(to_layer_identifier(input_layer_info));
CHECK_SUCCESS(status);
CHECK_SUCCESS(status);
}
+ for (const auto &input_layer_info : context_metadata.get_cache_input_layers()) {
+ status = resources_manager.free_channel_index(to_layer_identifier(input_layer_info));
+ CHECK_SUCCESS(status);
+ }
+
+ for (const auto &output_layer_info : context_metadata.get_cache_output_layers()) {
+ status = resources_manager.free_channel_index(to_layer_identifier(output_layer_info));
+ CHECK_SUCCESS(status);
+ }
+
return HAILO_SUCCESS;
}
std::vector<ContextSwitchConfigActionPtr> &processed_configuration_actions)
{
if (support_pre_fetch) {
- auto action = AddCcwBurstAction::create(config_stream_index, total_ccw_bursts);
- CHECK_EXPECTED_AS_STATUS(action);
- processed_configuration_actions.emplace_back(action.release());
+ TRY(const auto action, AddCcwBurstAction::create(config_stream_index, total_ccw_bursts));
+ processed_configuration_actions.emplace_back(std::move(action));
} else {
- const auto desc_count = config_resources.program_descriptors();
- CHECK_EXPECTED_AS_STATUS(desc_count);
-
- auto action = FetchCfgChannelDescriptorsAction::create(config_resources.channel_id(), desc_count.value());
- CHECK_EXPECTED_AS_STATUS(action);
- processed_configuration_actions.emplace_back(action.release());
+ TRY(const auto desc_count, config_resources.program_descriptors());
+ TRY(const auto action, FetchCfgChannelDescriptorsAction::create(config_resources.channel_id(), desc_count));
+ processed_configuration_actions.emplace_back(std::move(action));
}
return HAILO_SUCCESS;
ContextResources &context_resources, uint16_t context_index,
std::vector<ContextSwitchConfigActionPtr> &processed_configuration_actions)
{
- auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name);
- CHECK_EXPECTED_AS_STATUS(vdma_channel);
+ TRY(auto vdma_channel, resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name));
- const auto channel_id = vdma_channel.value()->get_channel_id();
+ const auto channel_id = vdma_channel->get_channel_id();
const bool is_dummy_stream = layer_info.context_index != context_index;
uint8_t stream_index = layer_info.stream_index;
if (is_dummy_stream) {
- auto dummy_stream_index = find_dummy_stream(layer_info, context_resources,
- HailoRTCommon::is_hailo1x_device_type(DeviceBase::hef_arch_to_device_arch(hw_arch)));
- CHECK_EXPECTED_AS_STATUS(dummy_stream_index);
- stream_index = *dummy_stream_index;
+ TRY(const auto dummy_stream_index, find_dummy_stream(layer_info, context_resources,
+ HailoRTCommon::is_hailo1x_device_type(DeviceBase::hef_arch_to_device_arch(hw_arch))));
+ stream_index = dummy_stream_index;
}
- auto action = ChangeVdmaToStreamMapping::create(channel_id, stream_index, is_dummy_stream);
- CHECK_EXPECTED_AS_STATUS(action);
- processed_configuration_actions.emplace_back(action.release());
+ TRY(const auto action, ChangeVdmaToStreamMapping::create(channel_id, stream_index, is_dummy_stream));
+ processed_configuration_actions.emplace_back(std::move(action));
return HAILO_SUCCESS;
}
// We parse ddr inputs before boundary/inter-context because otherwise on C2C mode we may lose some credit.
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::DDR, HAILO_D2H_STREAM)) {
- auto activate_action = ActivateDdrOutputChannelAction::create(edge_layer.channel_id,
+ TRY(const auto activate_action, ActivateDdrOutputChannelAction::create(edge_layer.channel_id,
edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info,
- edge_layer.layer_info.ddr_info.min_buffered_rows);
- CHECK_EXPECTED_AS_STATUS(activate_action);
- actions.emplace_back(activate_action.release());
+ edge_layer.layer_info.ddr_info.min_buffered_rows));
+ actions.emplace_back(std::move(activate_action));
}
if (!push_internal_only) {
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_D2H_STREAM)) {
- auto activate_action = ActivateBoundaryOutputChannelAction::create(edge_layer.channel_id,
+ TRY(const auto activate_action, ActivateBoundaryOutputChannelAction::create(edge_layer.channel_id,
edge_layer.layer_info.stream_index, edge_layer.layer_info.network_index,
- edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info);
- CHECK_EXPECTED_AS_STATUS(activate_action);
- actions.emplace_back(activate_action.release());
+ edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info));
+ actions.emplace_back(std::move(activate_action));
}
}
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::INTER_CONTEXT, HAILO_D2H_STREAM)) {
- auto activate_action = ActivateInterContextOutputChannelAction::create(edge_layer.channel_id,
+ TRY(auto activate_action, ActivateInterContextOutputChannelAction::create(edge_layer.channel_id,
edge_layer.layer_info.stream_index, edge_layer.layer_info.network_index,
- edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info);
- CHECK_EXPECTED_AS_STATUS(activate_action);
- actions.emplace_back(activate_action.release());
+ edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info));
+ actions.emplace_back(std::move(activate_action));
+ }
+
+ for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::CACHE, HAILO_D2H_STREAM)) {
+ // TODO: Cache edge_layers' buffer_info isn't correct - total_desc_count and bytes_in_pattern are calculated
+ // according to the cache size, not the transfer size (since the edge layer is backed by a buffer of cache_size)
+ // Think of a way of holding the correct buffer_info for cache edge_layers (or maybe removing host buffer info all together)
+ // (HRT-13775)
+ TRY(const auto activate_action, ActivateCacheOutputChannelAction::create(edge_layer.channel_id,
+ edge_layer.layer_info.stream_index, edge_layer.layer_info.network_index,
+ edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info));
+ actions.emplace_back(std::move(activate_action));
}
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::DDR, HAILO_H2D_STREAM)) {
const auto d2h_stream_index = edge_layer.layer_info.connected_context_info.stream_index;
- auto ddr_channels_info = context_resources.get_ddr_channels_info(d2h_stream_index);
- CHECK_EXPECTED_AS_STATUS(ddr_channels_info);
- const auto d2h_channel_id = ddr_channels_info->d2h_channel_id;
+ TRY(const auto ddr_channels_info, context_resources.get_ddr_channels_info(d2h_stream_index));
+ const auto d2h_channel_id = ddr_channels_info.d2h_channel_id;
- auto activate_action = ActivateDdrInputChannelAction::create(edge_layer.channel_id,
+ TRY(const auto activate_action, ActivateDdrInputChannelAction::create(edge_layer.channel_id,
edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info,
- edge_layer.layer_info.max_shmifo_size, d2h_channel_id);
- CHECK_EXPECTED_AS_STATUS(activate_action);
- actions.emplace_back(activate_action.release());
+ edge_layer.layer_info.max_shmifo_size, d2h_channel_id));
+ actions.emplace_back(std::move(activate_action));
}
if (!push_internal_only) {
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) {
- auto activate_action = ActivateBoundaryInputChannelAction::create(edge_layer.channel_id,
+ TRY(auto activate_action, ActivateBoundaryInputChannelAction::create(edge_layer.channel_id,
edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info,
- edge_layer.layer_info.max_shmifo_size);
- CHECK_EXPECTED_AS_STATUS(activate_action);
- actions.emplace_back(activate_action.release());
+ edge_layer.layer_info.max_shmifo_size));
+ actions.emplace_back(activate_action);
}
}
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::INTER_CONTEXT, HAILO_H2D_STREAM)) {
- auto activate_action = ActivateInterContextInputChannelAction::create(edge_layer.channel_id,
+ TRY(const auto activate_action, ActivateInterContextInputChannelAction::create(edge_layer.channel_id,
+ edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info,
+ edge_layer.layer_info.max_shmifo_size));
+ actions.emplace_back(std::move(activate_action));
+ }
+
+ for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::CACHE, HAILO_H2D_STREAM)) {
+ // TODO: Cache edge_layers' buffer_info isn't correct - total_desc_count and bytes_in_pattern are calculated
+ // according to the cache size, not the transfer size (since the edge layer is backed by a buffer of cache_size)
+ // Think of a way of holding the correct buffer_info for cache edge_layers (or maybe removing host buffer info all together)
+ // (HRT-13775)
+ TRY(const auto activate_action, ActivateCacheInputChannelAction::create(edge_layer.channel_id,
edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info,
- edge_layer.layer_info.max_shmifo_size);
- CHECK_EXPECTED_AS_STATUS(activate_action);
- actions.emplace_back(activate_action.release());
+ edge_layer.layer_info.max_shmifo_size));
+ actions.emplace_back(std::move(activate_action));
}
return HAILO_SUCCESS;
/* Open the boundary input channel */
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) {
- auto activate_action = ResumeVdmaChannel::create(edge_layer);
- CHECK_EXPECTED_AS_STATUS(activate_action);
- processed_configuration_actions.emplace_back(activate_action.release());
+ TRY(const auto activate_action, ResumeVdmaChannel::create(edge_layer));
+ processed_configuration_actions.emplace_back(std::move(activate_action));
}
}
if (trigger_new_data_from_input_group_end == action_index) {
auto boundary_input_edge_layers = context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM);
if (boundary_input_edge_layers.size() > 0) {
- auto start_burst_credits_task_action = StartBurstCreditsTaskAction::create();
- CHECK_EXPECTED_AS_STATUS(start_burst_credits_task_action);
- processed_configuration_actions.emplace_back(start_burst_credits_task_action.release());
+ TRY(const auto start_burst_credits_task_action, StartBurstCreditsTaskAction::create());
+ processed_configuration_actions.emplace_back(std::move(start_burst_credits_task_action));
}
}
for (uint8_t config_stream_index = 0; config_stream_index < config_resources.size(); config_stream_index++) {
const auto &config_buffer = config_resources[config_stream_index];
- auto activate_action = ActivateConfigChannelAction::create(config_stream_index, config_buffer.channel_id(),
- config_buffer.get_host_buffer_info());
- CHECK_EXPECTED_AS_STATUS(activate_action);
- processed_actions.push_back(activate_action.release());
+ TRY(const auto activate_action, ActivateConfigChannelAction::create(config_stream_index, config_buffer.channel_id(),
+ config_buffer.get_host_buffer_info()));
+ processed_actions.push_back(std::move(activate_action));
}
processed_actions.insert(processed_actions.end(), actions.begin(), actions.end());
for (uint8_t config_stream_index = 0; config_stream_index < config_resources.size(); config_stream_index++) {
const auto &config_buffer = config_resources[config_stream_index];
- auto deactivate_action = DeactivateConfigChannelAction::create(config_stream_index, config_buffer.channel_id());
- CHECK_EXPECTED_AS_STATUS(deactivate_action);
- processed_actions.push_back(deactivate_action.release());
+ TRY(const auto deactivate_action, DeactivateConfigChannelAction::create(config_stream_index, config_buffer.channel_id()));
+ processed_actions.push_back(std::move(deactivate_action));
}
actions = processed_actions;
action_index++;
}
- auto repeated_header_action = RepeatedAction::create(std::move(repeated_block));
- CHECK_EXPECTED_AS_STATUS(repeated_header_action);
- processed_configuration_actions.emplace_back(repeated_header_action.value());
+ TRY(const auto repeated_header_action, RepeatedAction::create(std::move(repeated_block)));
+ processed_configuration_actions.emplace_back(std::move(repeated_header_action));
}
else {
processed_configuration_actions.emplace_back(configuration_actions[action_index]);
// Mark first action buffer of context to know when new context is starting (needed for dynamic contexts)
bool is_first_action_buffer_of_context = true;
for (const auto &action : actions) {
- auto action_buffers = action->serialize(context_resources);
- CHECK_EXPECTED_AS_STATUS(action_buffers);
+ TRY(auto action_buffers, action->serialize(context_resources));
- for (auto &action_buffer : action_buffers.value()) {
- const bool last_action_buffer_in_context = (action_buffer == *(action_buffers.value().end() - 1)) &&
+ for (auto &action_buffer : action_buffers) {
+ const bool last_action_buffer_in_context = (action_buffer == *(action_buffers.end() - 1)) &&
(action == *(actions.end() - 1));
builder->write_action(MemoryView(action_buffer), context_resources.get_context_type(),
is_first_action_buffer_of_context, last_action_buffer_in_context);
auto action = should_validate ?
ValidateChannelAction::create(edge_layer, is_batch_switch_context) :
DeactivateChannelAction::create(edge_layer, is_batch_switch_context);
- CHECK_EXPECTED_AS_STATUS(action);
+ CHECK_EXPECTED_AS_STATUS(action); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
actions.emplace_back(action.release());
}
/* Pause the boundary input channel */
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) {
- auto activate_action = PauseVdmaChannel::create(edge_layer);
- CHECK_EXPECTED_AS_STATUS(activate_action);
- actions.emplace_back(activate_action.release());
+ TRY(const auto activate_action, PauseVdmaChannel::create(edge_layer));
+ actions.emplace_back(std::move(activate_action));
}
return HAILO_SUCCESS;
static hailo_status fill_context_recipes_for_multi_context(const HEFHwArch &hw_arch,
ContextResources &context_resources, ResourcesManager &resources_manager,
uint16_t context_index, const CoreOpMetadata &core_op_metadata, const ContextMetadata &context_metadata,
- bool is_single_context)
+ bool is_single_context, bool is_last_context, bool caches_in_use)
{
hailo_status status = HAILO_UNINITIALIZED;
status = add_config_channel_activation_actions(actions, context_resources.get_config_buffers());
CHECK_SUCCESS(status);
+ // End of context actions - these must be last in their respective context
if (is_single_context) {
// Single context network must wait for network group change event after they finish the dynamic context.
- auto wait_action = WaitForNetworkGroupChangeAction::create();
- CHECK_EXPECTED_AS_STATUS(wait_action);
- actions.emplace_back(wait_action.release());
- }
- else {
+ TRY(const auto wait_action, WaitForNetworkGroupChangeAction::create());
+ actions.emplace_back(std::move(wait_action));
+ } else {
const bool NOT_BATCH_SWITCH_CONTEXT = false;
status = add_edge_layer_end_of_context_actions(context_resources, actions, NOT_BATCH_SWITCH_CONTEXT);
+
+ if (is_last_context && caches_in_use) {
+ const auto cache_offset_env_var = get_env_variable(HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR);
+ if (cache_offset_env_var.has_value() &&
+ (cache_offset_env_var.value() == HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DISABLED)) {
+ LOGGER__INFO("Skipping cache offset updates");
+ } else {
+ // If caches are in use, we'll wait for the caches to be updated at the end of the last context
+ TRY(const auto action, WaitForCacheUpdatedAction::create());
+ actions.emplace_back(std::move(action));
+ }
+ }
}
status = handle_repeated_actions(actions);
ContextResources &context_resources, ResourcesManager &resources_manager,
std::shared_ptr<CoreOpMetadata> core_op_metadata, const HEFHwArch &hw_arch)
{
- auto hw_consts = Control::get_hw_consts(resources_manager.get_device());
- CHECK_EXPECTED_AS_STATUS(hw_consts);
- const bool should_optimize_credits = hw_consts->should_optimize_credits &&
+ TRY(const auto hw_consts, Control::get_hw_consts(resources_manager.get_device()));
+ const bool should_optimize_credits = hw_consts.should_optimize_credits &&
(HAILO_POWER_MODE_PERFORMANCE == resources_manager.get_power_mode());
for (const auto &layer_info : core_op_metadata->get_output_layer_infos()){
- auto status = fill_boundary_output_layer(context_resources, resources_manager, layer_info, *hw_consts,
+ auto status = fill_boundary_output_layer(context_resources, resources_manager, layer_info, hw_consts,
hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
for (const auto &layer_info : core_op_metadata->get_input_layer_infos()) {
- auto status = fill_boundary_input_layer(context_resources, resources_manager, layer_info, *hw_consts,
+ auto status = fill_boundary_input_layer(context_resources, resources_manager, layer_info, hw_consts,
hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
std::vector<ContextSwitchConfigActionPtr> actions;
- auto reset_burst_task_action = ResetBurstCreditsTaskAction::create();
- CHECK_EXPECTED_AS_STATUS(reset_burst_task_action);
- actions.emplace_back(reset_burst_task_action.release());
+ TRY(const auto reset_burst_task_action, ResetBurstCreditsTaskAction::create());
+ actions.emplace_back(std::move(reset_burst_task_action));
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY)) {
auto action = edge_layer.layer_info.direction == HAILO_H2D_STREAM ?
OpenBoundaryInputChannelAction::create(edge_layer.channel_id, edge_layer.buffer_info) :
OpenBoundaryOutputChannelAction::create(edge_layer.channel_id, edge_layer.buffer_info);
- CHECK_EXPECTED_AS_STATUS(action);
+ CHECK_EXPECTED_AS_STATUS(action); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
actions.emplace_back(action.release());
}
(ContextSwitchConfigAction::Type::EnableLcuNonDefault == action->get_type()), HAILO_INVALID_ARGUMENT,
"Invalid action type - must be enable lcu (default or non default) or switch lcu batch, Received type {}", action->get_type());
- const auto params_buffer = action->serialize_params(context_resources);
- CHECK_EXPECTED(params_buffer);
+ TRY(const auto params_buffer, action->serialize_params(context_resources));
if (ContextSwitchConfigAction::Type::EnableLcuDefault == action->get_type()) {
- const auto params = reinterpret_cast<const CONTEXT_SWITCH_DEFS__enable_lcu_action_default_data_t*>(params_buffer.value().data());
+ const auto params = reinterpret_cast<const CONTEXT_SWITCH_DEFS__enable_lcu_action_default_data_t*>(params_buffer.data());
cluster_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_CLUSTER_INDEX_READ(params->packed_lcu_id);
lcu_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_LCU_INDEX_READ(params->packed_lcu_id);
network_index = params->network_index;
kernel_done_count = CONTEXT_SWITCH_DEFS__ENABLE_LCU_DEFAULT_KERNEL_COUNT;
} else if (ContextSwitchConfigAction::Type::EnableLcuNonDefault == action->get_type()) {
- const auto params = reinterpret_cast<const CONTEXT_SWITCH_DEFS__enable_lcu_action_non_default_data_t*>(params_buffer.value().data());
+ const auto params = reinterpret_cast<const CONTEXT_SWITCH_DEFS__enable_lcu_action_non_default_data_t*>(params_buffer.data());
cluster_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_CLUSTER_INDEX_READ(params->packed_lcu_id);
lcu_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_LCU_INDEX_READ(params->packed_lcu_id);
network_index = params->network_index;
kernel_done_count = params->kernel_done_count;
} else if (ContextSwitchConfigAction::Type::SwitchLcuBatch == action->get_type()) {
- const auto params = reinterpret_cast<const CONTEXT_SWITCH_DEFS__switch_lcu_batch_action_data_t*>(params_buffer.value().data());
+ const auto params = reinterpret_cast<const CONTEXT_SWITCH_DEFS__switch_lcu_batch_action_data_t*>(params_buffer.data());
cluster_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_CLUSTER_INDEX_READ(params->packed_lcu_id);
lcu_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_LCU_INDEX_READ(params->packed_lcu_id);
network_index = params->network_index;
static hailo_status fill_batch_switching_context_edge_layers(ContextResources &context_resources, const CoreOpMetadata &core_op_metadata, ResourcesManager &resources_manager,
const HEFHwArch &hw_arch)
{
- auto hw_consts = Control::get_hw_consts(resources_manager.get_device());
- CHECK_EXPECTED_AS_STATUS(hw_consts);
- const bool should_optimize_credits = hw_consts->should_optimize_credits &&
+ TRY(const auto hw_consts, Control::get_hw_consts(resources_manager.get_device()));
+ const bool should_optimize_credits = hw_consts.should_optimize_credits &&
(HAILO_POWER_MODE_PERFORMANCE == resources_manager.get_power_mode());
for (const auto &output_layer_info : core_op_metadata.dynamic_contexts()[0].get_ddr_output_layers()) {
- auto status = fill_ddr_output_layer(context_resources, resources_manager, output_layer_info, *hw_consts, hw_arch);
+ auto status = fill_ddr_output_layer(context_resources, resources_manager, output_layer_info, hw_consts, hw_arch);
CHECK_SUCCESS(status);
}
for (const auto &output_layer_info : core_op_metadata.dynamic_contexts()[0].get_boundary_output_layers()) {
auto status = fill_boundary_output_layer(context_resources, resources_manager, output_layer_info,
- *hw_consts, hw_arch, should_optimize_credits);
+ hw_consts, hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
for (const auto &output_layer_info : core_op_metadata.dynamic_contexts()[0].get_inter_context_output_layers()) {
auto status = fill_inter_context_output_layer(context_resources, resources_manager, output_layer_info,
- *hw_consts, hw_arch, should_optimize_credits);
+ hw_consts, hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
for (const auto &input_layer_info : core_op_metadata.dynamic_contexts()[0].get_ddr_input_layers()) {
- auto status = fill_ddr_input_layer(context_resources, resources_manager, input_layer_info, *hw_consts, hw_arch);
+ auto status = fill_ddr_input_layer(context_resources, resources_manager, input_layer_info, hw_consts, hw_arch);
CHECK_SUCCESS(status);
}
for (const auto &input_layer_info : core_op_metadata.dynamic_contexts()[0].get_boundary_input_layers()) {
auto status = fill_boundary_input_layer(context_resources, resources_manager, input_layer_info,
- *hw_consts, hw_arch, should_optimize_credits);
+ hw_consts, hw_arch, should_optimize_credits);
CHECK_SUCCESS(status);
}
const auto lcu_batch_switch_actions = core_op_metadata.preliminary_context().get_actions_of_type(ENABLE_LCU_ACTIONS);
for (const auto &action : lcu_batch_switch_actions) {
- auto switch_lcu_batch_action = create_switch_lcu_batch_action(action, context_resources);
- CHECK_EXPECTED_AS_STATUS(switch_lcu_batch_action);
- actions.insert(actions.end(), switch_lcu_batch_action.release());
+ TRY(const auto switch_lcu_batch_action, create_switch_lcu_batch_action(action, context_resources));
+ actions.insert(actions.end(), std::move(switch_lcu_batch_action));
}
return HAILO_SUCCESS;
std::vector<ContextSwitchConfigActionPtr> &batch_switch_context_actions)
{
for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) {
- auto change_boundary_input_batch_action = ChangeBoundaryInputBatchAction::create(edge_layer.channel_id);
- CHECK_EXPECTED_AS_STATUS(change_boundary_input_batch_action);
- batch_switch_context_actions.emplace_back(change_boundary_input_batch_action.release());
+ TRY(const auto change_boundary_input_batch_action, ChangeBoundaryInputBatchAction::create(edge_layer.channel_id));
+ batch_switch_context_actions.emplace_back(std::move(change_boundary_input_batch_action));
}
- auto start_burst_credits_task_action = StartBurstCreditsTaskAction::create();
- CHECK_EXPECTED_AS_STATUS(start_burst_credits_task_action);
- batch_switch_context_actions.emplace_back(start_burst_credits_task_action.release());
+ TRY(const auto start_burst_credits_task_action, StartBurstCreditsTaskAction::create());
+ batch_switch_context_actions.emplace_back(std::move(start_burst_credits_task_action));
return HAILO_SUCCESS;
const auto BATCH_SWITCHING_CONTEXT = true;
for (const auto &edge_layer : context_resources.get_edge_layers()) {
if (edge_layer.layer_info.type != LayerType::BOUNDARY) {
- auto action = DeactivateChannelAction::create(edge_layer, BATCH_SWITCHING_CONTEXT);
- CHECK_EXPECTED_AS_STATUS(action);
- actions.emplace_back(action.release());
+ TRY(const auto action, DeactivateChannelAction::create(edge_layer, BATCH_SWITCHING_CONTEXT));
+ actions.emplace_back(std::move(action));
}
}
// We need to reset the ddr buffering task when we change the batch_size (since it depends on the batch_size param)
- auto reset_ddr_action = ResetDdrBufferingTaskAction::create();
- CHECK_EXPECTED_AS_STATUS(reset_ddr_action);
- actions.emplace_back(reset_ddr_action.release());
+ TRY(const auto reset_ddr_action, ResetDdrBufferingTaskAction::create());
+ actions.emplace_back(std::move(reset_ddr_action));
// Now re-open all the internal channels
const bool PUSH_INTERNAL_EDGE_LAYERS = true;
}
Expected<std::shared_ptr<ResourcesManager>> ResourcesManagerBuilder::build(uint8_t current_core_op_index, VdmaDevice &device,
- HailoRTDriver &driver, const ConfigureNetworkParams &config_params,
- std::shared_ptr<CoreOpMetadata> core_op_metadata, const HEFHwArch &hw_arch, std::shared_ptr<ShefFileHandle> shef_file_handle)
+ HailoRTDriver &driver, CacheManagerPtr cache_manager, const ConfigureNetworkParams &config_params,
+ std::shared_ptr<CoreOpMetadata> core_op_metadata, const HEFHwArch &hw_arch)
{
const auto num_contexts = core_op_metadata->dynamic_contexts().size() +
CONTROL_PROTOCOL__CONTEXT_SWITCH_NUMBER_OF_NON_DYNAMIC_CONTEXTS;
core_op_metadata->core_op_name(), network_params.first, HAILO_MAX_BATCH_SIZE);
}
- auto resources_manager = ResourcesManager::create(device, driver, config_params, core_op_metadata,
- current_core_op_index);
- CHECK_EXPECTED(resources_manager);
+ TRY(auto resources_manager, ResourcesManager::create(device, driver, config_params, cache_manager,
+ core_op_metadata, current_core_op_index));
// TODO: Use a new flag in config_params.stream_params_by_name to mark channels as async channels.
// will also used to mark streams as async in ConfiguredNetworkGroupBase::create_in/output_stream_from_config_params
// (HRT-9104)
- auto status = create_boundary_channels(resources_manager.value(), *core_op_metadata);
+ auto status = create_boundary_channels(resources_manager, *core_op_metadata);
CHECK_SUCCESS_AS_EXPECTED(status);
- status = resources_manager->fill_internal_buffers_info();
+ status = resources_manager.fill_internal_buffers_info();
CHECK_SUCCESS_AS_EXPECTED(status);
// No allocation of edge layers in the activation context. No need for context index here
auto INVLID_CONTEXT_INDEX = static_cast<uint16_t>(UINT16_MAX);
auto ACTIVATION_CONTEXT_INDEX = INVLID_CONTEXT_INDEX;
- auto activation_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION,
- ACTIVATION_CONTEXT_INDEX);
- CHECK_EXPECTED(activation_context);
- status = fill_activation_config_recepies_for_multi_context(activation_context.value().get(),
- resources_manager.value(), core_op_metadata, hw_arch);
+ TRY(auto activation_context, resources_manager.add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION,
+ ACTIVATION_CONTEXT_INDEX));
+ status = fill_activation_config_recepies_for_multi_context(activation_context.get(),
+ resources_manager, core_op_metadata, hw_arch);
CHECK_SUCCESS_AS_EXPECTED(status);
// No allocation of edge layers in the batch switching context. No need for context index here
auto BATCH_SWITCH_CONTEXT_INDEX = INVLID_CONTEXT_INDEX;
- auto batch_switching_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING,
- BATCH_SWITCH_CONTEXT_INDEX);
- CHECK_EXPECTED(batch_switching_context);
- status = fill_batch_switching_context_config_recepies_for_multi_context(batch_switching_context.value().get(),
- *core_op_metadata, resources_manager.value(), hw_arch);
+ TRY(auto batch_switching_context, resources_manager.add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING,
+ BATCH_SWITCH_CONTEXT_INDEX));
+ status = fill_batch_switching_context_config_recepies_for_multi_context(batch_switching_context.get(),
+ *core_op_metadata, resources_manager, hw_arch);
CHECK_SUCCESS_AS_EXPECTED(status);
- const bool is_single_context = core_op_metadata->dynamic_contexts().size() == 1;
-
- if (nullptr != shef_file_handle) {
- // We will start reading CCWs from the HEF file so we need to open it
- status = shef_file_handle->open();
- CHECK_SUCCESS_AS_EXPECTED(status);
- }
-
- auto PRELIMINARY_CONTEXT_INDEX = static_cast<uint16_t>(0);
- auto preliminary_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY,
- PRELIMINARY_CONTEXT_INDEX, core_op_metadata->preliminary_context().config_buffers_info());
- CHECK_EXPECTED(preliminary_context);
- status = fill_preliminary_config_recepies_for_multi_context(hw_arch, preliminary_context.value().get(),
- resources_manager.value(), core_op_metadata, core_op_metadata->preliminary_context(), is_single_context);
+ static const uint16_t PRELIMINARY_CONTEXT_INDEX = 0;
+ static const uint16_t FIRST_DYNAMIC_CONTEXT_INDEX = 1;
+ const auto is_single_context = (core_op_metadata->dynamic_contexts().size() == 1);
+ TRY(auto preliminary_context, resources_manager.add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY,
+ PRELIMINARY_CONTEXT_INDEX, core_op_metadata->preliminary_context().config_buffers_info()));
+ status = fill_preliminary_config_recepies_for_multi_context(hw_arch, preliminary_context.get(),
+ resources_manager, core_op_metadata, core_op_metadata->preliminary_context(), is_single_context);
CHECK_SUCCESS_AS_EXPECTED(status);
- uint16_t context_index = 0;
- auto FIRST_DYNAMIC_CONTEXT_INDEX = 1;
- for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) {
- auto new_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC,
- static_cast<uint16_t>(FIRST_DYNAMIC_CONTEXT_INDEX + context_index), context_metadata.config_buffers_info());
- CHECK_EXPECTED(new_context);
+ const auto caches_in_use = core_op_metadata->get_cache_layers_count() > 0;
+ CHECK_AS_EXPECTED(!caches_in_use || !is_single_context, HAILO_INVALID_ARGUMENT,
+ "Caches are in use but the network is single context");
- status = fill_context_recipes_for_multi_context(hw_arch, new_context.value().get(), resources_manager.value(),
- context_index, *core_op_metadata,
- context_metadata, is_single_context);
- CHECK_SUCCESS_AS_EXPECTED(status);
-
- context_index++;
- }
+ const auto num_dynamic_contexts = core_op_metadata->dynamic_contexts().size();
+ for (size_t context_index = 0; context_index < num_dynamic_contexts; context_index++) {
+ const auto &context_metadata = core_op_metadata->dynamic_contexts()[context_index];
+ TRY(auto new_context, resources_manager.add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC,
+ static_cast<uint16_t>(FIRST_DYNAMIC_CONTEXT_INDEX + context_index), context_metadata.config_buffers_info()));
- if (nullptr != shef_file_handle) {
- status = shef_file_handle->close();
+ const auto is_last_context = (context_index == (num_dynamic_contexts - 1));
+ status = fill_context_recipes_for_multi_context(hw_arch, new_context.get(), resources_manager,
+ static_cast<uint16_t>(context_index), *core_op_metadata, context_metadata, is_single_context,
+ is_last_context, caches_in_use);
CHECK_SUCCESS_AS_EXPECTED(status);
}
- status = resources_manager->configure();
+ status = resources_manager.configure();
CHECK_SUCCESS_AS_EXPECTED(status);
- auto resources_manager_ptr = make_shared_nothrow<ResourcesManager>(resources_manager.release());
+ auto resources_manager_ptr = make_shared_nothrow<ResourcesManager>(std::move(resources_manager));
CHECK_NOT_NULL_AS_EXPECTED(resources_manager_ptr, HAILO_OUT_OF_HOST_MEMORY);
return resources_manager_ptr;
#define _HAILO_RESOURCE_MANAGER_BUILDER_HPP_
#include "core_op/resource_manager/resource_manager.hpp"
+#include "core_op/resource_manager/cache_manager.hpp"
namespace hailort
{
-class ShefFileHandle;
+class Reader;
class ResourcesManagerBuilder final {
public:
ResourcesManagerBuilder() = delete;
static Expected<std::shared_ptr<ResourcesManager>> build(uint8_t net_group_index, VdmaDevice &device,
- HailoRTDriver &driver, const ConfigureNetworkParams &config_params,
- std::shared_ptr<CoreOpMetadata> core_op, const HEFHwArch &hw_arch, std::shared_ptr<ShefFileHandle> shef_file_handle);
+ HailoRTDriver &driver, CacheManagerPtr cache_manager, const ConfigureNetworkParams &config_params,
+ std::shared_ptr<CoreOpMetadata> core_op, const HEFHwArch &hw_arch);
};
return board_info;
}
-Expected<hailo_extended_device_information_t> control__parse_get_extended_device_information_results
- (CONTROL_PROTOCOL__get_extended_device_information_response_t &get_extended_device_information_response)
+Expected<hailo_extended_device_information_t> control__parse_get_extended_device_information_results(
+ const CONTROL_PROTOCOL__get_extended_device_information_response_t &get_extended_device_information_response)
{
uint8_t local_supported_features;
hailo_extended_device_information_t device_info;
return status;
}
+hailo_status Control::context_switch_init_cache_info(Device &device, const CONTROL_PROTOCOL__context_switch_cache_info_t &cache_info)
+{
+ CONTROL_PROTOCOL__request_t request{};
+ size_t request_size = 0;
+ uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {};
+ size_t response_size = RESPONSE_MAX_BUFFER_SIZE;
+ CONTROL_PROTOCOL__response_header_t *header = NULL;
+ CONTROL_PROTOCOL__payload_t *payload = NULL;
+
+ const auto common_status = CONTROL_PROTOCOL__pack_context_switch_init_cache_info_request(&request, &request_size,
+ device.get_control_sequence(), cache_info.cache_size, cache_info.current_read_offset, cache_info.write_offset_delta);
+ auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE;
+ CHECK_SUCCESS(status);
+
+ status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size);
+ CHECK_SUCCESS(status);
+
+ /* Parse response */
+ status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload,
+ &request, device);
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
+Expected<CONTROL_PROTOCOL__context_switch_cache_info_t> Control::context_switch_get_cache_info(Device &device)
+{
+ CONTROL_PROTOCOL__request_t request{};
+ size_t request_size = 0;
+ uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {};
+ size_t response_size = RESPONSE_MAX_BUFFER_SIZE;
+ CONTROL_PROTOCOL__response_header_t *header = NULL;
+ CONTROL_PROTOCOL__payload_t *payload = NULL;
+
+ const auto common_status = CONTROL_PROTOCOL__pack_context_switch_get_cache_info_request(&request, &request_size,
+ device.get_control_sequence());
+ auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE;
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ /* Parse response */
+ status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload,
+ &request, device);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ CONTROL_PROTOCOL__context_switch_cache_info_t result{};
+ result = *reinterpret_cast<CONTROL_PROTOCOL__context_switch_cache_info_t *>(payload->parameters);
+ return result;
+}
+
+hailo_status Control::context_switch_update_cache_read_offset(Device &device, int32_t read_offset_delta)
+{
+ CONTROL_PROTOCOL__request_t request{};
+ size_t request_size = 0;
+ uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {};
+ size_t response_size = RESPONSE_MAX_BUFFER_SIZE;
+ CONTROL_PROTOCOL__response_header_t *header = NULL;
+ CONTROL_PROTOCOL__payload_t *payload = NULL;
+
+ const auto common_status = CONTROL_PROTOCOL__pack_context_switch_update_cache_read_offset_request(&request, &request_size,
+ device.get_control_sequence(), read_offset_delta);
+ auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE;
+ CHECK_SUCCESS(status);
+
+ status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size);
+ CHECK_SUCCESS(status);
+
+ /* Parse response */
+ status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload,
+ &request, device);
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status Control::context_switch_signal_cache_updated(Device &device)
+{
+ CONTROL_PROTOCOL__request_t request{};
+ size_t request_size = 0;
+ uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {};
+ size_t response_size = RESPONSE_MAX_BUFFER_SIZE;
+ CONTROL_PROTOCOL__response_header_t *header = NULL;
+ CONTROL_PROTOCOL__payload_t *payload = NULL;
+
+ const auto common_status = CONTROL_PROTOCOL__pack_context_switch_signal_cache_updated_request(&request, &request_size,
+ device.get_control_sequence());
+ auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE;
+ CHECK_SUCCESS(status);
+
+ status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size);
+ CHECK_SUCCESS(status);
+
+ /* Parse response */
+ status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload,
+ &request, device);
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
hailo_status Control::context_switch_set_context_info(Device &device,
const std::vector<CONTROL_PROTOCOL__context_switch_context_info_chunk_t> &context_infos)
{
// Partial clusters layout is only relevant in HAILO_ARCH_HAILO8L and HAILO_ARCH_HAILO15M arch
return Expected<uint32_t>(PARTIAL_CLUSTERS_LAYOUT_IGNORE);
} else {
- auto extended_device_info_response = get_extended_device_info_response(device);
- CHECK_EXPECTED(extended_device_info_response);
- return BYTE_ORDER__ntohl(extended_device_info_response->partial_clusters_layout_bitmap);
+ TRY(const auto extended_device_info_response, get_extended_device_info_response(device));
+ return BYTE_ORDER__ntohl(extended_device_info_response.partial_clusters_layout_bitmap);
}
}
Expected<hailo_extended_device_information_t> Control::get_extended_device_information(Device &device)
{
- auto extended_device_info_response = get_extended_device_info_response(device);
- CHECK_EXPECTED(extended_device_info_response);
- return control__parse_get_extended_device_information_results(extended_device_info_response.value());
+ TRY(const auto extended_device_info_response, get_extended_device_info_response(device));
+ return control__parse_get_extended_device_information_results(extended_device_info_response);
}
Expected<hailo_health_info_t> Control::get_health_information(Device &device)
const std::vector<CONTROL_PROTOCOL__context_switch_context_info_chunk_t> &context_infos);
static hailo_status context_switch_set_network_group_header(Device &device,
const CONTROL_PROTOCOL__application_header_t &network_group_header);
+ static hailo_status context_switch_init_cache_info(Device &device, const CONTROL_PROTOCOL__context_switch_cache_info_t &cache_info);
+ static Expected<CONTROL_PROTOCOL__context_switch_cache_info_t> context_switch_get_cache_info(Device &device);
+ static hailo_status context_switch_update_cache_read_offset(Device &device, int32_t read_offset_delta);
+ static hailo_status context_switch_signal_cache_updated(Device &device);
static hailo_status wd_enable(Device &device, uint8_t cpu_id, bool should_enable);
static hailo_status wd_config(Device &device, uint8_t cpu_id, uint32_t wd_cycles, CONTROL_PROTOCOL__WATCHDOG_MODE_t wd_mode);
static hailo_status previous_system_state(Device &device, uint8_t cpu_id, CONTROL_PROTOCOL__system_state_t *system_state);
return status;
}
+HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_init_cache_info_request(
+ CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence,
+ uint32_t cache_size, uint32_t current_read_offset, int32_t write_offset_delta)
+{
+ HAILO_COMMON_STATUS_t status = HAILO_COMMON_STATUS__UNINITIALIZED;
+ size_t local_request_size = 0;
+
+ if ((NULL == request) || (NULL == request_size)) {
+ status = HAILO_STATUS__CONTROL_PROTOCOL__NULL_ARGUMENT_PASSED;
+ goto exit;
+ }
+
+ /* Header */
+ local_request_size = CONTROL_PROTOCOL__REQUEST_BASE_SIZE + sizeof(CONTROL_PROTOCOL__context_switch_init_cache_info_request_t);
+ control_protocol__pack_request_header(request, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_INIT_CACHE_INFO, 1);
+
+ /* Body */
+ request->parameters.context_switch_init_cache_info_request.cache_info_length =
+ BYTE_ORDER__htonl(sizeof(request->parameters.context_switch_init_cache_info_request.cache_info));
+ request->parameters.context_switch_init_cache_info_request.cache_info.cache_size = cache_size;
+ request->parameters.context_switch_init_cache_info_request.cache_info.current_read_offset = current_read_offset;
+ request->parameters.context_switch_init_cache_info_request.cache_info.write_offset_delta = write_offset_delta;
+
+ *request_size = local_request_size;
+ status = HAILO_COMMON_STATUS__SUCCESS;
+exit:
+ return status;
+}
+
+HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_get_cache_info_request(
+ CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence)
+{
+ return control_protocol__pack_empty_request(request, request_size, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_GET_CACHE_INFO);
+}
+
+HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_update_cache_read_offset_request(
+ CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, int32_t read_offset_delta)
+{
+ HAILO_COMMON_STATUS_t status = HAILO_COMMON_STATUS__UNINITIALIZED;
+ size_t local_request_size = 0;
+
+ if ((NULL == request) || (NULL == request_size)) {
+ status = HAILO_STATUS__CONTROL_PROTOCOL__NULL_ARGUMENT_PASSED;
+ goto exit;
+ }
+
+ /* Header */
+ local_request_size = CONTROL_PROTOCOL__REQUEST_BASE_SIZE + sizeof(CONTROL_PROTOCOL__context_switch_update_cache_read_offset_request_t);
+ control_protocol__pack_request_header(request, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_UPDATE_CACHE_READ_OFFSET, 1);
+
+ /* read_offset_delta */
+ request->parameters.context_switch_update_cache_read_offset_request.read_offset_delta_length =
+ BYTE_ORDER__htonl(sizeof(read_offset_delta));
+ request->parameters.context_switch_update_cache_read_offset_request.read_offset_delta = read_offset_delta;
+
+ *request_size = local_request_size;
+ status = HAILO_COMMON_STATUS__SUCCESS;
+exit:
+ return status;
+}
+
+HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_signal_cache_updated_request(
+ CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence)
+{
+ return control_protocol__pack_empty_request(request, request_size, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_SIGNAL_CACHE_UPDATED);
+}
+
HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_idle_time_get_measuremment_request(CONTROL_PROTOCOL__request_t *request,
size_t *request_size,
uint32_t sequence)
HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_set_network_group_header_request(
CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence,
const CONTROL_PROTOCOL__application_header_t *network_group_header);
+HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_init_cache_info_request(
+ CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, uint32_t cache_size,
+ uint32_t current_read_offset, int32_t write_offset_delta);
+HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_get_cache_info_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence);
+HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_update_cache_read_offset_request(
+ CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, int32_t read_offset_delta);
+HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_signal_cache_updated_request(
+ CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence);
HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_set_context_info_request(
- CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence,
+ CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence,
const CONTROL_PROTOCOL__context_switch_context_info_chunk_t *context_info);
HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_idle_time_set_measuremment_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, uint8_t measurement_enable);
HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_idle_time_get_measuremment_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence);
static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_health_monitor_clock_changed_event_notification(D2H_EVENT_MESSAGE_t *d2h_notification_message);
static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_hw_infer_manager_infer_done_notification(D2H_EVENT_MESSAGE_t *d2h_notification_message);
static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_context_switch_run_time_error_notification(D2H_EVENT_MESSAGE_t *d2h_notification_message);
+static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_start_update_cache_offset(D2H_EVENT_MESSAGE_t *d2h_notification_message);
/**********************************************************************
* Globals
D2H_EVENTS__parse_context_switch_breakpoint_reached,
D2H_EVENTS__parse_health_monitor_clock_changed_event_notification,
D2H_EVENTS__parse_hw_infer_manager_infer_done_notification,
- D2H_EVENTS__parse_context_switch_run_time_error_notification
+ D2H_EVENTS__parse_context_switch_run_time_error_notification,
+ D2H_EVENTS__parse_start_update_cache_offset,
};
/**********************************************************************
* Internal Functions
return status;
}
+static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_start_update_cache_offset(D2H_EVENT_MESSAGE_t *d2h_notification_message)
+{
+ if (D2H_EVENT_START_UPDATE_CACHE_OFFSET_PARAMETER_COUNT != d2h_notification_message->header.parameter_count) {
+ LOGGER__ERROR("d2h notification invalid parameter count: {}", d2h_notification_message->header.parameter_count);
+ return HAILO_STATUS__D2H_EVENTS__INCORRECT_PARAMETER_COUNT;
+ }
+
+ if(d2h_notification_message->header.payload_length != sizeof(d2h_notification_message->message_parameters.start_update_cache_offset_event)) {
+ LOGGER__ERROR("d2h notification invalid payload_length: {}", d2h_notification_message->header.payload_length);
+ return HAILO_STATUS__D2H_EVENTS__INCORRECT_PARAMETER_LENGTH;
+ }
+
+ LOGGER__TRACE("Got can update cache offset notification - cache_id_bitmask={}",
+ d2h_notification_message->message_parameters.start_update_cache_offset_event.cache_id_bitmask);
+
+ return HAILO_COMMON_STATUS__SUCCESS;
+}
+
static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_health_monitor_closed_streams_notification(D2H_EVENT_MESSAGE_t *d2h_notification_message)
{
HAILO_COMMON_STATUS_t status = HAILO_COMMON_STATUS__UNINITIALIZED;
return std::vector<std::string>{IntegratedDevice::DEVICE_ID};
}
else {
- auto pcie_device_infos = PcieDevice::scan();
- CHECK_EXPECTED(pcie_device_infos);
+ TRY(auto pcie_device_infos, PcieDevice::scan());
std::vector<std::string> results;
- results.reserve(pcie_device_infos->size());
+ results.reserve(pcie_device_infos.size());
- for (const auto pcie_device_info : pcie_device_infos.release()) {
- auto device_id = pcie_device_info_to_string(pcie_device_info);
- CHECK_EXPECTED(device_id);
- results.emplace_back(device_id.release());
+ for (const auto pcie_device_info : pcie_device_infos) {
+ TRY(auto device_id, pcie_device_info_to_string(pcie_device_info));
+ results.emplace_back(device_id);
}
return results;
Expected<std::unique_ptr<Device>> Device::create()
{
- auto device_ids = scan();
- CHECK_EXPECTED(device_ids, "Failed scan devices");
- CHECK_AS_EXPECTED(device_ids->size() >= 1, HAILO_INVALID_OPERATION, "There is no hailo device on the system");
+ TRY(const auto device_ids, scan(), "Failed scan devices");
+ CHECK_AS_EXPECTED(device_ids.size() >= 1, HAILO_INVALID_OPERATION,
+ "There is no hailo device on the system");
// Choose the first device.
- return Device::create(device_ids->at(0));
+ return Device::create(device_ids.at(0));
}
Expected<std::unique_ptr<Device>> Device::create(const std::string &device_id)
Expected<std::unique_ptr<Device>> Device::create_pcie()
{
- auto pcie_device = PcieDevice::create();
- CHECK_EXPECTED(pcie_device);
+ TRY(auto pcie_device, PcieDevice::create());
// Upcasting to Device unique_ptr (from PcieDevice unique_ptr)
- auto device = std::unique_ptr<Device>(pcie_device.release());
+ auto device = std::unique_ptr<Device>(std::move(pcie_device));
return device;
}
Expected<std::unique_ptr<Device>> Device::create_pcie(const hailo_pcie_device_info_t &device_info)
{
- auto pcie_device = PcieDevice::create(device_info);
- CHECK_EXPECTED(pcie_device);
+ TRY(auto pcie_device, PcieDevice::create(device_info));
// Upcasting to Device unique_ptr (from PcieDevice unique_ptr)
- auto device = std::unique_ptr<Device>(pcie_device.release());
+ auto device = std::unique_ptr<Device>(std::move(pcie_device));
return device;
}
Expected<std::unique_ptr<Device>> Device::create_eth(const hailo_eth_device_info_t &device_info)
{
- auto eth_device = EthernetDevice::create(device_info);
- CHECK_EXPECTED(eth_device);
+ TRY(auto eth_device, EthernetDevice::create(device_info));
// Upcasting to Device unique_ptr (from EthernetDevice unique_ptr)
- auto device = std::unique_ptr<Device>(eth_device.release());
+ auto device = std::unique_ptr<Device>(std::move(eth_device));
return device;
}
Expected<std::unique_ptr<Device>> Device::create_eth(const std::string &ip_addr)
{
- auto eth_device = EthernetDevice::create(ip_addr);
- CHECK_EXPECTED(eth_device);
+ TRY(auto eth_device, EthernetDevice::create(ip_addr));
// Upcasting to Device unique_ptr (from EthernetDevice unique_ptr)
- auto device = std::unique_ptr<Device>(eth_device.release());
+ auto device = std::unique_ptr<Device>(std::move(eth_device));
return device;
}
return HAILO_NOT_IMPLEMENTED;
}
+hailo_status Device::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ (void) dmabuf_fd;
+ (void) size;
+ (void) direction;
+ return HAILO_NOT_IMPLEMENTED;
+}
+
+hailo_status Device::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ (void) dmabuf_fd;
+ (void) size;
+ (void) direction;
+ return HAILO_NOT_IMPLEMENTED;
+}
+
hailo_status Device::direct_write_memory(uint32_t address, const void *buffer, uint32_t size)
{
(void) address;
hailo_status Device::update_fw_state()
{
// Assuming FW is loaded, send identify
- auto board_info_expected = Control::identify(*this);
- CHECK_EXPECTED_AS_STATUS(board_info_expected);
- hailo_device_identity_t board_info = board_info_expected.release();
+ TRY(auto board_info, Control::identify(*this));
if ((FIRMWARE_VERSION_MAJOR == board_info.fw_version.major) &&
(FIRMWARE_VERSION_MINOR == board_info.fw_version.minor)) {
CHECK_ARG_NOT_NULL_AS_EXPECTED(batch_counter);
// Allocate room for an action list of at most max_size bytes
- auto action_list = Buffer::create(max_size);
- CHECK_EXPECTED(action_list);
+ TRY(auto action_list, Buffer::create(max_size));
uint32_t base_address_local = 0;
uint32_t batch_counter_local = 0;
uint16_t actual_size = 0;
const auto status = Control::download_context_action_list(*this, network_group_id,
- (CONTROL_PROTOCOL__context_switch_context_type_t)context_type, context_index, action_list->size(),
- &base_address_local, action_list->data(), &actual_size, &batch_counter_local);
+ (CONTROL_PROTOCOL__context_switch_context_type_t)context_type, context_index, action_list.size(),
+ &base_address_local, action_list.data(), &actual_size, &batch_counter_local);
CHECK_SUCCESS_AS_EXPECTED(status);
CHECK_AS_EXPECTED(actual_size <= max_size, HAILO_INTERNAL_FAILURE);
// Create a copy of the list, truncating to the needed size
- auto final_action_list = Buffer::create(action_list->data(), actual_size);
- CHECK_EXPECTED(action_list);
+ TRY(auto final_action_list, Buffer::create(action_list.data(), actual_size));
// Transfer ownership of out params
*base_address = base_address_local;
*batch_counter = batch_counter_local;
- return final_action_list.release();
+ return final_action_list;
}
hailo_status Device::set_context_action_list_timestamp_batch(uint16_t batch_index)
return static_cast<uint8_t>(breakpoint_status);
}
+// TODO: remove init_cache_info, get_cache_info and update_cache_read_offset (HRT-13896)
+hailo_status Device::init_cache_info(const hailo_cache_info_t &cache_info)
+{
+ CONTROL_PROTOCOL__context_switch_cache_info_t control_cache_info = {
+ cache_info.cache_size,
+ cache_info.current_read_offset,
+ cache_info.write_offset_delta
+ };
+ return Control::context_switch_init_cache_info(*this, control_cache_info);
+}
+
+Expected<hailo_cache_info_t> Device::get_cache_info()
+{
+ TRY(const auto cache_info, Control::context_switch_get_cache_info(*this));
+ return hailo_cache_info_t{cache_info.cache_size, cache_info.current_read_offset, cache_info.write_offset_delta};
+}
+
+hailo_status Device::update_cache_read_offset(int32_t read_offset_delta)
+{
+ return Control::context_switch_update_cache_read_offset(*this, read_offset_delta);
+}
+
Expected<std::unique_ptr<Device>> Device::create_core()
{
- auto integrated_device = IntegratedDevice::create();
- CHECK_EXPECTED(integrated_device);
+ TRY(auto integrated_device, IntegratedDevice::create());
// Upcasting to Device unique_ptr (from IntegratedDevice unique_ptr)
- auto device = std::unique_ptr<Device>(integrated_device.release());
+ auto device = std::unique_ptr<Device>(std::move(integrated_device));
return device;
}
Expected<NetworkGroupsParamsMap> Device::create_configure_params(Hef &hef) const
{
- auto stream_interface = get_default_streams_interface();
- CHECK_EXPECTED(stream_interface, "Failed to get default streams interface");
-
- return hef.create_configure_params(stream_interface.release());
+ TRY(const auto stream_interface, get_default_streams_interface(), "Failed to get default streams interface");
+ return hef.create_configure_params(stream_interface);
}
Expected<ConfigureNetworkParams> Device::create_configure_params(Hef &hef, const std::string &network_group_name) const
{
- auto stream_interface = get_default_streams_interface();
- CHECK_EXPECTED(stream_interface, "Failed to get default streams interface");
-
- return hef.create_configure_params(stream_interface.release(), network_group_name);
+ TRY(const auto stream_interface, get_default_streams_interface(), "Failed to get default streams interface");
+ return hef.create_configure_params(stream_interface, network_group_name);
}
} /* namespace hailort */
auto status = check_hef_is_compatible(hef);
CHECK_SUCCESS_AS_EXPECTED(status);
- auto network_groups = add_hef(hef, configure_params);
- CHECK_EXPECTED(network_groups);
+ TRY(auto network_groups, add_hef(hef, configure_params));
auto elapsed_time_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start_time).count();
LOGGER__INFO("Configuring HEF took {} milliseconds", elapsed_time_ms);
Expected<firmware_type_t> DeviceBase::get_fw_type()
{
firmware_type_t firmware_type;
- const auto architecture = get_architecture();
- CHECK_EXPECTED(architecture);
+ TRY(const auto architecture, get_architecture());
- if ((architecture.value() == HAILO_ARCH_HAILO8) || (architecture.value() == HAILO_ARCH_HAILO8L)) {
+ if ((architecture == HAILO_ARCH_HAILO8) || (architecture == HAILO_ARCH_HAILO8L)) {
firmware_type = FIRMWARE_TYPE_HAILO8;
}
- else if ((architecture.value() == HAILO_ARCH_HAILO15H ) || (architecture.value() == HAILO_ARCH_HAILO15M)) {
+ else if ((architecture == HAILO_ARCH_HAILO15H ) || (architecture == HAILO_ARCH_HAILO15M)) {
firmware_type = FIRMWARE_TYPE_HAILO15;
}
- else if (architecture.value() == HAILO_ARCH_PLUTO) {
+ else if (architecture == HAILO_ARCH_PLUTO) {
firmware_type = FIRMWARE_TYPE_PLUTO;
}
else {
- LOGGER__ERROR("Invalid device arcitecture. {}", architecture.value());
+ LOGGER__ERROR("Invalid device arcitecture. {}", architecture);
return make_unexpected(HAILO_INVALID_DEVICE_ARCHITECTURE);
}
MD5_Update(&md5_ctx, firmware_binary.data(), firmware_binary.size());
MD5_Final(md5_sum, &md5_ctx);
- const auto firmware_type = get_fw_type();
- CHECK_EXPECTED_AS_STATUS(firmware_type);
+ TRY(const auto firmware_type, get_fw_type());
- fw_header_status = FIRMWARE_HEADER_UTILS__validate_fw_headers((uintptr_t) firmware_binary.data(), static_cast<uint32_t>(firmware_binary.size()), false,
- &new_app_firmware_header, &new_core_firmware_header, NULL, firmware_type.value());
+ fw_header_status = FIRMWARE_HEADER_UTILS__validate_fw_headers((uintptr_t)firmware_binary.data(),
+ static_cast<uint32_t>(firmware_binary.size()), false, &new_app_firmware_header,
+ &new_core_firmware_header, NULL, firmware_type);
CHECK(HAILO_COMMON_STATUS__SUCCESS == fw_header_status, HAILO_INVALID_FIRMWARE,
"FW update validation failed with status {}", fw_header_status);
// TODO: Are we ok with doing another identify here?
- auto board_info_before_update_expected = Control::identify(*this);
- CHECK_EXPECTED_AS_STATUS(board_info_before_update_expected);
- hailo_device_identity_t board_info_before_update = board_info_before_update_expected.release();
+ TRY(auto board_info_before_update, Control::identify(*this));
if (board_info_before_update.device_architecture != HAILO_ARCH_HAILO8_A0) {
if ((new_app_firmware_header->firmware_major != new_core_firmware_header->firmware_major) ||
return HAILO_SUCCESS;
}
- CHECK_EXPECTED_AS_STATUS(board_info_after_install_expected);
+ CHECK_EXPECTED_AS_STATUS(board_info_after_install_expected); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
hailo_device_identity_t board_info_after_install = board_info_after_install_expected.release();
LOGGER__INFO("New App FW version: {}.{}.{}{}", board_info_after_install.fw_version.major, board_info_after_install.fw_version.minor,
MD5_Update(&md5_ctx, second_stage_binary, second_stage_binary_length);
MD5_Final(md5_sum, &md5_ctx);
- const auto firmware_type = get_fw_type();
- CHECK_EXPECTED_AS_STATUS(firmware_type);
+ TRY(const auto firmware_type, get_fw_type());
- second_stage_header_status = FIRMWARE_HEADER_UTILS__validate_second_stage_headers((uintptr_t) second_stage_binary,
- second_stage_binary_length, &new_second_stage_header, firmware_type.value());
+ second_stage_header_status = FIRMWARE_HEADER_UTILS__validate_second_stage_headers((uintptr_t)second_stage_binary,
+ second_stage_binary_length, &new_second_stage_header, firmware_type);
CHECK(HAILO_COMMON_STATUS__SUCCESS == second_stage_header_status, HAILO_INVALID_SECOND_STAGE,
"Second stage update validation failed with status {}", second_stage_header_status);
CHECK(sensor_type != HAILO_SENSOR_TYPES_HAILO8_ISP, HAILO_INVALID_ARGUMENT,
"store_sensor_config intended only for sensor config, for ISP config use store_isp");
- auto control_buffers = SensorConfigUtils::read_config_file(config_file_path);
- CHECK_EXPECTED_AS_STATUS(control_buffers, "Failed reading config file");
-
- return store_sensor_control_buffers(control_buffers.value(), section_index, sensor_type,
+ TRY(auto control_buffers, SensorConfigUtils::read_config_file(config_file_path), "Failed reading config file");
+ return store_sensor_control_buffers(control_buffers, section_index, sensor_type,
reset_config_size, config_height, config_width, config_fps, config_name);
}
hailo_status DeviceBase::store_isp_config(uint32_t reset_config_size, uint16_t config_height, uint16_t config_width, uint16_t config_fps,
const std::string &isp_static_config_file_path, const std::string &isp_runtime_config_file_path, const std::string &config_name)
{
- auto control_buffers = SensorConfigUtils::read_isp_config_file(isp_static_config_file_path, isp_runtime_config_file_path);
- CHECK_EXPECTED_AS_STATUS(control_buffers, "Failed reading ISP config file");
-
- return store_sensor_control_buffers(control_buffers.value(), SENSOR_CONFIG__ISP_SECTION_INDEX, HAILO_SENSOR_TYPES_HAILO8_ISP,
+ TRY(auto control_buffers, SensorConfigUtils::read_isp_config_file(isp_static_config_file_path, isp_runtime_config_file_path),
+ "Failed reading ISP config file");
+ return store_sensor_control_buffers(control_buffers, SENSOR_CONFIG__ISP_SECTION_INDEX, HAILO_SENSOR_TYPES_HAILO8_ISP,
reset_config_size, config_height, config_width, config_fps, config_name);
}
Expected<Buffer> DeviceBase::sensor_get_sections_info()
{
- auto buffer = Buffer::create(SENSOR_SECTIONS_INFO_SIZE);
- CHECK_EXPECTED(buffer);
+ TRY(auto buffer, Buffer::create(SENSOR_SECTIONS_INFO_SIZE));
- hailo_status status = Control::sensor_get_sections_info(*this, buffer->data());
+ hailo_status status = Control::sensor_get_sections_info(*this, buffer.data());
CHECK_SUCCESS_AS_EXPECTED(status);
return buffer;
hailo_status DeviceBase::sensor_dump_config(uint32_t section_index, const std::string &config_file_path)
{
CHECK(SENSOR_CONFIG__TOTAL_SECTIONS_BLOCK_COUNT > section_index, HAILO_INVALID_ARGUMENT, "Section {} is invalid. Section index must be in the range [0 - {}]", section_index, (SENSOR_CONFIG__TOTAL_SECTIONS_BLOCK_COUNT - 1));
- auto sections_info_buffer = sensor_get_sections_info();
- CHECK_EXPECTED_AS_STATUS(sections_info_buffer);
+ TRY(auto sections_info_buffer, sensor_get_sections_info());
- SENSOR_CONFIG__section_info_t *section_info_ptr = &((SENSOR_CONFIG__section_info_t *)sections_info_buffer->data())[section_index];
+ SENSOR_CONFIG__section_info_t *section_info_ptr = &((SENSOR_CONFIG__section_info_t *)sections_info_buffer.data())[section_index];
CHECK(section_info_ptr->is_free == 0, HAILO_NOT_FOUND, "Section {} is not active", section_index);
CHECK(0 == (section_info_ptr->config_size % sizeof(SENSOR_CONFIG__operation_cfg_t)), HAILO_INVALID_OPERATION, "Section config size is invalid.");
/* Read config data from device */
- auto operation_cfg = Buffer::create(section_info_ptr->config_size);
- CHECK_EXPECTED_AS_STATUS(operation_cfg);
+ TRY(auto operation_cfg, Buffer::create(section_info_ptr->config_size));
size_t read_full_buffer_count = (section_info_ptr->config_size / MAX_CONFIG_ENTRIES_DATA_SIZE);
uint32_t residue_to_read = static_cast<uint32_t>(section_info_ptr->config_size - (read_full_buffer_count * MAX_CONFIG_ENTRIES_DATA_SIZE));
hailo_status status = HAILO_UNINITIALIZED;
for (uint32_t i = 0; i < read_full_buffer_count; i++) {
- status = Control::sensor_get_config(*this, section_index, offset, (uint32_t)MAX_CONFIG_ENTRIES_DATA_SIZE, (operation_cfg->data() + offset));
+ status = Control::sensor_get_config(*this, section_index, offset,
+ (uint32_t)MAX_CONFIG_ENTRIES_DATA_SIZE, (operation_cfg.data() + offset));
CHECK_SUCCESS(status);
offset += static_cast<uint32_t>(MAX_CONFIG_ENTRIES_DATA_SIZE);
}
if (0 < residue_to_read) {
- status = Control::sensor_get_config(*this, section_index, offset, residue_to_read, (operation_cfg->data() + offset));
+ status = Control::sensor_get_config(*this, section_index, offset, residue_to_read, (operation_cfg.data() + offset));
CHECK_SUCCESS(status);
}
- status = SensorConfigUtils::dump_config_to_csv((SENSOR_CONFIG__operation_cfg_t*)operation_cfg->data(), config_file_path, entries_count);
+ status = SensorConfigUtils::dump_config_to_csv((SENSOR_CONFIG__operation_cfg_t*)operation_cfg.data(), config_file_path, entries_count);
CHECK_SUCCESS(status);
return HAILO_SUCCESS;
Expected<Buffer> DeviceBase::read_board_config()
{
- auto result = Buffer::create(BOARD_CONFIG_SIZE, 0);
- CHECK_EXPECTED(result);
-
- auto status = Control::read_board_config(*this, result->data(), static_cast<uint32_t>(result->size()));
+ TRY(auto result, Buffer::create(BOARD_CONFIG_SIZE, 0));
+ auto status = Control::read_board_config(*this, result.data(), static_cast<uint32_t>(result.size()));
CHECK_SUCCESS_AS_EXPECTED(status);
- return result.release();
+ return result;
}
hailo_status DeviceBase::write_board_config(const MemoryView &buffer)
Expected<Buffer> DeviceBase::read_user_config()
{
- auto user_config_info = examine_user_config();
- CHECK_EXPECTED(user_config_info, "Failed to examine user config");
+ TRY(auto user_config_info, examine_user_config(), "Failed to examine user config");
+ TRY(auto result, Buffer::create(user_config_info.total_size, 0));
- auto result = Buffer::create(user_config_info->total_size, 0);
- CHECK_EXPECTED(result);
-
- auto status = Control::read_user_config(*this, result->data(), static_cast<uint32_t>(result->size()));
+ auto status = Control::read_user_config(*this, result.data(), static_cast<uint32_t>(result.size()));
CHECK_SUCCESS_AS_EXPECTED(status);
- return result.release();
+ return result;
}
hailo_status DeviceBase::write_user_config(const MemoryView &buffer)
hailo_status DeviceBase::check_hef_is_compatible(Hef &hef)
{
- const auto device_arch = get_architecture();
- CHECK_EXPECTED_AS_STATUS(device_arch, "Can't get device architecture (is the FW loaded?)");
+ TRY(const auto device_arch, get_architecture(), "Can't get device architecture (is the FW loaded?)");
- if (!is_hef_compatible(device_arch.value(), static_cast<HEFHwArch>(hef.pimpl->get_device_arch()))) {
- auto device_arch_str = HailoRTCommon::get_device_arch_str(device_arch.value());
+ if (!is_hef_compatible(device_arch, static_cast<HEFHwArch>(hef.pimpl->get_device_arch()))) {
+ auto device_arch_str = HailoRTCommon::get_device_arch_str(device_arch);
auto hef_arch_str =
HailoRTCommon::get_device_arch_str(hef_arch_to_device_arch(static_cast<HEFHwArch>(hef.pimpl->get_device_arch())));
}
// TODO: MSW-227 check clock rate for hailo15 as well.
- if ((HAILO_ARCH_HAILO8 == device_arch.value()) || (HAILO_ARCH_HAILO8L == device_arch.value())) {
- auto extended_device_info_expected = Control::get_extended_device_information(*this);
- CHECK_EXPECTED_AS_STATUS(extended_device_info_expected, "Can't get device extended info");
- hailo_extended_device_information_t extended_device_information = extended_device_info_expected.release();
+ if ((HAILO_ARCH_HAILO8 == device_arch) || (HAILO_ARCH_HAILO8L == device_arch)) {
+ TRY(auto extended_device_information, Control::get_extended_device_information(*this), "Can't get device extended info");
check_clock_rate_for_hailo8(extended_device_information.neural_network_core_clock_rate,
static_cast<HEFHwArch>(hef.pimpl->get_device_arch()));
}
if ((static_cast<ProtoHEFHwArch>(HEFHwArch::HW_ARCH__HAILO8L) == hef.pimpl->get_device_arch()) &&
- (HAILO_ARCH_HAILO8 == device_arch.value())) {
+ (HAILO_ARCH_HAILO8 == device_arch)) {
LOGGER__WARNING("HEF was compiled for Hailo8L device, while the device itself is Hailo8. " \
"This will result in lower performance.");
} else if ((static_cast<ProtoHEFHwArch>(HEFHwArch::HW_ARCH__HAILO15M) == hef.pimpl->get_device_arch()) &&
- (HAILO_ARCH_HAILO15H == device_arch.value())) {
+ (HAILO_ARCH_HAILO15H == device_arch)) {
LOGGER__WARNING("HEF was compiled for Hailo15M device, while the device itself is Hailo15H. " \
"This will result in lower performance.");
}
case CONTEXT_SWITCH_RUN_TIME_ERROR:
*hailo_notification_id = HAILO_NOTIFICATION_ID_CONTEXT_SWITCH_RUN_TIME_ERROR_EVENT;
break;
+ case START_UPDATE_CACHE_OFFSET_ID:
+ *hailo_notification_id = HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET;
+ break;
default:
status = HAILO_INVALID_ARGUMENT;
goto l_exit;
case HAILO_ARCH_HAILO8L:
return (hef_arch == HEFHwArch::HW_ARCH__HAILO8L);
case HAILO_ARCH_HAILO15H:
+ case HAILO_ARCH_HAILO10H:
// Compare with HW_ARCH__LAVENDER and HW_ARCH__GINGER to support hefs compiled for them
return (hef_arch == HEFHwArch::HW_ARCH__GINGER) || (hef_arch == HEFHwArch::HW_ARCH__LAVENDER) ||
- (hef_arch == HEFHwArch::HW_ARCH__HAILO15H) || (hef_arch == HEFHwArch::HW_ARCH__HAILO15M);
+ (hef_arch == HEFHwArch::HW_ARCH__HAILO15H) || (hef_arch == HEFHwArch::HW_ARCH__HAILO15M) || (hef_arch == HEFHwArch::HW_ARCH__HAILO10H);
case HAILO_ARCH_PLUTO:
return (hef_arch == HEFHwArch::HW_ARCH__PLUTO);
case HAILO_ARCH_HAILO15M:
return HAILO_ARCH_PLUTO;
case HEFHwArch::HW_ARCH__HAILO15M:
return HAILO_ARCH_HAILO15M;
+ case HEFHwArch::HW_ARCH__HAILO10H:
+ return HAILO_ARCH_HAILO10H;
default:
return HAILO_ARCH_MAX_ENUM;
HW_ARCH__HAILO8L = 3,
HW_ARCH__HAILO15H = 103,
HW_ARCH__HAILO15M = 4,
+ HW_ARCH__HAILO10H = 5,
HW_ARCH__SAGE_A0 = 100,
HW_ARCH__SAGE_B0 = 101,
CONTROL_PROTOCOL__payload_t *payload = NULL;
/* Create udp socket */
- auto udp = Udp::create(m_device_info.device_address.sin_addr, m_device_info.device_address.sin_port,
- m_device_info.host_address.sin_addr, m_device_info.host_address.sin_port);
- CHECK_EXPECTED_AS_STATUS(udp);
+ TRY(auto udp, Udp::create(m_device_info.device_address.sin_addr, m_device_info.device_address.sin_port,
+ m_device_info.host_address.sin_addr, m_device_info.host_address.sin_port));
- status = udp->set_timeout(std::chrono::milliseconds(WAIT_FOR_DEVICE_WAKEUP_TIMEOUT));
+ status = udp.set_timeout(std::chrono::milliseconds(WAIT_FOR_DEVICE_WAKEUP_TIMEOUT));
CHECK_SUCCESS(status);
- status = udp->set_max_number_of_attempts(WAIT_FOR_DEVICE_WAKEUP_MAX_ATTEMPTS);
+ status = udp.set_max_number_of_attempts(WAIT_FOR_DEVICE_WAKEUP_MAX_ATTEMPTS);
CHECK_SUCCESS(status);
/* Create and send identify-control until it runs successfully */
status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE;
CHECK_SUCCESS(status);
- status = udp->fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size,
+ status = udp.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size,
m_control_sequence);
// Always increment sequence
hailo_status status = HAILO_UNINITIALIZED;
// Creates control socket
- auto udp = Udp::create(device_info.device_address.sin_addr, device_info.device_address.sin_port,
- device_info.host_address.sin_addr, device_info.host_address.sin_port);
- CHECK_EXPECTED(udp, "Failed to init control socket.");
+ TRY(auto udp, Udp::create(device_info.device_address.sin_addr, device_info.device_address.sin_port,
+ device_info.host_address.sin_addr, device_info.host_address.sin_port), "Failed to init control socket.");
- auto device = std::unique_ptr<EthernetDevice>(new (std::nothrow) EthernetDevice(device_info, udp.release(), status));
+ auto device = std::unique_ptr<EthernetDevice>(new (std::nothrow) EthernetDevice(device_info, std::move(udp), status));
CHECK_AS_EXPECTED((nullptr != device), HAILO_OUT_OF_HOST_MEMORY);
- if (HAILO_SUCCESS != status) {
- LOGGER__ERROR("Failed creating EthernetDevice");
- return make_unexpected(status);
- }
+ CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating EthernetDevice");
return device;
}
Expected<std::unique_ptr<EthernetDevice>> EthernetDevice::create(const std::string &ip_addr)
{
const bool LOG_ON_FAILURE = true;
- auto device_info = parse_eth_device_info(ip_addr, LOG_ON_FAILURE);
- CHECK_EXPECTED(device_info, "Failed to parse ip address {}", ip_addr);
- return create(device_info.release());
+ TRY(const auto device_info, parse_eth_device_info(ip_addr, LOG_ON_FAILURE),
+ "Failed to parse ip address {}", ip_addr);
+ return create(device_info);
}
EthernetDevice::EthernetDevice(const hailo_eth_device_info_t &device_info, Udp &&control_udp, hailo_status &status) :
std::chrono::milliseconds timeout)
{
// Convert interface name to IP address
- auto interface_ip_address = EthernetUtils::get_ip_from_interface(interface_name);
- CHECK_EXPECTED(interface_ip_address);
-
- return scan_by_host_address(*interface_ip_address, timeout);
+ TRY(const auto interface_ip_address, EthernetUtils::get_ip_from_interface(interface_name));
+ return scan_by_host_address(interface_ip_address, timeout);
}
hailo_status get_udp_broadcast_params(const char *host_address, struct in_addr &interface_ip_address,
CHECK_SUCCESS_AS_EXPECTED(status);
/* Create broadcast udp object */
- auto udp_broadcast = Udp::create(broadcast_ip_address, HAILO_DEFAULT_ETH_CONTROL_PORT, interface_ip_address, 0);
- CHECK_EXPECTED(udp_broadcast);
- status = udp_broadcast->set_timeout(timeout);
+ TRY(auto udp_broadcast, Udp::create(broadcast_ip_address, HAILO_DEFAULT_ETH_CONTROL_PORT, interface_ip_address, 0));
+ status = udp_broadcast.set_timeout(timeout);
CHECK_SUCCESS_AS_EXPECTED(status);
/* Build identify request */
CHECK_SUCCESS_AS_EXPECTED(status);
/* Send broadcast identify request */
- status = udp_broadcast->send((uint8_t *)&request, &request_size, false, MAX_UDP_PAYLOAD_SIZE);
+ status = udp_broadcast.send((uint8_t *)&request, &request_size, false, MAX_UDP_PAYLOAD_SIZE);
CHECK_SUCCESS_AS_EXPECTED(status);
/* Receive all responses */
- return eth_device__receive_responses(*udp_broadcast);
+ return eth_device__receive_responses(udp_broadcast);
}
Expected<hailo_eth_device_info_t> EthernetDevice::parse_eth_device_info(const std::string &ip_addr,
auto status = Control::reset_context_switch_state_machine(*this);
CHECK_SUCCESS_AS_EXPECTED(status);
- auto added_network_groups = create_networks_group_vector(hef, configure_params);
- CHECK_EXPECTED(added_network_groups);
+ TRY(auto added_network_groups, create_networks_group_vector(hef, configure_params));
return added_network_groups;
}
Expected<ConfiguredNetworkGroupVector> EthernetDevice::create_networks_group_vector(Hef &hef, const NetworkGroupsParamsMap &configure_params)
{
- auto partial_clusters_layout_bitmap_exp = Control::get_partial_clusters_layout_bitmap(*this);
- CHECK_EXPECTED(partial_clusters_layout_bitmap_exp);
- auto partial_clusters_layout_bitmap = partial_clusters_layout_bitmap_exp.release();
+ TRY(auto partial_clusters_layout_bitmap, Control::get_partial_clusters_layout_bitmap(*this));
auto &hef_network_groups = hef.pimpl->network_groups();
auto configure_params_copy = configure_params;
config_params = configure_params_copy.at(network_group_name);
configure_params_copy.erase(network_group_name);
} else if (configure_params.empty()) {
- auto config_params_exp = create_configure_params(hef, network_group_name);
- CHECK_EXPECTED(config_params_exp);
- config_params = config_params_exp.release();
+ TRY(config_params, create_configure_params(hef, network_group_name));
} else {
continue;
}
- auto net_group_config = create_core_op_metadata(hef, network_group_name, partial_clusters_layout_bitmap);
- CHECK_EXPECTED(net_group_config);
+ TRY(auto net_group_config, create_core_op_metadata(hef, network_group_name, partial_clusters_layout_bitmap));
// TODO: move to func, support multiple core ops
std::vector<std::shared_ptr<CoreOp>> core_ops_ptrs;
- auto core_op_metadata = hef.pimpl->get_core_op_metadata(network_group_name);
- CHECK_EXPECTED(core_op_metadata);
- auto core_op_metadata_ptr = core_op_metadata.release();
+ TRY(auto core_op_metadata_ptr, hef.pimpl->get_core_op_metadata(network_group_name));
auto metadata = hef.pimpl->network_group_metadata(core_op_metadata_ptr->core_op_name());
auto status = HAILO_UNINITIALIZED;
- auto single_context_app = HcpConfigCoreOp(*this, m_active_core_op_holder, net_group_config.release(),
+ auto single_context_app = HcpConfigCoreOp(*this, m_active_core_op_holder, std::move(net_group_config),
config_params, core_op_metadata_ptr, status);
CHECK_SUCCESS_AS_EXPECTED(status);
m_core_ops.push_back(core_op_ptr);
core_ops_ptrs.push_back(core_op_ptr);
- auto net_group_expected = ConfiguredNetworkGroupBase::create(config_params, std::move(core_ops_ptrs), std::move(metadata));
- CHECK_EXPECTED(net_group_expected);
- auto net_group_ptr = net_group_expected.release();
+ TRY(auto net_group_ptr, ConfiguredNetworkGroupBase::create(config_params,
+ std::move(core_ops_ptrs), std::move(metadata)));
added_network_groups.emplace_back(net_group_ptr);
m_network_groups.push_back(net_group_ptr);
Expected<std::vector<WriteMemoryInfo>> EthernetDevice::create_core_op_metadata(Hef &hef, const std::string &core_op_name, uint32_t partial_clusters_layout_bitmap)
{
- auto device_arch_exp = get_architecture();
- CHECK_EXPECTED(device_arch_exp);
- auto device_arch = device_arch_exp.release();
+ TRY(const auto device_arch, get_architecture());
auto hef_arch = hef.pimpl->get_device_arch();
auto &hef_core_ops = hef.pimpl->core_ops(core_op_name);
assert(1 == hef_core_ops.size());
const auto &core_op = hef_core_ops[0];
- auto expected_partial_core_op = Hef::Impl::get_core_op_per_arch(core_op, hef_arch, device_arch,
- partial_clusters_layout_bitmap);
- CHECK_EXPECTED(expected_partial_core_op);
- auto partial_core_op = expected_partial_core_op.release();
+ TRY(auto partial_core_op, Hef::Impl::get_core_op_per_arch(core_op, hef_arch, device_arch,
+ partial_clusters_layout_bitmap));
// TODO: decide about core_op names - align with the Compiler
/* Update preliminary_config and dynamic_contexts recepies */
auto &proto_preliminary_config = partial_core_op->preliminary_config;
- auto core_op_config = Hef::Impl::create_single_context_core_op_config(proto_preliminary_config);
- CHECK_EXPECTED(core_op_config);
+ TRY(auto core_op_config, Hef::Impl::create_single_context_core_op_config(proto_preliminary_config));
return core_op_config;
}
while (offset < offset_end_without_remainder) {
transfer_size = offset_end_without_remainder - offset;
- auto expected_bytes_written = sync_write_raw_buffer(MemoryView::create_const(static_cast<const uint8_t*>(buffer) + offset, transfer_size));
- if (HAILO_STREAM_ABORT == expected_bytes_written.status()) {
- LOGGER__INFO("sync_write_raw_buffer was aborted!");
- return expected_bytes_written.status();
- }
- CHECK_EXPECTED_AS_STATUS(expected_bytes_written);
- offset += expected_bytes_written.release();
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto bytes_written,
+ sync_write_raw_buffer(MemoryView::create_const(static_cast<const uint8_t*>(buffer) + offset, transfer_size)));
+ offset += bytes_written;
}
if (0 < remainder_size) {
- auto expected_bytes_written = sync_write_raw_buffer(MemoryView::create_const(static_cast<const uint8_t*>(buffer) + offset, remainder_size));
- if (HAILO_STREAM_ABORT == expected_bytes_written.status()) {
- LOGGER__INFO("sync_write_raw_buffer was aborted!");
- return expected_bytes_written.status();
- }
- CHECK_EXPECTED_AS_STATUS(expected_bytes_written);
- assert(expected_bytes_written.value() == remainder_size);
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto bytes_written,
+ sync_write_raw_buffer(MemoryView::create_const(static_cast<const uint8_t*>(buffer) + offset, remainder_size)));
+ (void)bytes_written;
+ assert(bytes_written == remainder_size);
}
return HAILO_SUCCESS;
(void)token_bucket.consumeWithBorrowAndWait(MAX_CONSUME_SIZE, rate_bytes_per_sec, BURST_SIZE);
transfer_size = offset_end_without_remainder - offset;
- auto expected_bytes_written = sync_write_raw_buffer(MemoryView::create_const(static_cast<const uint8_t*>(buffer) + offset, transfer_size));
- if (HAILO_STREAM_ABORT == expected_bytes_written.status()) {
- LOGGER__INFO("sync_write_raw_buffer was aborted!");
- return expected_bytes_written.status();
- }
- CHECK_EXPECTED_AS_STATUS(expected_bytes_written);
- offset += expected_bytes_written.release();
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto bytes_written,
+ sync_write_raw_buffer(MemoryView::create_const(static_cast<const uint8_t*>(buffer) + offset, transfer_size)));
+ offset += bytes_written;
}
if (0 < remainder_size) {
// We don't static_assert that "remainder_size <= BURST_SIZE", so the call could fail in theory.
// However, since remainder_size is modulo MAX_UDP_PAYLOAD_SIZE and BURST_SIZE == MAX_UDP_PAYLOAD_SIZE, it should be smaller.
(void)token_bucket.consumeWithBorrowAndWait(static_cast<double>(remainder_size), rate_bytes_per_sec, BURST_SIZE);
-
- auto expected_bytes_written = sync_write_raw_buffer(MemoryView::create_const(static_cast<const uint8_t*>(buffer) + offset, remainder_size));
- if (HAILO_STREAM_ABORT == expected_bytes_written.status()) {
- LOGGER__INFO("sync_write_raw_buffer was aborted!");
- return expected_bytes_written.status();
- }
- CHECK_EXPECTED_AS_STATUS(expected_bytes_written);
- assert(expected_bytes_written.value() == remainder_size);
+
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto bytes_written,
+ sync_write_raw_buffer(MemoryView::create_const(static_cast<const uint8_t*>(buffer) + offset, remainder_size)));
+ (void)bytes_written;
+ assert(bytes_written == remainder_size);
}
return HAILO_SUCCESS;
Expected<std::unique_ptr<TrafficControlEthernetInputStream>> TrafficControlEthernetInputStream::create(
Device &device, Udp &&udp, EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info)
{
- auto board_ip = get_interface_address(&udp.m_device_address.sin_addr);
- CHECK_EXPECTED(board_ip, "get_interface_address failed with status {}", board_ip.status());
-
+ TRY(const auto board_ip, get_interface_address(&udp.m_device_address.sin_addr));
const auto board_port = BYTE_ORDER__ntohs(udp.m_device_address.sin_port);
- auto tc = TrafficControl::create(board_ip.value(), board_port, rate_bytes_per_sec);
- CHECK_EXPECTED(tc, "Creating traffic control at rate {} failed with error {}", rate_bytes_per_sec, tc.status());
+ TRY(auto tc, TrafficControl::create(board_ip, board_port, rate_bytes_per_sec),
+ "Creating traffic control at rate {} failed", rate_bytes_per_sec);
auto status = HAILO_UNINITIALIZED;
// Note: we don't use make_unique because TrafficControlEthernetInputStream's ctor is private
auto tc_ptr = std::unique_ptr<TrafficControlEthernetInputStream>(new (std::nothrow)
TrafficControlEthernetInputStream(device, std::move(udp), std::move(core_op_activated_event), rate_bytes_per_sec,
- tc.release(), layer_info, status));
+ std::move(tc), layer_info, status));
CHECK_AS_EXPECTED(nullptr != tc_ptr, HAILO_OUT_OF_HOST_MEMORY);
CHECK_SUCCESS_AS_EXPECTED(status);
return tc_ptr;
Expected<std::string> TrafficControlEthernetInputStream::get_interface_address(const struct in_addr *addr)
{
- auto ip = Buffer::create(IPV4_STRING_MAX_LENGTH, 0);
- CHECK_EXPECTED(ip);
+ TRY(const auto ip, Buffer::create(IPV4_STRING_MAX_LENGTH, 0));
- const auto result = Socket::ntop(AF_INET, addr, ip->as_pointer<char>(), EthernetUtils::MAX_INTERFACE_SIZE);
+ const auto result = Socket::ntop(AF_INET, addr, ip.as_pointer<char>(), EthernetUtils::MAX_INTERFACE_SIZE);
CHECK_SUCCESS_AS_EXPECTED(result, "Failed parsing IP to string with status {}", result);
- return ip->to_string();
+ return ip.to_string();
}
TrafficControlEthernetInputStream::TrafficControlEthernetInputStream(Device &device, Udp &&udp,
std::unique_ptr<EthernetInputStream> local_stream;
auto stream_index = edge_layer.stream_index;
- auto udp = eth_stream__create_udp(eth_device, params.host_address, stream_index, params.device_port, true);
- CHECK_EXPECTED(udp);
+ TRY(auto udp, eth_stream__create_udp(eth_device, params.host_address, stream_index, params.device_port, true));
if (params.rate_limit_bytes_per_sec == 0) {
local_stream = std::unique_ptr<EthernetInputStream>(
- new (std::nothrow) EthernetInputStream(device, udp.release(), std::move(core_op_activated_event), edge_layer, status));
+ new (std::nothrow) EthernetInputStream(device, std::move(udp), std::move(core_op_activated_event), edge_layer, status));
CHECK_SUCCESS_AS_EXPECTED(status);
} else {
#ifdef _MSC_VER
// TODO: Add factory class
local_stream = std::unique_ptr<EthernetInputStream>(
- new (std::nothrow) TokenBucketEthernetInputStream(device, udp.release(),
+ new (std::nothrow) TokenBucketEthernetInputStream(device, std::move(udp),
std::move(core_op_activated_event), params.rate_limit_bytes_per_sec, edge_layer, status));
CHECK_SUCCESS_AS_EXPECTED(status);
#else
- auto stream_expected = TrafficControlEthernetInputStream::create(device, udp.release(),
- std::move(core_op_activated_event), params.rate_limit_bytes_per_sec, edge_layer);
- CHECK_EXPECTED(stream_expected);
- local_stream = stream_expected.release();
+ TRY(local_stream, TrafficControlEthernetInputStream::create(device, std::move(udp),
+ std::move(core_op_activated_event), params.rate_limit_bytes_per_sec, edge_layer));
#endif
}
CHECK_AS_EXPECTED((nullptr != local_stream), HAILO_OUT_OF_HOST_MEMORY);
local_stream->m_is_stream_activated = false;
- auto device_architecture = eth_device->get_architecture();
- CHECK_EXPECTED(device_architecture);
- if ((HAILO_ARCH_HAILO8 == device_architecture.value()) || (HAILO_ARCH_HAILO8L == device_architecture.value())) {
+ TRY(const auto device_architecture, eth_device->get_architecture());
+ if ((HAILO_ARCH_HAILO8 == device_architecture) || (HAILO_ARCH_HAILO8L == device_architecture)) {
local_stream->configuration.use_dataflow_padding = true;
}
else {
while (offset < offset_end) {
transfer_size = offset_end - offset;
MemoryView buffer_view(static_cast<uint8_t*>(buffer) + offset, transfer_size);
- auto expected_bytes_read = this->sync_read_raw_buffer(buffer_view);
- if (HAILO_STREAM_ABORT == expected_bytes_read.status()) {
- LOGGER__INFO("sync_read_raw_buffer was aborted!");
- return expected_bytes_read.status();
- }
- CHECK_EXPECTED_AS_STATUS(expected_bytes_read);
- offset += expected_bytes_read.release();
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, auto bytes_read, this->sync_read_raw_buffer(buffer_view));
+ offset += bytes_read;
}
return HAILO_SUCCESS;
MemoryView leftover_buffer_view(this->leftover_buffer, last_packet_size);
auto expected_bytes_read = sync_read_raw_buffer(leftover_buffer_view);
CHECK(HAILO_TIMEOUT != expected_bytes_read.status(), HAILO_INVALID_FRAME, "Got timeout on last sync, marking last frame as invalid");
- CHECK_EXPECTED_AS_STATUS(expected_bytes_read, "Recv error");
+ CHECK_EXPECTED_AS_STATUS(expected_bytes_read, "Recv error"); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
last_packet_size = expected_bytes_read.release();
if (is_sync_packet(this->leftover_buffer, 0, last_packet_size)) {
auto eth_device = reinterpret_cast<EthernetDevice*>(&device);
const auto stream_index = edge_layer.stream_index;
- auto udp = eth_stream__create_udp(eth_device, params.host_address, stream_index, params.device_port, false);
- CHECK_EXPECTED(udp);
+ TRY(auto udp, eth_stream__create_udp(eth_device, params.host_address, stream_index, params.device_port, false));
local_stream = std::unique_ptr<EthernetOutputStream>(new (std::nothrow) EthernetOutputStream(device,
- edge_layer,
- udp.release(), std::move(core_op_activated_event), status));
+ edge_layer, std::move(udp), std::move(core_op_activated_event), status));
CHECK((nullptr != local_stream), make_unexpected(HAILO_OUT_OF_HOST_MEMORY));
CHECK_SUCCESS_AS_EXPECTED(status);
return make_unexpected(HAILO_INVALID_OPERATION);
}
+bool HcpConfigCoreOp::has_caches() const
+{
+ return false;
+}
+
+Expected<uint32_t> HcpConfigCoreOp::get_cache_read_size() const
+{
+ LOGGER__ERROR("get_cache_read_size function is not supported on ETH core-ops");
+ return make_unexpected(HAILO_INVALID_OPERATION);
+}
+
+Expected<uint32_t> HcpConfigCoreOp::get_cache_write_size() const
+{
+ LOGGER__ERROR("get_cache_write_size function is not supported on ETH core-ops");
+ return make_unexpected(HAILO_INVALID_OPERATION);
+}
+
+
+hailo_status HcpConfigCoreOp::init_cache(uint32_t read_offset, int32_t write_offset_delta)
+{
+ (void) read_offset;
+ (void) write_offset_delta;
+ LOGGER__ERROR("init_cache function is not supported on ETH core-ops");
+ return HAILO_INVALID_OPERATION;
+}
+
+Expected<hailo_cache_info_t> HcpConfigCoreOp::get_cache_info() const
+{
+ LOGGER__ERROR("get_cache_info function is not supported on ETH core-ops");
+ return make_unexpected(HAILO_INVALID_OPERATION);
+}
+
+hailo_status HcpConfigCoreOp::update_cache_offset(int32_t offset_delta_bytes)
+{
+ (void) offset_delta_bytes;
+ LOGGER__ERROR("update_cache_offset function is not supported on ETH core-ops");
+ return HAILO_INVALID_OPERATION;
+}
+
hailo_status HcpConfigCoreOp::activate_impl(uint16_t /* dynamic_batch_size */)
{
// Close older dataflows
virtual hailo_status deactivate_impl() override;
virtual hailo_status shutdown() override;
virtual Expected<HwInferResults> run_hw_infer_estimator() override;
+ virtual bool has_caches() const override;
+ virtual Expected<uint32_t> get_cache_read_size() const override;
+ virtual Expected<uint32_t> get_cache_write_size() const override;
+ virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override;
+ virtual Expected<hailo_cache_info_t> get_cache_info() const override;
+ virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override;
virtual ~HcpConfigCoreOp() = default;
HcpConfigCoreOp(const HcpConfigCoreOp &other) = delete;
{
assert(nullptr != hef);
- auto all_streams_infos = hef->get_all_stream_infos(network_group_name);
- CHECK_EXPECTED(all_streams_infos);
+ TRY(auto all_streams_infos, hef->get_all_stream_infos(network_group_name));
// We expect to have two or more streams (atleast one for input and one for output)
- if (all_streams_infos->size() < 2) {
+ if (all_streams_infos.size() < 2) {
return make_unexpected(HAILO_INVALID_HEF);
}
Expected<std::map<uint16_t, uint32_t>> NetworkUdpRateCalculator::get_udp_ports_rates_dict(
std::vector<std::reference_wrapper<InputStream>> &udp_input_streams, uint32_t fps, uint32_t max_supported_bandwidth)
{
- auto rates_per_name = calculate_inputs_bandwith(fps, max_supported_bandwidth);
- CHECK_EXPECTED(rates_per_name);
+ TRY(const auto rates_per_name, calculate_inputs_bandwith(fps, max_supported_bandwidth));
std::map<uint16_t, uint32_t> results = {};
for (const auto &input_stream : udp_input_streams) {
"Invalid stream index {}", stream_index);
const uint16_t remote_port = static_cast<uint16_t>(stream_index + HailoRTCommon::ETH_INPUT_BASE_PORT);
results.insert(std::make_pair(remote_port,
- rates_per_name->at(input_stream.get().name())));
+ rates_per_name.at(input_stream.get().name())));
}
return results;
hailo_status NetworkUdpRateCalculator::set_rate_limit(const std::string &ip, uint16_t port, uint32_t rate_bytes_per_sec)
{
#if defined(__GNUC__)
- auto tc = TrafficControlUtil::create(ip, port, rate_bytes_per_sec);
- CHECK_EXPECTED_AS_STATUS(tc);
- CHECK_SUCCESS(tc->set_rate_limit());
+ TRY(auto tc, TrafficControlUtil::create(ip, port, rate_bytes_per_sec));
+ CHECK_SUCCESS(tc.set_rate_limit());
return HAILO_SUCCESS;
#else
hailo_status NetworkUdpRateCalculator::reset_rate_limit(const std::string &ip, uint16_t port)
{
#if defined(__GNUC__)
- auto tc = TrafficControlUtil::create(ip, port, 0);
- CHECK_EXPECTED_AS_STATUS(tc);
- CHECK_SUCCESS(tc->reset_rate_limit());
+ TRY(auto tc, TrafficControlUtil::create(ip, port, 0));
+ CHECK_SUCCESS(tc.reset_rate_limit());
return HAILO_SUCCESS;
#else
uint16_t host_port)
{
auto status = HAILO_UNINITIALIZED;
- auto socket = Socket::create(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
- CHECK_EXPECTED(socket);
- auto object = Udp(device_ip, device_port, host_ip, host_port, socket.release(), status);
+ TRY(auto socket, Socket::create(AF_INET, SOCK_DGRAM, IPPROTO_UDP));
+ auto object = Udp(device_ip, device_port, host_ip, host_port, std::move(socket), status);
CHECK_SUCCESS_AS_EXPECTED(status);
return object;
// TODO: hailo_device_dma_map_buffer/hailo_device_dma_unmap_buffer aren't thread safe when crossed with
// hailo_allocate_buffer/hailo_free_buffer (HRT-10669)
-hailo_status hailo_device_dma_map_buffer(hailo_device device,void *address, size_t size, hailo_dma_buffer_direction_t direction)
+hailo_status hailo_device_dma_map_buffer(hailo_device device, void *address, size_t size, hailo_dma_buffer_direction_t direction)
{
CHECK_ARG_NOT_NULL(device);
CHECK_ARG_NOT_NULL(address);
return reinterpret_cast<Device*>(device)->dma_unmap(address, size, direction);
}
-hailo_status hailo_vdevice_dma_map_buffer(hailo_vdevice vdevice,void *address, size_t size, hailo_dma_buffer_direction_t direction)
+hailo_status hailo_vdevice_dma_map_buffer(hailo_vdevice vdevice, void *address, size_t size, hailo_dma_buffer_direction_t direction)
{
CHECK_ARG_NOT_NULL(vdevice);
CHECK_ARG_NOT_NULL(address);
return reinterpret_cast<VDevice*>(vdevice)->dma_unmap(address, size, direction);
}
+hailo_status hailo_device_dma_map_dmabuf(hailo_device device, int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ CHECK_ARG_NOT_NULL(device);
+ return reinterpret_cast<Device*>(device)->dma_map_dmabuf(dmabuf_fd, size, direction);
+}
+
+hailo_status hailo_device_dma_unmap_dmabuf(hailo_device device, int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ CHECK_ARG_NOT_NULL(device);
+ return reinterpret_cast<Device*>(device)->dma_unmap_dmabuf(dmabuf_fd, size, direction);
+}
+
+hailo_status hailo_vdevice_dma_map_dmabuf(hailo_vdevice vdevice, int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ CHECK_ARG_NOT_NULL(vdevice);
+ return reinterpret_cast<VDevice*>(vdevice)->dma_map_dmabuf(dmabuf_fd, size, direction);
+}
+
+hailo_status hailo_vdevice_dma_unmap_dmabuf(hailo_vdevice vdevice, int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ CHECK_ARG_NOT_NULL(vdevice);
+ return reinterpret_cast<VDevice*>(vdevice)->dma_unmap_dmabuf(dmabuf_fd, size, direction);
+}
+
hailo_status hailo_calculate_eth_input_rate_limits(hailo_hef hef, const char *network_group_name, uint32_t fps,
hailo_rate_limit_t *rates, size_t *rates_length)
{
MemoryView buffer_view(buffer, size);
auto status = (reinterpret_cast<OutputStream*>(stream))->read(buffer_view);
+ if (HAILO_STREAM_ABORT == status) {
+ return status;
+ }
CHECK_SUCCESS(status);
return HAILO_SUCCESS;
}
CHECK_ARG_NOT_NULL(buffer);
auto status = (reinterpret_cast<InputStream*>(stream))->write(MemoryView::create_const(buffer, size));
+ if (HAILO_STREAM_ABORT == status) {
+ return status;
+ }
CHECK_SUCCESS(status);
return HAILO_SUCCESS;
}
CHECK_AS_EXPECTED(m_action_list_type < CONTEXT_SWITCH_DEFS__ACTION_TYPE_COUNT, HAILO_INTERNAL_FAILURE,
"Action cannot be serialized");
- auto header = serialize_header();
- CHECK_EXPECTED(header);
+ TRY(auto header, serialize_header());
+ TRY(auto params, serialize_params(context_resources));
+ TRY(auto serialized_action, Buffer::create(header.size() + params.size()));
- auto params = serialize_params(context_resources);
- CHECK_EXPECTED(params);
-
- auto serialized_action = Buffer::create(header->size() + params->size());
- CHECK_EXPECTED(serialized_action);
-
- std::copy(header->begin(), header->end(), serialized_action->data());
- std::copy(params->begin(), params->end(), serialized_action->data() + header->size());
+ std::copy(header.begin(), header.end(), serialized_action.data());
+ std::copy(params.begin(), params.end(), serialized_action.data() + header.size());
std::vector<Buffer> buffers;
- buffers.emplace_back(serialized_action.release());
+ buffers.emplace_back(std::move(serialized_action));
return buffers;
}
CHECK_SUCCESS(status);
if (should_support_pre_fetch && is_last_write) {
- auto desc_count = config_buffer.program_descriptors();
- CHECK_EXPECTED_AS_STATUS(desc_count);
+ TRY(const auto desc_count, config_buffer.program_descriptors());
+ (void)desc_count;
}
return HAILO_SUCCESS;
}
-Expected<ContextSwitchConfigActionPtr> WriteDataCcwAction::create(uint32_t offset, size_t size, uint8_t config_stream_index,
- size_t total_ccw_burst, std::shared_ptr<ShefFileHandle> shef_file_handle)
+Expected<ContextSwitchConfigActionPtr> WriteDataCcwAction::create(std::vector<ccw_write_ptr_t> &&ccw_write_ptrs, uint8_t config_stream_index,
+ uint16_t total_ccw_burst, std::shared_ptr<SeekableBytesReader> hef_reader)
{
- CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(total_ccw_burst), HAILO_INVALID_HEF,
- "Too many ccw burst {} (must fit in uint16)", total_ccw_burst);
auto result = ContextSwitchConfigActionPtr(new (std::nothrow) WriteDataCcwAction(
- offset, size, config_stream_index, static_cast<uint16_t>(total_ccw_burst), shef_file_handle));
+ std::move(ccw_write_ptrs), config_stream_index, total_ccw_burst, hef_reader));
CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY);
return result;
}
WriteDataCcwActionByBuffer::WriteDataCcwActionByBuffer(Buffer &&data, uint8_t config_stream_index, uint16_t total_ccw_burst) :
- WriteDataCcwAction(0, 0, config_stream_index, total_ccw_burst, nullptr),
+ WriteDataCcwAction({}, config_stream_index, total_ccw_burst, nullptr),
m_data(std::move(data))
{}
-WriteDataCcwAction::WriteDataCcwAction(uint32_t offset, size_t size, uint8_t config_stream_index, uint16_t total_ccw_burst,
- std::shared_ptr<ShefFileHandle> shef_file_handle) :
+WriteDataCcwAction::WriteDataCcwAction(std::vector<ccw_write_ptr_t> &&ccw_write_ptrs, uint8_t config_stream_index,
+ uint16_t total_ccw_burst, std::shared_ptr<SeekableBytesReader> hef_reader) :
ContextSwitchConfigAction(Type::WriteDataCcw),
- m_offset(offset),
- m_size(size),
+ m_ccw_write_ptrs(std::move(ccw_write_ptrs)),
m_config_stream_index(config_stream_index),
m_total_ccw_burst(total_ccw_burst),
- m_shef_file_handle(shef_file_handle)
+ m_hef_reader(hef_reader)
{}
Expected<std::vector<Buffer>> WriteDataCcwAction::serialize(const ContextResources &) const
hailo_status WriteDataCcwAction::write_to_config_buffer(ConfigBuffer& config_buffer, bool should_support_pre_fetch)
{
- bool is_last_write = config_buffer.size_left() == size();
+ uint64_t total_ccw_size = 0;
+ for (const auto &ccw_write_ptr : m_ccw_write_ptrs) {
+ total_ccw_size += ccw_write_ptr.size;
+ }
+
+ bool is_last_write = config_buffer.size_left() == total_ccw_size;
+ if (should_support_pre_fetch && is_last_write) {
+ auto status = config_buffer.pad_with_nops();
+ CHECK_SUCCESS(status);
+ }
+
+ auto status = m_hef_reader->open();
+ CHECK_SUCCESS(status);
- auto buffer = m_shef_file_handle->read(m_offset, m_size);
- CHECK_EXPECTED_AS_STATUS(buffer);
+ for (const auto &ccw_write_ptr : m_ccw_write_ptrs) {
+ TRY(auto buffer, Buffer::create_shared(ccw_write_ptr.size));
+ MemoryView mem_view(buffer->data(), buffer->size());
+ assert(ccw_write_ptr.offset <= SIZE_MAX);
+ status = m_hef_reader->read_from_offset(static_cast<size_t>(ccw_write_ptr.offset), mem_view, ccw_write_ptr.size);
+ CHECK_SUCCESS(status);
+ status = config_buffer.write(mem_view);
+ CHECK_SUCCESS(status);
+ }
- auto status = config_buffer.write(MemoryView(buffer.value()));
+ status = m_hef_reader->close();
CHECK_SUCCESS(status);
if (should_support_pre_fetch && is_last_write) {
- auto desc_count = config_buffer.program_descriptors();
- CHECK_EXPECTED_AS_STATUS(desc_count);
+ TRY(const auto desc_count, config_buffer.program_descriptors());
+ (void)desc_count;
}
return HAILO_SUCCESS;
return Buffer::create(0);
}
+Expected<ContextSwitchConfigActionPtr> WaitForCacheUpdatedAction::create()
+{
+ auto result = ContextSwitchConfigActionPtr(new (std::nothrow) WaitForCacheUpdatedAction());
+ CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY);
+ return result;
+}
+
+WaitForCacheUpdatedAction::WaitForCacheUpdatedAction() :
+ ContextSwitchConfigAction(Type::WaitForCacheUpdated, CONTEXT_SWITCH_DEFS__ACTION_TYPE_WAIT_FOR_CACHE_UPDATED)
+{}
+
+bool WaitForCacheUpdatedAction::supports_repeated_block() const
+{
+ // We don't support repeated blocks for this action, since only one is added per group of consecutive
+ // TriggerNewDataFromDataInput actions.
+ return false;
+}
+
+Expected<Buffer> WaitForCacheUpdatedAction::serialize_params(const ContextResources &) const
+{
+ return Buffer::create(0);
+}
+
Expected<ContextSwitchConfigActionPtr> WaitForNetworkGroupChangeAction::create()
{
auto result = ContextSwitchConfigActionPtr(new (std::nothrow) WaitForNetworkGroupChangeAction());
auto repeated_header = ContextSwitchConfigAction::serialize(context_resources);
CHECK_EXPECTED(repeated_header);
- CHECK_AS_EXPECTED(repeated_header->size() == 1, HAILO_INTERNAL_FAILURE, "Repeated action header should contain one buffer");
+ CHECK_AS_EXPECTED(repeated_header->size() == 1, HAILO_INTERNAL_FAILURE,
+ "Repeated action header should contain one buffer");
buffers.emplace_back(std::move(repeated_header->at(0)));
for (const auto &action : m_actions) {
assert(action->get_action_list_type() == m_sub_action_type);
- auto action_buffer = action->serialize_params(context_resources);
- CHECK_EXPECTED(action_buffer);
- buffers.emplace_back(action_buffer.release());
+ TRY(auto action_buffer, action->serialize_params(context_resources));
+ buffers.emplace_back(std::move(action_buffer));
}
return buffers;
Expected<Buffer> AllowInputDataflowAction::serialize_params(const ContextResources &context_resources) const
{
// H2D direction because it is Input actions
- const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_H2D_STREAM);
- CHECK_EXPECTED(edge_layer);
+ TRY(const auto edge_layer,
+ context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_H2D_STREAM));
CONTEXT_SWITCH_DEFS__fetch_data_action_data_t params{};
- params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id);
+ params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id);
params.stream_index = m_stream_index;
- params.network_index = edge_layer->layer_info.network_index;
- params.host_buffer_type = static_cast<CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t>(edge_layer->buffer_info.buffer_type);
+ params.network_index = edge_layer.layer_info.network_index;
+ params.host_buffer_type = static_cast<CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t>(edge_layer.buffer_info.buffer_type);
- switch (edge_layer->layer_info.type) {
+ switch (edge_layer.layer_info.type) {
case LayerType::BOUNDARY:
params.credit_type = CONTEXT_SWITCH_DEFS__CREDIT_IN_BYTES;
- params.frame_periph_size = edge_layer->layer_info.nn_stream_config.periph_bytes_per_buffer *
- edge_layer->layer_info.nn_stream_config.periph_buffers_per_frame;
+ params.frame_periph_size = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer *
+ edge_layer.layer_info.nn_stream_config.periph_buffers_per_frame;
break;
case LayerType::INTER_CONTEXT:
+ case LayerType::CACHE:
params.credit_type = CONTEXT_SWITCH_DEFS__CREDIT_IN_DESCRIPTORS;
- params.frame_periph_size = ((edge_layer->buffer_info.bytes_in_pattern - 1) / (edge_layer->buffer_info.desc_page_size)) + 1;
+ params.frame_periph_size = ((edge_layer.buffer_info.bytes_in_pattern - 1) / (edge_layer.buffer_info.desc_page_size)) + 1;
break;
default:
- LOGGER__ERROR("Invalid layer type {} for stream {}", static_cast<int>(edge_layer->layer_info.type), m_stream_index);
+ LOGGER__ERROR("Invalid layer type {} for stream {}", static_cast<int>(edge_layer.layer_info.type), m_stream_index);
return make_unexpected(HAILO_INTERNAL_FAILURE);
}
Expected<Buffer> WaitOutputTransferDoneAction::serialize_params(const ContextResources &context_resources) const
{
// D2H direction because it is output action
- const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_D2H_STREAM);
- CHECK_EXPECTED(edge_layer);
+ TRY(const auto edge_layer,
+ context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_D2H_STREAM));
CONTEXT_SWITCH_DEFS__vdma_dataflow_interrupt_data_t params{};
- params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id);
+ params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id);
params.stream_index = m_stream_index;
- params.network_index = edge_layer->layer_info.network_index;
- params.is_inter_context = static_cast<uint8_t>(LayerType::INTER_CONTEXT == edge_layer->layer_info.type);
- params.host_buffer_type = static_cast<CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t>(edge_layer->buffer_info.buffer_type);
+ params.network_index = edge_layer.layer_info.network_index;
+ params.is_inter_context = static_cast<uint8_t>(LayerType::INTER_CONTEXT == edge_layer.layer_info.type);
+ params.host_buffer_type = static_cast<CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t>(edge_layer.buffer_info.buffer_type);
return Buffer::create(reinterpret_cast<uint8_t*>(¶ms), sizeof(params));
}
CONTEXT_SWITCH_DEFS__open_boundary_input_channel_data_t params{};
// H2D direction because it is Input actions
- const auto edge_layer = context_resources.get_edge_layer_by_channel_id(m_channel_id);
- CHECK_EXPECTED(edge_layer);
+ TRY(const auto edge_layer, context_resources.get_edge_layer_by_channel_id(m_channel_id));
- params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id);
+ params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id);
params.host_buffer_info = m_host_buffer_info;
- params.stream_index = edge_layer->layer_info.stream_index;
- params.network_index = edge_layer->layer_info.network_index;
- params.periph_bytes_per_buffer = edge_layer->layer_info.nn_stream_config.periph_bytes_per_buffer;
- params.frame_periph_size = edge_layer->layer_info.nn_stream_config.periph_bytes_per_buffer *
- edge_layer->layer_info.nn_stream_config.periph_buffers_per_frame;
+ params.stream_index = edge_layer.layer_info.stream_index;
+ params.network_index = edge_layer.layer_info.network_index;
+ params.periph_bytes_per_buffer = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer;
+ params.frame_periph_size = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer *
+ edge_layer.layer_info.nn_stream_config.periph_buffers_per_frame;
return Buffer::create(reinterpret_cast<uint8_t*>(¶ms), sizeof(params));
}
return Buffer::create(reinterpret_cast<uint8_t*>(¶ms), sizeof(params));
}
+Expected<ContextSwitchConfigActionPtr> ActivateCacheInputChannelAction::create(const vdma::ChannelId &channel_id,
+ uint8_t stream_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config,
+ const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info, uint32_t initial_credit_size)
+{
+ auto result = ContextSwitchConfigActionPtr(new (std::nothrow) ActivateCacheInputChannelAction(channel_id,
+ stream_index, nn_stream_config, host_buffer_info, initial_credit_size));
+ CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY);
+ return result;
+}
+
+ActivateCacheInputChannelAction::ActivateCacheInputChannelAction(const vdma::ChannelId &channel_id,
+ uint8_t stream_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config,
+ const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info, uint32_t initial_credit_size) :
+ ContextSwitchConfigAction(ContextSwitchConfigAction::Type::ActivateCacheInputChannel,
+ CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_INPUT),
+ m_channel_id(channel_id),
+ m_stream_index(stream_index),
+ m_nn_stream_config(nn_stream_config),
+ m_host_buffer_info(host_buffer_info),
+ m_initial_credit_size(initial_credit_size)
+{}
+
+bool ActivateCacheInputChannelAction::supports_repeated_block() const
+{
+ // Activate actions shouldn't be repeated (for easier debugging).
+ return false;
+}
+
+Expected<Buffer> ActivateCacheInputChannelAction::serialize_params(const ContextResources &) const
+{
+ CONTEXT_SWITCH_DEFS__activate_cache_input_data_t params{};
+ params.packed_vdma_channel_id = pack_vdma_channel_id(m_channel_id);
+ params.stream_index = m_stream_index;
+ params.stream_reg_info = parse_nn_config(m_nn_stream_config);
+ params.host_buffer_info = m_host_buffer_info;
+ params.initial_credit_size = m_initial_credit_size;
+ return Buffer::create(reinterpret_cast<uint8_t*>(¶ms), sizeof(params));
+}
+
+Expected<ContextSwitchConfigActionPtr> ActivateCacheOutputChannelAction::create(const vdma::ChannelId &channel_id,
+ uint8_t stream_index, uint8_t network_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config,
+ const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info)
+{
+ auto result = ContextSwitchConfigActionPtr(new (std::nothrow) ActivateCacheOutputChannelAction(channel_id,
+ stream_index, network_index, nn_stream_config, host_buffer_info));
+ CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY);
+ return result;
+}
+
+ActivateCacheOutputChannelAction::ActivateCacheOutputChannelAction(const vdma::ChannelId &channel_id,
+ uint8_t stream_index, uint8_t network_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config,
+ const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info) :
+ ContextSwitchConfigAction(ContextSwitchConfigAction::Type::ActivateCacheOutputChannel,
+ CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_OUTPUT),
+ m_channel_id(channel_id),
+ m_stream_index(stream_index),
+ m_network_index(network_index),
+ m_nn_stream_config(nn_stream_config),
+ m_host_buffer_info(host_buffer_info)
+{}
+
+bool ActivateCacheOutputChannelAction::supports_repeated_block() const
+{
+ // Activate actions shouldn't be repeated (for easier debugging).
+ return false;
+}
+
+Expected<Buffer> ActivateCacheOutputChannelAction::serialize_params(const ContextResources &) const
+{
+ CONTEXT_SWITCH_DEFS__activate_cache_output_data_t params{};
+ params.packed_vdma_channel_id = pack_vdma_channel_id(m_channel_id);
+ params.stream_index = m_stream_index;
+ params.network_index = m_network_index;
+ params.stream_reg_info = parse_nn_config(m_nn_stream_config);
+ params.host_buffer_info = m_host_buffer_info;
+ return Buffer::create(reinterpret_cast<uint8_t*>(¶ms), sizeof(params));
+}
+
Expected<ContextSwitchConfigActionPtr> ValidateChannelAction::create(const EdgeLayer &edge_layer,
const bool is_batch_switch_context)
{
Expected<Buffer> WaitDmaIdleAction::serialize_params(const ContextResources &context_resources) const
{
// D2H direction because it is output action
- const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_D2H_STREAM);
- CHECK_EXPECTED(edge_layer);
+ TRY(const auto edge_layer,
+ context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_D2H_STREAM));
CONTEXT_SWITCH_DEFS__wait_dma_idle_data_t params{};
- params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id);
- params.is_inter_context = static_cast<uint8_t>(LayerType::INTER_CONTEXT == edge_layer->layer_info.type);
+ params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id);
+ params.is_inter_context = static_cast<uint8_t>(LayerType::INTER_CONTEXT == edge_layer.layer_info.type);
params.stream_index = m_stream_index;
- params.host_buffer_type = static_cast<CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t>(edge_layer->buffer_info.buffer_type);
+ params.host_buffer_type = static_cast<CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t>(edge_layer.buffer_info.buffer_type);
return Buffer::create(reinterpret_cast<uint8_t*>(¶ms), sizeof(params));
}
#include "hailo/expected.hpp"
#include "hailo/buffer.hpp"
+#include "common/file_utils.hpp"
#include "vdma/channel/channel_id.hpp"
#include "hef/layer_info.hpp"
#include "context_switch_defs.h"
#include "core_op/resource_manager/config_buffer.hpp"
-
namespace hailort
{
class ContextResources;
struct EdgeLayer;
+#pragma pack(push, 1)
+typedef struct {
+ uint64_t offset;
+ uint32_t size;
+} ccw_write_ptr_t;
+#pragma pack(pop)
class ContextSwitchConfigAction;
using ContextSwitchConfigActionPtr = std::shared_ptr<ContextSwitchConfigAction>;
+
class ContextSwitchConfigAction
{
public:
ActivateInterContextOutputChannel,
ActivateDdrInputChannel,
ActivateDdrOutputChannel,
+ ActivateCacheInputChannel,
+ ActivateCacheOutputChannel,
ValidateChannel,
DeactivateChannel,
WaitDmaIdle,
ChangeBoundaryInputBatchAction,
PauseVdmaChannel,
ResumeVdmaChannel,
+ WaitForCacheUpdated,
};
ContextSwitchConfigAction(ContextSwitchConfigAction &&) = default;
const vdma::ChannelId m_channel_id;
};
-class ShefFileHandle;
class WriteDataCcwAction : public ContextSwitchConfigAction
{
public:
- static Expected<ContextSwitchConfigActionPtr> create(uint32_t offset, size_t size, uint8_t config_stream_index,
- size_t total_ccw_burst, std::shared_ptr<ShefFileHandle> shef_file_handle);
+ static Expected<ContextSwitchConfigActionPtr> create(std::vector<ccw_write_ptr_t> &&ccw_write_ptrs, uint8_t config_stream_index,
+ uint16_t total_ccw_burst, std::shared_ptr<SeekableBytesReader> hef_reader);
WriteDataCcwAction(WriteDataCcwAction &&) = default;
WriteDataCcwAction(const WriteDataCcwAction &) = delete;
WriteDataCcwAction &operator=(WriteDataCcwAction &&) = delete;
virtual bool supports_repeated_block() const override;
virtual Expected<Buffer> serialize_params(const ContextResources &context_resources) const override;
- uint8_t config_stream_index() const { return m_config_stream_index; }
- uint16_t total_ccw_burst() const { return m_total_ccw_burst; }
virtual size_t size() const { return m_size; }
+ virtual uint8_t config_stream_index() const { return m_config_stream_index; }
virtual hailo_status write_to_config_buffer(ConfigBuffer& config_buffer, bool should_support_pre_fetch);
+ uint16_t total_ccw_burst() const { return m_total_ccw_burst; }
protected:
- WriteDataCcwAction(uint32_t offset, size_t size, uint8_t config_stream_index,
- uint16_t total_ccw_burst, std::shared_ptr<ShefFileHandle> shef_file_handle);
+ WriteDataCcwAction(std::vector<ccw_write_ptr_t> &&ccw_write_ptrs, uint8_t config_stream_index,
+ uint16_t total_ccw_burst, std::shared_ptr<SeekableBytesReader> hef_reader);
- uint32_t m_offset;
+ const std::vector<ccw_write_ptr_t> m_ccw_write_ptrs;
size_t m_size;
const uint8_t m_config_stream_index;
- const uint16_t m_total_ccw_burst;
- std::shared_ptr<ShefFileHandle> m_shef_file_handle;
+ uint16_t m_total_ccw_burst;
+ std::shared_ptr<SeekableBytesReader> m_hef_reader;
};
class WriteDataCcwActionByBuffer : public WriteDataCcwAction
ResetBurstCreditsTaskAction();
};
+class WaitForCacheUpdatedAction : public ContextSwitchConfigAction
+{
+public:
+ static Expected<ContextSwitchConfigActionPtr> create();
+
+ WaitForCacheUpdatedAction(WaitForCacheUpdatedAction &&) = default;
+ WaitForCacheUpdatedAction(const WaitForCacheUpdatedAction &) = delete;
+ WaitForCacheUpdatedAction &operator=(WaitForCacheUpdatedAction &&) = delete;
+ WaitForCacheUpdatedAction &operator=(const WaitForCacheUpdatedAction &) = delete;
+ virtual ~WaitForCacheUpdatedAction() = default;
+ virtual bool supports_repeated_block() const override;
+ virtual Expected<Buffer> serialize_params(const ContextResources &context_resources) const override;
+
+private:
+ WaitForCacheUpdatedAction();
+};
+
class WaitForNetworkGroupChangeAction : public ContextSwitchConfigAction
{
public:
const uint32_t m_buffered_rows_count;
};
+class ActivateCacheInputChannelAction : public ContextSwitchConfigAction
+{
+public:
+ static Expected<ContextSwitchConfigActionPtr> create(const vdma::ChannelId &channel_id,
+ uint8_t stream_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config,
+ const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info, uint32_t initial_credit_size);
+
+ virtual bool supports_repeated_block() const override;
+ virtual Expected<Buffer> serialize_params(const ContextResources &context_resources) const override;
+
+private:
+ ActivateCacheInputChannelAction(const vdma::ChannelId &channel_id,
+ uint8_t stream_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config,
+ const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info,
+ uint32_t initial_credit_size);
+
+ const vdma::ChannelId m_channel_id;
+ const uint8_t m_stream_index;
+ const CONTROL_PROTOCOL__nn_stream_config_t m_nn_stream_config;
+ const CONTROL_PROTOCOL__host_buffer_info_t m_host_buffer_info;
+ const uint32_t m_initial_credit_size;
+};
+
+class ActivateCacheOutputChannelAction : public ContextSwitchConfigAction
+{
+public:
+ static Expected<ContextSwitchConfigActionPtr> create(const vdma::ChannelId &channel_id, uint8_t stream_index,
+ uint8_t network_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config,
+ const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info);
+
+ virtual bool supports_repeated_block() const override;
+ virtual Expected<Buffer> serialize_params(const ContextResources &context_resources) const override;
+
+private:
+ ActivateCacheOutputChannelAction(const vdma::ChannelId &channel_id, uint8_t stream_index,
+ uint8_t network_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config,
+ const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info);
+
+ const vdma::ChannelId m_channel_id;
+ const uint8_t m_stream_index;
+ const uint8_t m_network_index;
+ const CONTROL_PROTOCOL__nn_stream_config_t m_nn_stream_config;
+ const CONTROL_PROTOCOL__host_buffer_info_t m_host_buffer_info;
+};
+
class ValidateChannelAction : public ContextSwitchConfigAction
{
public:
}
}
+void ContextMetadata::add_cache_layer(const LayerInfo &layer_info)
+{
+ if (HAILO_H2D_STREAM == layer_info.direction) {
+ m_cache_input_layers.push_back(layer_info);
+ } else {
+ m_cache_output_layers.push_back(layer_info);
+ }
+}
+
const std::vector<LayerInfo> &ContextMetadata::get_boundary_input_layers() const
{
return m_boundary_input_layers;
return m_ddr_output_layers;
}
+const std::vector<LayerInfo> &ContextMetadata::get_cache_input_layers() const
+{
+ return m_cache_input_layers;
+}
+
+const std::vector<LayerInfo> &ContextMetadata::get_cache_output_layers() const
+{
+ return m_cache_output_layers;
+}
+
Expected<size_t> ContextMetadata::get_layers_transfer_size(const std::vector<LayerInfo> &layer_infos) const
{
size_t total_transfer_size = 0;
for (const auto &layer_info : layer_infos) {
- auto transfer_size = LayerInfoUtils::get_transfer_size(layer_info);
- CHECK_EXPECTED(transfer_size);
- total_transfer_size += transfer_size.release();
+ TRY(const auto transfer_size, LayerInfoUtils::get_transfer_size(layer_info));
+ total_transfer_size += transfer_size;
}
return total_transfer_size;
}
// Calc config buffers
for (const auto &config_buffer_sizes : m_config_buffers_info) {
- total_transfer_size += std::accumulate(config_buffer_sizes.second.begin(), config_buffer_sizes.second.end(), 0);
+ total_transfer_size += std::accumulate(config_buffer_sizes.second.bursts_sizes.begin(),
+ config_buffer_sizes.second.bursts_sizes.end(), 0);
}
// Calc all edge layers
- auto boundary_input_transfer_size = get_layers_transfer_size(m_boundary_input_layers);
- CHECK_EXPECTED(boundary_input_transfer_size);
- auto boundary_output_transfer_size = get_layers_transfer_size(m_boundary_output_layers);
- CHECK_EXPECTED(boundary_output_transfer_size);
- auto ddr_input_transfer_size = get_layers_transfer_size(m_ddr_input_layers);
- CHECK_EXPECTED(ddr_input_transfer_size);
- auto ddr_output_transfer_size = get_layers_transfer_size(m_ddr_output_layers);
- CHECK_EXPECTED(ddr_output_transfer_size);
- auto inter_context_input_transfer_size = get_layers_transfer_size(m_inter_context_input_layers);
- CHECK_EXPECTED(inter_context_input_transfer_size);
- auto inter_context_output_transfer_size = get_layers_transfer_size(m_inter_context_output_layers);
- CHECK_EXPECTED(inter_context_output_transfer_size);
+ TRY(const auto boundary_input_transfer_size, get_layers_transfer_size(m_boundary_input_layers));
+ TRY(const auto boundary_output_transfer_size, get_layers_transfer_size(m_boundary_output_layers));
+ TRY(const auto ddr_input_transfer_size, get_layers_transfer_size(m_ddr_input_layers));
+ TRY(const auto ddr_output_transfer_size, get_layers_transfer_size(m_ddr_output_layers));
+ TRY(const auto inter_context_input_transfer_size, get_layers_transfer_size(m_inter_context_input_layers));
+ TRY(const auto inter_context_output_transfer_size, get_layers_transfer_size(m_inter_context_output_layers));
total_transfer_size +=
- boundary_input_transfer_size.release() + boundary_output_transfer_size.release() +
- ddr_input_transfer_size.release() + ddr_output_transfer_size.release() +
- inter_context_input_transfer_size.release() + inter_context_output_transfer_size.release();
+ boundary_input_transfer_size + boundary_output_transfer_size +
+ ddr_input_transfer_size + ddr_output_transfer_size +
+ inter_context_input_transfer_size + inter_context_output_transfer_size;
return total_transfer_size;
}
Expected<std::vector<LayerInfo>> CoreOpMetadata::get_all_layer_infos(const std::string &network_name) const
{
- auto input_layer_infos = get_input_layer_infos(network_name);
- CHECK_EXPECTED(input_layer_infos);
-
- auto output_layer_infos = get_output_layer_infos(network_name);
- CHECK_EXPECTED(output_layer_infos);
+ TRY(const auto input_layer_infos, get_input_layer_infos(network_name));
+ TRY(const auto output_layer_infos, get_output_layer_infos(network_name));
std::vector<LayerInfo> res;
- res.reserve(input_layer_infos->size() + output_layer_infos->size());
- res.insert(res.end(), input_layer_infos->begin(), input_layer_infos->end());
- res.insert(res.end(), output_layer_infos->begin(), output_layer_infos->end());
+ res.reserve(input_layer_infos.size() + output_layer_infos.size());
+ res.insert(res.end(), input_layer_infos.begin(), input_layer_infos.end());
+ res.insert(res.end(), output_layer_infos.begin(), output_layer_infos.end());
return res;
}
+size_t CoreOpMetadata::get_cache_layers_count() const
+{
+ size_t cache_layers_count = 0;
+ for (const auto &context : m_dynamic_contexts) {
+ cache_layers_count += context.get_cache_input_layers().size() + context.get_cache_output_layers().size();
+ }
+ return cache_layers_count;
+}
+
Expected<std::vector<hailo_stream_info_t>> CoreOpMetadata::get_input_stream_infos(const std::string &network_name) const
{
std::vector<hailo_stream_info_t> res;
- auto input_layers = get_input_layer_infos(network_name);
- CHECK_EXPECTED(input_layers);
- for (auto &layer_info : input_layers.value()) {
+ TRY(const auto input_layers, get_input_layer_infos(network_name));
+ for (auto &layer_info : input_layers) {
const auto &stream_infos = LayerInfoUtils::get_stream_infos_from_layer_info(layer_info);
res.insert(res.end(), stream_infos.begin(), stream_infos.end());
}
Expected<std::vector<hailo_stream_info_t>> CoreOpMetadata::get_output_stream_infos(const std::string &network_name) const
{
std::vector<hailo_stream_info_t> res;
- auto output_layers = get_output_layer_infos(network_name);
- CHECK_EXPECTED(output_layers);
- for (auto &layer_info : output_layers.value()) {
+ TRY(const auto output_layers, get_output_layer_infos(network_name));
+ for (auto &layer_info : output_layers) {
const auto &stream_infos = LayerInfoUtils::get_stream_infos_from_layer_info(layer_info);
res.insert(res.end(), stream_infos.begin(), stream_infos.end());
}
Expected<std::vector<hailo_stream_info_t>> CoreOpMetadata::get_all_stream_infos(const std::string &network_name) const
{
- auto input_stream_infos = get_input_stream_infos(network_name);
- CHECK_EXPECTED(input_stream_infos);
-
- auto output_stream_infos = get_output_stream_infos(network_name);
- CHECK_EXPECTED(output_stream_infos);
+ TRY(const auto input_stream_infos, get_input_stream_infos(network_name));
+ TRY(const auto output_stream_infos, get_output_stream_infos(network_name));
std::vector<hailo_stream_info_t> res;
- res.reserve(input_stream_infos->size() + output_stream_infos->size());
- res.insert(res.end(), input_stream_infos->begin(), input_stream_infos->end());
- res.insert(res.end(), output_stream_infos->begin(), output_stream_infos->end());
+ res.reserve(input_stream_infos.size() + output_stream_infos.size());
+ res.insert(res.end(), input_stream_infos.begin(), input_stream_infos.end());
+ res.insert(res.end(), output_stream_infos.begin(), output_stream_infos.end());
return res;
}
{
size_t total_transfer_size = 0;
for (const auto &dynamic_context : m_dynamic_contexts) {
- auto context_size = dynamic_context.get_context_transfer_size();
- CHECK_EXPECTED(context_size);
- total_transfer_size += context_size.release();
+ TRY(const auto context_size, dynamic_context.get_context_transfer_size());
+ total_transfer_size += context_size;
}
return total_transfer_size;
}
so should be same across all clusters layouts */
{
CHECK_AS_EXPECTED(1 == m_core_ops_metadata_per_arch.size(), HAILO_INTERNAL_FAILURE);
- auto core_op_metadata_exp = m_core_ops_metadata_per_arch.begin()->second.get_metadata(PARTIAL_CLUSTERS_LAYOUT_IGNORE);
- CHECK_EXPECTED(core_op_metadata_exp);
+ TRY(auto core_op_metadata, m_core_ops_metadata_per_arch.begin()->second.get_metadata(PARTIAL_CLUSTERS_LAYOUT_IGNORE));
- auto core_op_metadata = core_op_metadata_exp.release();
return core_op_metadata;
}
Expected<std::vector<LayerInfo>> NetworkGroupMetadata::get_all_layer_infos() const
{
- auto core_op_metadata = get_core_op_metadata();
- CHECK_EXPECTED(core_op_metadata);
+ TRY(const auto core_op_metadata, get_core_op_metadata());
- return core_op_metadata.value()->get_all_layer_infos();
+ return core_op_metadata->get_all_layer_infos();
}
Expected<std::vector<LayerInfo>> NetworkGroupMetadata::get_input_layer_infos(const std::string &network_name) const
{
- auto core_op_metadata = get_core_op_metadata();
- CHECK_EXPECTED(core_op_metadata);
+ TRY(const auto core_op_metadata, get_core_op_metadata());
- return core_op_metadata.value()->get_input_layer_infos(network_name);
+ return core_op_metadata->get_input_layer_infos(network_name);
}
Expected<std::vector<LayerInfo>> NetworkGroupMetadata::get_output_layer_infos(const std::string &network_name) const
{
- auto core_op_metadata = get_core_op_metadata();
- CHECK_EXPECTED(core_op_metadata);
+ TRY(const auto core_op_metadata, get_core_op_metadata());
- return core_op_metadata.value()->get_output_layer_infos(network_name);
+ return core_op_metadata->get_output_layer_infos(network_name);
}
Expected<std::vector<hailo_vstream_info_t>> NetworkGroupMetadata::get_input_vstream_infos(const std::string &network_name) const
{
- auto input_layer_infos = get_input_layer_infos(network_name);
- CHECK_EXPECTED(input_layer_infos);
+ TRY(const auto input_layer_infos, get_input_layer_infos(network_name));
std::vector<hailo_vstream_info_t> input_vstream_infos;
- for (auto &layer_info : input_layer_infos.value()) {
+ for (auto &layer_info : input_layer_infos) {
auto vstreams_info = LayerInfoUtils::get_vstream_infos_from_layer_info(layer_info);
input_vstream_infos.insert(input_vstream_infos.end(),
std::make_move_iterator(vstreams_info.begin()), std::make_move_iterator(vstreams_info.end()));
Expected<std::vector<hailo_vstream_info_t>> NetworkGroupMetadata::get_output_vstream_infos(const std::string &network_name) const
{
- auto output_layer_infos = get_output_layer_infos(network_name);
- CHECK_EXPECTED(output_layer_infos);
+ TRY(const auto output_layer_infos, get_output_layer_infos(network_name));
std::vector<hailo_vstream_info_t> output_vstream_infos;
- for (auto &layer_info : output_layer_infos.value()) {
+ for (auto &layer_info : output_layer_infos) {
if (std::any_of(m_ops_metadata.begin(), m_ops_metadata.end(),
[&layer_info](auto &op_metadata) { return contains(op_metadata->get_input_names(), layer_info.name); })) {
continue; // all output_vstream_infos that relates to the op are coming from the op itself instead of layer_infos
}
}
for (auto &metadata : m_ops_metadata) {
- auto vstream_info = metadata->get_output_vstream_info();
- CHECK_EXPECTED(vstream_info);
- output_vstream_infos.push_back(vstream_info.release());
+ TRY(const auto vstream_info, metadata->get_output_vstream_info());
+ output_vstream_infos.push_back(std::move(vstream_info));
}
// Sort vstream infos by sorted_output_names
Expected<std::vector<hailo_vstream_info_t>> NetworkGroupMetadata::get_all_vstream_infos(const std::string &network_name) const
{
- auto input_vstream_infos = get_input_vstream_infos(network_name);
- CHECK_EXPECTED(input_vstream_infos);
-
- auto output_vstream_infos = get_output_vstream_infos(network_name);
- CHECK_EXPECTED(output_vstream_infos);
+ TRY(const auto input_vstream_infos, get_input_vstream_infos(network_name));
+ TRY(const auto output_vstream_infos, get_output_vstream_infos(network_name));
std::vector<hailo_vstream_info_t> res;
- res.reserve(input_vstream_infos->size() + output_vstream_infos->size());
- res.insert(res.end(), input_vstream_infos->begin(), input_vstream_infos->end());
- res.insert(res.end(), output_vstream_infos->begin(), output_vstream_infos->end());
+ res.reserve(input_vstream_infos.size() + output_vstream_infos.size());
+ res.insert(res.end(), input_vstream_infos.begin(), input_vstream_infos.end());
+ res.insert(res.end(), output_vstream_infos.begin(), output_vstream_infos.end());
return res;
}
}
}
- auto all_layers_infos = get_all_layer_infos();
- CHECK_EXPECTED(all_layers_infos);
- for (auto &layer_info : all_layers_infos.release()) {
+ TRY(const auto all_layers_infos, get_all_layer_infos());
+ for (const auto &layer_info : all_layers_infos) {
if (layer_info.is_multi_planar) {
for (auto &plane : layer_info.planes) {
if (stream_name == plane.name) {
}
}
- auto all_layers_infos = get_all_layer_infos();
- CHECK_EXPECTED(all_layers_infos);
- for (auto &layer_info : all_layers_infos.value()) {
+ TRY(const auto all_layers_infos, get_all_layer_infos());
+ for (const auto &layer_info : all_layers_infos) {
if (layer_info.is_mux) {
if (is_edge_under_mux(layer_info, vstream_name)) {
// vstream_name is a demux of the layer info
bool batch_register_config = false;
};
+struct ConfigBufferInfo {
+ /**
+ * Sizes of all the successive ccw's (ccw burst).
+ * When working with descriptors list, each burst is programed independently to its descriptors.
+ */
+ std::vector<uint32_t> bursts_sizes;
+ /**
+ * Default offset = 0. In case of continuous pre-allocated buffer,
+ * we use this var to get the config buffer offset from the beginning of the hef user address.
+ */
+ uint64_t offset_from_hef_base = 0;
+};
+
// For each config_stream_index we store vector of all ccw write length. The vector is used to build the config buffer.g
-using ConfigBufferInfoMap = std::unordered_map<uint8_t, std::vector<uint32_t>>;
+using ConfigBufferInfoMap = std::unordered_map<uint8_t, ConfigBufferInfo>;
class ContextMetadata final {
void add_boundary_layer(const LayerInfo &layer_info);
void add_inter_context_layer(const LayerInfo &layer_info);
void add_ddr_layer(const LayerInfo &layer_info);
+ void add_cache_layer(const LayerInfo &layer_info);
const std::vector<LayerInfo> &get_boundary_input_layers() const;
const std::vector<LayerInfo> &get_boundary_output_layers() const;
const std::vector<LayerInfo> &get_inter_context_output_layers() const;
const std::vector<LayerInfo> &get_ddr_input_layers() const;
const std::vector<LayerInfo> &get_ddr_output_layers() const;
+ const std::vector<LayerInfo> &get_cache_input_layers() const;
+ const std::vector<LayerInfo> &get_cache_output_layers() const;
Expected<size_t> get_layers_transfer_size(const std::vector<LayerInfo> &layer_infos) const;
Expected<size_t> get_context_transfer_size() const;
std::vector<LayerInfo> m_inter_context_output_layers;
std::vector<LayerInfo> m_ddr_input_layers;
std::vector<LayerInfo> m_ddr_output_layers;
+ std::vector<LayerInfo> m_cache_input_layers;
+ std::vector<LayerInfo> m_cache_output_layers;
};
struct ConfigChannelInfo {
Expected<std::vector<LayerInfo>> get_input_layer_infos(const std::string &network_name) const;
Expected<std::vector<LayerInfo>> get_output_layer_infos(const std::string &network_name) const;
Expected<std::vector<LayerInfo>> get_all_layer_infos(const std::string &network_name) const;
+ size_t get_cache_layers_count() const;
const ContextMetadata &preliminary_context() const;
const std::vector<ContextMetadata> &dynamic_contexts() const;
#include "common/string_utils.hpp"
#include "common/utils.hpp"
#include "common/logger_macros.hpp"
-#include "common/file_utils.hpp"
#include "net_flow/ops/nms_post_process.hpp"
#include "net_flow/ops/yolov5_post_process.hpp"
#include "net_flow/ops/softmax_post_process.hpp"
#include "net_flow/ops/yolov5_seg_post_process.hpp"
#include "net_flow/ops/yolov8_post_process.hpp"
+#include "net_flow/ops/yolov8_bbox_only_post_process.hpp"
#include "hef/hef_internal.hpp"
#include "vdma/pcie/pcie_device.hpp"
#include "vdma/vdma_config_manager.hpp"
return !(*this == other);
}
-
// Note: Can't add the definition in the header. This will lead to the following error:
// /usr/include/c++/7/bits/unique_ptr.h: In instantiation of 'void std::default_delete<_Tp>::operator()(_Tp*) const [with _Tp = Hef::Impl]':
// /usr/include/c++/7/bits/unique_ptr.h:263:17: required from 'std::unique_ptr<_Tp, _Dp>::~unique_ptr() [with _Tp = Hef::Impl; _Dp = std::default_delete<Hef::Impl>]'
Expected<Hef> Hef::create(const std::string &hef_path)
{
- auto impl = Hef::Impl::create(hef_path);
- CHECK_EXPECTED(impl);
+ TRY(auto impl, Hef::Impl::create(hef_path));
// TODO: can we do this without the copy ctor here (i.e. make the impl as a unique_ptr to begin with)
- return Hef(make_unique_nothrow<Impl>(impl.release()));
+ return Hef(make_unique_nothrow<Impl>(std::move(impl)));
}
Expected<Hef> Hef::create(const MemoryView &hef_buffer)
{
- auto impl = Hef::Impl::create(hef_buffer);
- CHECK_EXPECTED(impl);
+ TRY(auto impl, Hef::Impl::create(hef_buffer));
// TODO: can we do this without the copy ctor here (i.e. make the impl as a unique_ptr to begin with)
- return Hef(make_unique_nothrow<Impl>(impl.release()));
+ return Hef(make_unique_nothrow<Impl>(std::move(impl)));
}
Hef::Hef(std::unique_ptr<Impl> pimpl) :
Expected<std::vector<hailo_stream_info_t>> Hef::get_input_stream_infos(const std::string &name) const
{
- auto network_pair = pimpl->get_network_group_and_network_name(name);
- CHECK_EXPECTED(network_pair);
-
- return pimpl->get_input_stream_infos(network_pair.value().first, network_pair.value().second);
+ TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name));
+ return pimpl->get_input_stream_infos(network_pair.first, network_pair.second);
}
Expected<std::vector<hailo_stream_info_t>> Hef::get_output_stream_infos(const std::string &name) const
{
- auto network_pair = pimpl->get_network_group_and_network_name(name);
- CHECK_EXPECTED(network_pair);
-
- return pimpl->get_output_stream_infos(network_pair.value().first, network_pair.value().second);
+ TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name));
+ return pimpl->get_output_stream_infos(network_pair.first, network_pair.second);
}
Expected<std::vector<hailo_stream_info_t>> Hef::get_all_stream_infos(const std::string &name) const
{
- auto network_pair = pimpl->get_network_group_and_network_name(name);
- CHECK_EXPECTED(network_pair);
-
- return pimpl->get_all_stream_infos(network_pair.value().first, network_pair.value().second);
+ TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name));
+ return pimpl->get_all_stream_infos(network_pair.first, network_pair.second);
}
Expected<std::vector<hailo_network_info_t>> Hef::get_network_infos(const std::string &net_group_name) const
{
- auto names_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(names_pair);
- return pimpl->get_network_infos(names_pair->first);
+ TRY(const auto names_pair, pimpl->get_network_group_and_network_name(net_group_name));
+ return pimpl->get_network_infos(names_pair.first);
}
Expected<hailo_stream_info_t> Hef::get_stream_info_by_name(const std::string &stream_name,
hailo_stream_direction_t stream_direction, const std::string &net_group_name) const
{
// Addressing the situation where net_group_name == ""
- auto net_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(net_group_name_pair);
- auto net_group_name_str = net_group_name_pair->first;
+ TRY(const auto net_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name));
+ const auto &net_group_name_str = net_group_name_pair.first;
return pimpl->get_stream_info_by_name(stream_name, stream_direction, net_group_name_str);
}
Expected<std::vector<hailo_vstream_info_t>> Hef::get_input_vstream_infos(const std::string &name) const
{
- auto network_pair = pimpl->get_network_group_and_network_name(name);
- CHECK_EXPECTED(network_pair);
-
- return pimpl->get_input_vstream_infos(network_pair.value().first, network_pair.value().second);
+ TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name));
+ return pimpl->get_input_vstream_infos(network_pair.first, network_pair.second);
}
Expected<std::vector<hailo_vstream_info_t>> Hef::get_output_vstream_infos(const std::string &name) const
{
- auto network_pair = pimpl->get_network_group_and_network_name(name);
- CHECK_EXPECTED(network_pair);
-
- return pimpl->get_output_vstream_infos(network_pair.value().first, network_pair.value().second);
+ TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name));
+ return pimpl->get_output_vstream_infos(network_pair.first, network_pair.second);
}
Expected<std::vector<hailo_vstream_info_t>> Hef::get_all_vstream_infos(const std::string &name) const
{
- auto network_pair = pimpl->get_network_group_and_network_name(name);
- CHECK_EXPECTED(network_pair);
-
- return pimpl->get_all_vstream_infos(network_pair.value().first, network_pair.value().second);
+ TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name));
+ return pimpl->get_all_vstream_infos(network_pair.first, network_pair.second);
}
Expected<std::vector<std::string>> Hef::get_sorted_output_names(const std::string &net_group_name) const
{
// Addressing the situation where net_group_name == ""
- auto net_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(net_group_name_pair);
- auto net_group_name_str = net_group_name_pair->first;
-
+ TRY(const auto net_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name));
+ const auto &net_group_name_str = net_group_name_pair.first;
return pimpl->get_sorted_output_names(net_group_name_str);
}
Expected<size_t> Hef::get_number_of_input_streams(const std::string &net_group_name) const
{
// Addressing the situation where net_group_name == ""
- auto net_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(net_group_name_pair);
- auto net_group_name_str = net_group_name_pair->first;
-
+ TRY(const auto net_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name));
+ const auto &net_group_name_str = net_group_name_pair.first;
return pimpl->get_number_of_input_streams(net_group_name_str);
}
Expected<size_t> Hef::get_number_of_output_streams(const std::string &net_group_name) const
{
// Addressing the situation where net_group_name == ""
- auto net_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(net_group_name_pair);
- auto net_group_name_str = net_group_name_pair->first;
-
+ TRY(const auto net_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name));
+ const auto &net_group_name_str = net_group_name_pair.first;
return pimpl->get_number_of_output_streams(net_group_name_str);
}
Expected<std::vector<std::string>> Hef::get_stream_names_from_vstream_name(const std::string &vstream_name,
const std::string &net_group_name) const
{
- auto network_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(network_group_name_pair);
- auto net_group_name_str = network_group_name_pair->first;
-
+ TRY(const auto network_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name));
+ const auto &net_group_name_str = network_group_name_pair.first;
return pimpl->get_stream_names_from_vstream_name(vstream_name, net_group_name_str);
}
Expected<std::vector<std::string>> Hef::get_vstream_names_from_stream_name(const std::string &stream_name,
const std::string &net_group_name) const
{
- auto network_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(network_group_name_pair);
- auto net_group_name_str = network_group_name_pair->first;
-
+ TRY(const auto network_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name));
+ const auto &net_group_name_str = network_group_name_pair.first;
return pimpl->get_vstream_names_from_stream_name(stream_name, net_group_name_str);
}
-ShefFileHandle::ShefFileHandle(const std::string &hef_path, uint32_t ccws_buffer_offset)
- : m_hef_path(hef_path), m_ccws_buffer_offset(ccws_buffer_offset) {}
-
-hailo_status ShefFileHandle::open()
-{
- m_hef_file = std::ifstream(m_hef_path, std::ios::in | std::ios::binary);
- CHECK(m_hef_file.is_open(), HAILO_OPEN_FILE_FAILURE, "Failed to open HEF file \"{}\". errno: {}", m_hef_path, errno);
- return HAILO_SUCCESS;
-}
-
-Expected<Buffer> ShefFileHandle::read(uint32_t offset, size_t size)
-{
- auto buffer = Buffer::create(size);
- CHECK_EXPECTED(buffer);
-
- m_hef_file.seekg(m_ccws_buffer_offset + offset, m_hef_file.beg);
- CHECK_AS_EXPECTED(m_hef_file.good(), HAILO_FILE_OPERATION_FAILURE, "Seeking in file failed");
-
- m_hef_file.read(reinterpret_cast<char*>(buffer->data()), size);
- CHECK_AS_EXPECTED(m_hef_file.good(), HAILO_FILE_OPERATION_FAILURE, "Failed reading ccw");
-
- return buffer;
-}
-
-hailo_status ShefFileHandle::close()
-{
- m_hef_file.close();
- CHECK_AS_EXPECTED(m_hef_file.good(), HAILO_CLOSE_FAILURE, "Closing file failed");
- return HAILO_SUCCESS;
-}
-
Expected<Hef::Impl> Hef::Impl::create(const std::string &hef_path)
{
hailo_status status = HAILO_UNINITIALIZED;
return hef;
}
+Expected<size_t> calc_hef_residue_size(std::shared_ptr<SeekableBytesReader> hef_reader, uint32_t version)
+{
+ TRY(auto total_size, hef_reader->get_size());
+ if (HEADER_VERSION_0 == version) {
+ return total_size - HEF_HEADER_SIZE_V0;
+ } else { // version is 1
+ return total_size - HEF_HEADER_SIZE_V1;
+ }
+}
+
+static hailo_status calc_buffer_md5(const uint8_t *buffer, const size_t buffer_size, MD5_SUM_t &calculated_md5)
+{
+ MD5_CTX md5 = {};
+ MD5_Init(&md5);
+ MD5_Update(&md5, buffer, buffer_size);
+ MD5_Final(calculated_md5, &md5);
+
+ return HAILO_SUCCESS;
+}
+
static hailo_status calc_istream_md5(std::ifstream &s, MD5_SUM_t &calculated_md5)
{
char md5_buffer[HEF__MD5_BUFFER_SIZE] = {};
CHECK(HEADER_MAGIC == header.magic, HAILO_INVALID_HEF,
"HEF magic does not match. detected magic - {:x}", header.magic);
- auto version = header.version;
- CHECK((HEADER_VERSION_0 == version) || (HEADER_VERSION_1 == version), HAILO_INVALID_HEF, "HEF version does not match");
+ CHECK((HEADER_VERSION_0 == header.version) , HAILO_INTERNAL_FAILURE,
+ "HEF version does not match. Should be {} but detected {}", HEADER_VERSION_0, header.version);
- CHECK(hef_file_residue_size == header.hef_proto_size + header.ccws_size, HAILO_INVALID_HEF,
+ CHECK(hef_file_residue_size == header.hef_proto_size, HAILO_INVALID_HEF,
"HEF file length does not match");
- CHECK(0 == memcmp(&calculated_md5, &header.expected_md5, sizeof(MD5_SUM_t)), HAILO_INVALID_HEF,
+ CHECK(0 == memcmp(&calculated_md5, &header.distinct.v0.expected_md5, sizeof(MD5_SUM_t)), HAILO_INVALID_HEF,
"HEF md5 does not match");
return HAILO_SUCCESS;
}
+hailo_status Hef::Impl::validate_hef_header(const hef__header_t &header, const uint32_t &crc_32, size_t hef_file_residue_size)
+{
+ CHECK(HEADER_MAGIC == header.magic, HAILO_INVALID_HEF,
+ "HEF magic does not match. Should be {:x} but detected magic - {:x}", HEADER_MAGIC, header.magic);
+
+ CHECK((HEADER_VERSION_1 == header.version), HAILO_INTERNAL_FAILURE,
+ "HEF version does not match. Should be {} but detected {}", HEADER_VERSION_1, header.version);
+
+ CHECK(hef_file_residue_size == header.hef_proto_size + header.distinct.v1.ccws_size, HAILO_INVALID_HEF,
+ "HEF file length does not match");
+
+ CHECK(0 == memcmp(&crc_32, &header.distinct.v1.crc, sizeof(crc_32)), HAILO_INVALID_HEF,
+ "HEF crc does not match");
+
+ return HAILO_SUCCESS;
+}
+
hailo_status Hef::Impl::validate_hef_extensions()
{
std::vector<std::string> unsupported_extensions;
memcpy(m_md5, calculated_md5, sizeof(m_md5));
}
+void Hef::Impl::init_crc(uint32_t crc_32)
+{
+ memcpy(&m_crc, &crc_32, sizeof(crc_32));
+}
+
+void Hef::Impl::init_hef_version(uint32_t version)
+{
+ m_hef_version = version;
+}
+
void Hef::Impl::clear_hef_buffer()
{
#ifdef HAILO_SUPPORT_MULTI_PROCESS
#endif // HAILO_SUPPORT_MULTI_PROCESS
}
+Expected<hef__header_t> Hef::Impl::parse_hef_header_before_distinct(std::shared_ptr<SeekableBytesReader> hef_reader)
+{
+ hef__header_t hef_header = {};
+ auto status = hef_reader->read(reinterpret_cast<uint8_t*>(&hef_header), HEF_COMMON_SIZE);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ hef_header.magic = BYTE_ORDER__htonl(hef_header.magic);
+ hef_header.version = BYTE_ORDER__htonl(hef_header.version);
+ hef_header.hef_proto_size = BYTE_ORDER__htonl(hef_header.hef_proto_size);
+
+ return hef_header;
+}
+
+hailo_status Hef::Impl::fill_v1_hef_header(hef__header_t &hef_header, std::shared_ptr<SeekableBytesReader> hef_reader)
+{
+ auto status = hef_reader->read(reinterpret_cast<uint8_t*>(&hef_header.distinct), sizeof(hef__header_distinct_t::v1));
+ CHECK_SUCCESS(status);
+
+ hef_header.distinct.v1.ccws_size = BYTE_ORDER__htonll(hef_header.distinct.v1.ccws_size);
+ hef_header.distinct.v1.crc = BYTE_ORDER__htonl(hef_header.distinct.v1.crc);
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status Hef::Impl::fill_core_ops_and_networks_metadata(uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
+{
+ fill_core_ops();
+
+ auto status = fill_networks_metadata(hef_version, hef_reader, ccws_offset);
+ CHECK_SUCCESS(status);
+
+ // Must be called after fill_networks_metadata
+ status = validate_hef_extensions();
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
+// TODO HRT-13920: remove duplications between parse_hef_file and parse_hef_memview
hailo_status Hef::Impl::parse_hef_file(const std::string &hef_path)
{
#ifdef HAILO_SUPPORT_MULTI_PROCESS
- auto hef_buffer = read_binary_file(hef_path);
- CHECK_EXPECTED_AS_STATUS(hef_buffer);
- m_hef_buffer = hef_buffer.release();
+ TRY(m_hef_buffer, read_binary_file(hef_path));
#endif // HAILO_SUPPORT_MULTI_PROCESS
- auto hef_file = std::ifstream(hef_path, std::ios::in | std::ios::binary);
- CHECK(hef_file.is_open(), HAILO_OPEN_FILE_FAILURE, "Failed to open HEF file \"{}\". errno: {}", hef_path, errno);
+ TRY(auto hef_reader, SeekableBytesReader::create_reader(hef_path));
+ auto status = hef_reader->open();
+ CHECK_SUCCESS(status);
+ m_hef_reader = hef_reader;
- hef__header_t header = {};
- hef_file.read(reinterpret_cast<char*>(&header), sizeof(header));
- CHECK(hef_file.good(), HAILO_FILE_OPERATION_FAILURE, "Failed reading HEF header");
+ TRY(auto hef_header, parse_hef_header_before_distinct(hef_reader));
+ init_hef_version(hef_header.version);
- header.magic = BYTE_ORDER__htonl(header.magic);
- header.version = BYTE_ORDER__htonl(header.version);
- header.hef_proto_size = BYTE_ORDER__htonl(header.hef_proto_size);
- header.ccws_size = BYTE_ORDER__htonl(header.ccws_size);
+ size_t ccws_offset = 0; // Relevant only for HEADER_VERSION_1
- auto hef_file_residue_size = get_istream_size(hef_file);
- CHECK_EXPECTED_AS_STATUS(hef_file_residue_size);
+ if (HEADER_VERSION_0 == hef_header.version) {
- MD5_SUM_t calculated_md5 = {};
- auto status = calc_istream_md5(hef_file, calculated_md5);
- CHECK_SUCCESS(status);
+ status = hef_reader->read(reinterpret_cast<uint8_t*>(&hef_header.distinct), sizeof(hef__header_distinct_t::v0));
+ CHECK_SUCCESS(status);
- status = validate_hef_header(header, calculated_md5, hef_file_residue_size.value());
- CHECK_SUCCESS(status);
+ MD5_SUM_t calculated_md5 = {};
+ status = calc_istream_md5(*hef_reader->get_fstream(), calculated_md5);
+ CHECK_SUCCESS(status);
- if (HEADER_VERSION_1 == header.version) {
- auto ptr = make_shared_nothrow<ShefFileHandle>(hef_path,
- static_cast<uint32_t>(sizeof(header) + header.hef_proto_size));
- CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY);
- m_shef_file_handle = ptr;
- }
+ TRY(const auto hef_file_residue_size, hef_reader->calculate_remaining_size());
+
+ status = validate_hef_header(hef_header, calculated_md5, hef_file_residue_size);
+ CHECK_SUCCESS(status);
+
+ init_md5(calculated_md5);
+
+ } else { // HEADER_VERSION_1 == common_hef_header.version
+
+ status = fill_v1_hef_header(hef_header, hef_reader);
+ CHECK_SUCCESS(status);
+
+ ccws_offset = HEF_HEADER_SIZE_V1 + hef_header.hef_proto_size;
+
+ TRY(auto calculated_residue_size, calc_hef_residue_size(hef_reader, hef_header.version));
+ TRY(auto calculated_crc, CRC32::calc_crc_on_stream(*hef_reader->get_fstream(), calculated_residue_size));
- init_md5(calculated_md5);
+ status = validate_hef_header(hef_header, calculated_crc, calculated_residue_size);
+ CHECK_SUCCESS(status);
+
+ init_crc(calculated_crc);
+
+ }
ProtoHEFHef hef_message;
- google::protobuf::io::IstreamInputStream zero_copy_input(&hef_file);
- auto rb = hef_message.ParseFromBoundedZeroCopyStream(&zero_copy_input, header.hef_proto_size);
+ google::protobuf::io::IstreamInputStream zero_copy_input(hef_reader->get_fstream().get());
+ auto rb = hef_message.ParseFromBoundedZeroCopyStream(&zero_copy_input, hef_header.hef_proto_size); // This line corrupts the file
CHECK(rb, HAILO_INVALID_HEF, "Failed parsing HEF file");
+ hef_reader->get_fstream()->clear(); // The call to ParseFromBoundedZeroCopyStream might corrupt the file, so we need to clear it's error flags
+ // TODO: Remove this reset after stopping support for V0 (in the new format (V1), the file is not corrupted after parsing the protobuf message).
status = transfer_protobuf_field_ownership(hef_message);
CHECK_SUCCESS(status);
- fill_core_ops();
-
- status = fill_networks_metadata();
+ status = fill_core_ops_and_networks_metadata(hef_header.version, hef_reader, ccws_offset);
CHECK_SUCCESS(status);
- // Must be called after fill_networks_metadata
- status = validate_hef_extensions();
+ status = hef_reader->close();
CHECK_SUCCESS(status);
TRACE(HefLoadedTrace, hef_path, m_header.sdk_version(), m_md5);
return HAILO_SUCCESS;
}
+hailo_status Hef::Impl::parse_hef_memview_internal(const size_t proto_size, const uint8_t *proto_buffer, const uint32_t hef_version,
+ std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
+{
+ ProtoHEFHef hef_message;
+ auto rb = hef_message.ParseFromArray(proto_buffer, static_cast<int>(proto_size));
+ CHECK(rb, HAILO_INVALID_HEF, "Failed parsing HEF buffer");
+ auto status = transfer_protobuf_field_ownership(hef_message);
+ CHECK_SUCCESS(status);
+
+ status = fill_core_ops_and_networks_metadata(hef_version, hef_reader, ccws_offset);
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
+// TODO HRT-13920: remove duplications between parse_hef_file and parse_hef_memview
hailo_status Hef::Impl::parse_hef_memview(const MemoryView &hef_memview)
{
#ifdef HAILO_SUPPORT_MULTI_PROCESS
- auto hef_buffer = Buffer::create(hef_memview.data(), hef_memview.size());
- CHECK_EXPECTED_AS_STATUS(hef_buffer);
- m_hef_buffer = hef_buffer.release();
+ TRY(m_hef_buffer, Buffer::create(hef_memview.data(), hef_memview.size()));
#endif // HAILO_SUPPORT_MULTI_PROCESS
+ TRY(auto hef_reader, SeekableBytesReader::create_reader(hef_memview));
+ m_hef_reader = hef_reader;
+
+ TRY(auto hef_header, parse_hef_header_before_distinct(hef_reader));
+ init_hef_version(hef_header.version);
+
CHECK(hef_memview.size() >= sizeof(hef__header_t), HAILO_INVALID_HEF, "Invalid HEF header");
- const hef__header_t &raw_header = reinterpret_cast<const hef__header_t&>(*hef_memview.data());
- auto header = raw_header;
- header.magic = BYTE_ORDER__htonl(header.magic);
- header.version = BYTE_ORDER__htonl(header.version);
- header.hef_proto_size = BYTE_ORDER__htonl(header.hef_proto_size);
- header.ccws_size = BYTE_ORDER__htonl(header.ccws_size);
- auto proto_buffer = (hef_memview.data() + sizeof(header));
- auto proto_size = (hef_memview.size() - sizeof(header));
+ size_t ccws_offset = 0; // Relevant only for HEADER_VERSION_1
- MD5_CTX md5 = {};
- MD5_SUM_t calculated_md5 = {};
- MD5_Init(&md5);
- MD5_Update(&md5, proto_buffer, proto_size);
- MD5_Final(calculated_md5, &md5);
+ if (HEADER_VERSION_0 == hef_header.version) {
+ auto status = hef_reader->read(reinterpret_cast<uint8_t*>(&hef_header.distinct), sizeof(hef__header_distinct_t::v0));
+ CHECK_SUCCESS(status);
- auto status = validate_hef_header(header, calculated_md5, proto_size);
- CHECK_SUCCESS(status);
+ auto proto_buffer = (hef_memview.data() + HEF_HEADER_SIZE_V0);
+ auto proto_size = (hef_memview.size() - HEF_HEADER_SIZE_V0);
- init_md5(calculated_md5);
+ MD5_SUM_t calculated_md5 = {};
+ status = calc_buffer_md5(proto_buffer, proto_size, calculated_md5);
+ CHECK_SUCCESS(status);
- ProtoHEFHef hef_message;
- auto rb = hef_message.ParseFromArray(proto_buffer, static_cast<int>(proto_size));
- CHECK(rb, HAILO_INVALID_HEF, "Failed parsing HEF buffer");
- status = transfer_protobuf_field_ownership(hef_message);
- CHECK_SUCCESS(status);
+ status = validate_hef_header(hef_header, calculated_md5, proto_size);
+ CHECK_SUCCESS(status);
- fill_core_ops();
+ init_md5(calculated_md5);
- status = fill_networks_metadata();
- CHECK_SUCCESS(status);
+ return parse_hef_memview_internal(proto_size, proto_buffer, hef_header.version, hef_reader, ccws_offset);
- // Must be called after fill_networks_metadata
- status = validate_hef_extensions();
- CHECK_SUCCESS(status);
+ } else { // version is HEADER_VERSION_1
+ auto status = fill_v1_hef_header(hef_header, hef_reader);
+ CHECK_SUCCESS(status);
- return HAILO_SUCCESS;
+ auto proto_and_ccw_buffer = hef_memview.data() + HEF_HEADER_SIZE_V1;
+ auto proto_size = hef_memview.size() - HEF_HEADER_SIZE_V1 - hef_header.distinct.v1.ccws_size;
+
+ ccws_offset = HEF_HEADER_SIZE_V1 + hef_header.hef_proto_size;
+
+ TRY(auto proto_and_ccws_size, calc_hef_residue_size(hef_reader, hef_header.version));
+ auto proto_and_ccws_buffer = MemoryView::create_const(hef_memview.data() + HEF_HEADER_SIZE_V1, proto_and_ccws_size);
+ TRY(auto calculated_crc, CRC32::calc_crc_on_buffer(proto_and_ccws_buffer));
+
+ status = validate_hef_header(hef_header, calculated_crc, proto_and_ccws_size);
+ CHECK_SUCCESS(status);
+
+ init_crc(calculated_crc);
+
+ return parse_hef_memview_internal(static_cast<size_t>(proto_size), proto_and_ccw_buffer, hef_header.version, hef_reader, ccws_offset);
+ }
}
return (hw_arch == ProtoHEFHwArch::PROTO__HW_ARCH__HAILO8L) || (hw_arch == ProtoHEFHwArch::PROTO__HW_ARCH__HAILO15M);
}
-hailo_status Hef::Impl::fill_networks_metadata()
+hailo_status Hef::Impl::fill_networks_metadata(uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
{
fill_extensions_bitset();
if (m_supported_features.hailo_net_flow) {
for (auto &partial_core_op : core_op.partial_core_ops) {
partial_clusters_layout_bitmap = partial_core_op->layout.partial_clusters_layout_bitmap();
- auto metadata_per_arch_exp = create_metadata_per_arch(*(partial_core_op->core_op), sorted_network_names);
- CHECK_EXPECTED_AS_STATUS(metadata_per_arch_exp);
- auto metadata_per_arch = metadata_per_arch_exp.release();
+ TRY(const auto metadata_per_arch,
+ create_metadata_per_arch(*(partial_core_op->core_op), sorted_network_names, hef_version, hef_reader, ccws_offset));
- auto expected_ops_metadata = create_ops_metadata(*network_group, *metadata_per_arch, get_device_arch());
- CHECK_EXPECTED_AS_STATUS(expected_ops_metadata);
- m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), expected_ops_metadata.value()});
+ TRY(const auto ops_metadata, create_ops_metadata(*network_group, *metadata_per_arch, get_device_arch()));
+ m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), ops_metadata});
core_op_metadata.add_metadata(metadata_per_arch, partial_clusters_layout_bitmap);
}
} else {
{}
};
- auto metadata_per_arch_exp = create_metadata_per_arch(partial_core_op, sorted_network_names);
- CHECK_EXPECTED_AS_STATUS(metadata_per_arch_exp);
- auto metadata_per_arch = metadata_per_arch_exp.release();
+ TRY(const auto metadata_per_arch, create_metadata_per_arch(partial_core_op, sorted_network_names, hef_version, hef_reader, ccws_offset));
std::vector<net_flow::PostProcessOpMetadataPtr> empty_metadata_ops;
m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), empty_metadata_ops});
}
} else {
partial_clusters_layout_bitmap = PARTIAL_CLUSTERS_LAYOUT_IGNORE;
- auto metadata_per_arch_exp = create_metadata_per_arch(core_op, sorted_network_names);
- CHECK_EXPECTED_AS_STATUS(metadata_per_arch_exp);
- auto metadata_per_arch = metadata_per_arch_exp.release();
+ TRY(const auto metadata_per_arch, create_metadata_per_arch(core_op, sorted_network_names, hef_version, hef_reader, ccws_offset));
- auto expected_ops_metadata = create_ops_metadata(*network_group, *metadata_per_arch, get_device_arch());
- CHECK_EXPECTED_AS_STATUS(expected_ops_metadata);
- m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), expected_ops_metadata.value()});
+ TRY(auto ops_metadata, create_ops_metadata(*network_group, *metadata_per_arch, get_device_arch()));
+ m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), ops_metadata});
core_op_metadata.add_metadata(metadata_per_arch, partial_clusters_layout_bitmap);
}
// Taking the full-layout's name (name is same across all layouts)
- auto metadata_exp = core_op_metadata.get_metadata(PARTIAL_CLUSTERS_LAYOUT_IGNORE);
- CHECK_EXPECTED_AS_STATUS(metadata_exp);
- auto core_op_name = metadata_exp.value()->core_op_name();
+ TRY(const auto metadata, core_op_metadata.get_metadata(PARTIAL_CLUSTERS_LAYOUT_IGNORE));
+ auto core_op_name = metadata->core_op_name();
std::map<std::string, CoreOpMetadataPerArch> core_op_metadata_map;
core_op_metadata_map[core_op_name] = core_op_metadata;
// Prepare network_group_metadata
sorted_output_names.push_back(name);
}
}
- auto network_group_metadata = NetworkGroupMetadata::create(network_group_name, std::move(core_op_metadata_map),
- sorted_output_names, m_supported_features, sorted_network_names, m_post_process_ops_metadata_per_group.at(network_group_name));
-
- CHECK_EXPECTED_AS_STATUS(network_group_metadata);
- m_network_group_metadata.emplace(network_group_name, network_group_metadata.release());
+ TRY(auto network_group_metadata, NetworkGroupMetadata::create(network_group_name, std::move(core_op_metadata_map),
+ sorted_output_names, m_supported_features, sorted_network_names, m_post_process_ops_metadata_per_group.at(network_group_name)));
+ m_network_group_metadata.emplace(network_group_name, std::move(network_group_metadata));
}
return HAILO_SUCCESS;
}
return config_channels_info;
}
-Expected<CoreOpMetadataPtr> Hef::Impl::create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op, const std::vector<std::string> &sorted_network_names)
+Expected<CoreOpMetadataPtr> Hef::Impl::create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op, const std::vector<std::string> &sorted_network_names,
+ uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
{
- auto preliminary_context = HefUtils::parse_preliminary_context(core_op.preliminary_config, m_supported_features, m_shef_file_handle);
- CHECK_EXPECTED(preliminary_context);
-
- auto dynamic_contexts = HefUtils::parse_dynamic_contexts(core_op, m_supported_features, get_device_arch(), m_shef_file_handle);
- CHECK_EXPECTED(dynamic_contexts);
-
- auto config_channels_info = parse_config_channels_info(core_op);
- CHECK_EXPECTED(config_channels_info);
+ // TODO: validate that there's a read+write layer for each cache + no cache_id is only read or written without the
+ // other. They can be across different contexts (HRT-13655)
+ TRY(auto preliminary_context, HefUtils::parse_preliminary_context(core_op.preliminary_config, m_supported_features, hef_version, hef_reader, ccws_offset));
+ TRY_V(auto dynamic_contexts, HefUtils::parse_dynamic_contexts(core_op, m_supported_features, get_device_arch(), hef_version, hef_reader, ccws_offset));
+ TRY(auto config_channels_info, parse_config_channels_info(core_op));
// If const input layer is found in the preliminary context, or first dynamic context we can't use fast batch switch
const auto can_fast_batch_switch =
- !(preliminary_context.value().const_input_layer_found() || dynamic_contexts.value()[0].const_input_layer_found());
+ !(preliminary_context.const_input_layer_found() || dynamic_contexts[0].const_input_layer_found());
// Currently, CoreOp name is the same as network_group_name, thats why we init it with it.
// TODO: HRT-9551 - Change it when supporting multi core ops.
auto metadata_per_arch = make_shared_nothrow<CoreOpMetadata>(core_op.network_group_metadata.network_group_name(),
- preliminary_context.release(), dynamic_contexts.release(), config_channels_info.release(),
+ std::move(preliminary_context), std::move(dynamic_contexts), std::move(config_channels_info),
m_supported_features, sorted_network_names, can_fast_batch_switch);
CHECK_NOT_NULL_AS_EXPECTED(metadata_per_arch, HAILO_OUT_OF_HOST_MEMORY);
return metadata_per_arch;
}
#endif // HAILO_SUPPORT_MULTI_PROCESS
-Hef::Impl::Impl(const std::string &hef_path, hailo_status &status) : m_shef_file_handle(nullptr)
+Hef::Impl::Impl(const std::string &hef_path, hailo_status &status)
{
status = HAILO_UNINITIALIZED;
GOOGLE_PROTOBUF_VERIFY_VERSION;
status = HAILO_SUCCESS;
}
-Hef::Impl::Impl(const MemoryView &hef_memview, hailo_status &status) : m_shef_file_handle(nullptr)
+Hef::Impl::Impl(const MemoryView &hef_memview, hailo_status &status)
{
status = HAILO_UNINITIALIZED;
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
auto nms_config = create_post_process_nms_config(op_proto);
- auto yolo_config = create_yolov5_config(op_proto.nms_op().yolo_nms_op().bbox_decoders(),
- op_proto.nms_op().yolo_nms_op().image_height(), op_proto.nms_op().yolo_nms_op().image_width(), pad_index_to_streams_info);
- CHECK_EXPECTED(yolo_config);
-
- auto inputs_metadata = create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads);
- CHECK_EXPECTED(inputs_metadata);
+ TRY(auto yolo_config, create_yolov5_config(op_proto.nms_op().yolo_nms_op().bbox_decoders(),
+ op_proto.nms_op().yolo_nms_op().image_height(), op_proto.nms_op().yolo_nms_op().image_width(), pad_index_to_streams_info));
+ TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads));
std::unordered_map<std::string, net_flow::BufferMetaData> outputs_metadata;
net_flow::BufferMetaData output_metadata{};
outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata});
- return net_flow::Yolov5OpMetadata::create(inputs_metadata.release(), outputs_metadata, nms_config, yolo_config.release(),
+ return net_flow::Yolov5OpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolo_config,
network_name);
}
{
auto nms_config = create_post_process_nms_config(op_proto);
- auto yolo_v5_config = create_yolov5_config(op_proto.nms_op().yolo_nms_op().bbox_decoders(),
- op_proto.nms_op().yolo_nms_op().image_height(), op_proto.nms_op().yolo_nms_op().image_width(), pad_index_to_streams_info);
- CHECK_EXPECTED(yolo_v5_config);
-
- auto inputs_metadata = create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads);
- CHECK_EXPECTED(inputs_metadata);
+ TRY(auto yolo_v5_config, create_yolov5_config(op_proto.nms_op().yolo_nms_op().bbox_decoders(),
+ op_proto.nms_op().yolo_nms_op().image_height(), op_proto.nms_op().yolo_nms_op().image_width(), pad_index_to_streams_info));
+ TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads));
std::unordered_map<std::string, net_flow::BufferMetaData> outputs_metadata;
net_flow::BufferMetaData output_metadata{};
- uint32_t num_of_proposals = compute_num_of_proposals(inputs_metadata.value(), yolo_v5_config->anchors);
+ uint32_t num_of_proposals = compute_num_of_proposals(inputs_metadata, yolo_v5_config.anchors);
output_metadata.shape = {1, num_of_proposals, YOLOV5_BBOX_NUM_OF_VALUES + op_proto.nms_op().classes()};
output_metadata.format = net_flow::NmsOpMetadata::expand_output_format_autos_by_op_type(
{ HAILO_FORMAT_TYPE_AUTO, HAILO_FORMAT_ORDER_AUTO, HAILO_FORMAT_FLAGS_NONE }, net_flow::OperationType::YOLOV5, nms_config.bbox_only);
outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata});
- return net_flow::Yolov5BboxOnlyOpMetadata::create(inputs_metadata.release(), outputs_metadata, nms_config, yolo_v5_config.release(),
+ return net_flow::Yolov5BboxOnlyOpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolo_v5_config,
network_name);
}
const std::string &network_name)
{
auto nms_config = create_post_process_nms_config(op_proto);
- auto yolov5_config = create_yolov5_config(op_proto.nms_op().yolo_seg_op().bbox_decoders(),
- op_proto.nms_op().yolo_seg_op().image_height(), op_proto.nms_op().yolo_seg_op().image_width(), pad_index_to_streams_info);
- CHECK_EXPECTED(yolov5_config);
-
- auto inputs_metadata = create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads);
- CHECK_EXPECTED(inputs_metadata);
+ TRY(auto yolov5_config, create_yolov5_config(op_proto.nms_op().yolo_seg_op().bbox_decoders(),
+ op_proto.nms_op().yolo_seg_op().image_height(), op_proto.nms_op().yolo_seg_op().image_width(), pad_index_to_streams_info));
+ TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads));
auto proto_layer_name = op_proto.nms_op().yolo_seg_op().proto_info().proto_layer();
- CHECK_AS_EXPECTED(contains(inputs_metadata.value(), proto_layer_name), HAILO_INVALID_HEF);
+ CHECK_AS_EXPECTED(contains(inputs_metadata, proto_layer_name), HAILO_INVALID_HEF);
const uint32_t SIZE_FACTOR = 2;
net_flow::YoloV5SegPostProcessConfig yolov5_seg_config = {};
yolov5_seg_config.mask_threshold = static_cast<float32_t>(op_proto.nms_op().yolo_seg_op().mask_threshold());
yolov5_seg_config.max_accumulated_mask_size = static_cast<uint32_t>(
- yolov5_config->image_height * yolov5_config->image_width * SIZE_FACTOR);
+ yolov5_config.image_height * yolov5_config.image_width * SIZE_FACTOR);
yolov5_seg_config.proto_layer_name = proto_layer_name;
std::unordered_map<std::string, net_flow::BufferMetaData> outputs_metadata;
net_flow::OperationType::YOLOV5SEG, nms_config.bbox_only);
outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata});
- return net_flow::Yolov5SegOpMetadata::create(inputs_metadata.release(), outputs_metadata, nms_config, yolov5_config.release(),
+ return net_flow::Yolov5SegOpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolov5_config,
yolov5_seg_config, network_name);
}
-Expected<net_flow::PostProcessOpMetadataPtr> create_yolov8_op_metadata(const ProtoHEFOp &op_proto,
- const std::map<size_t, LayerInfo> &pad_index_to_streams_info, const std::map<size_t, size_t> &input_to_output_pads,
- const std::string &network_name)
+Expected<net_flow::Yolov8PostProcessConfig> create_yolov8_config(const ProtoHEFOp &op_proto, const std::map<size_t, LayerInfo> &pad_index_to_streams_info)
{
- auto nms_config = create_post_process_nms_config(op_proto);
-
net_flow::Yolov8PostProcessConfig yolov8_config{};
yolov8_config.image_height = (float32_t)op_proto.nms_op().yolov8_nms_op().image_height();
yolov8_config.image_width = (float32_t)op_proto.nms_op().yolov8_nms_op().image_width();
- std::unordered_map<std::string, net_flow::BufferMetaData> inputs_metadata;
- std::unordered_map<std::string, net_flow::BufferMetaData> outputs_metadata;
- net_flow::BufferMetaData output_metadata{};
- output_metadata.format = net_flow::NmsOpMetadata::expand_output_format_autos_by_op_type(
- { HAILO_FORMAT_TYPE_AUTO, HAILO_FORMAT_ORDER_AUTO, HAILO_FORMAT_FLAGS_NONE }, net_flow::OperationType::YOLOV8, nms_config.bbox_only);
- outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata});
-
for (auto &bbox_proto : op_proto.nms_op().yolov8_nms_op().bbox_decoders()) {
assert(contains(pad_index_to_streams_info, static_cast<size_t>(bbox_proto.reg_pad_index())));
auto reg_name = pad_index_to_streams_info.at(bbox_proto.reg_pad_index()).name;
auto cls_name = pad_index_to_streams_info.at(bbox_proto.cls_pad_index()).name;
yolov8_config.reg_to_cls_inputs.emplace_back(net_flow::Yolov8MatchingLayersNames{reg_name, cls_name, bbox_proto.stride()});
}
+ return yolov8_config;
+}
- for (auto &input_pad : op_proto.input_pads()) {
- CHECK_AS_EXPECTED(contains(input_to_output_pads, static_cast<size_t>(input_pad.index())), HAILO_INVALID_HEF,
- "NMS op is not connected to core op");
- auto output_pad_index = input_to_output_pads.at(input_pad.index());
- CHECK_AS_EXPECTED(contains(pad_index_to_streams_info, output_pad_index), HAILO_INVALID_HEF,
- "Pad {} of post-process {} is not connected to any core output stream",
- input_pad.index(), op_proto.name());
- const auto &op_input_stream = pad_index_to_streams_info.at(output_pad_index);
- net_flow::BufferMetaData input_metadata{};
- input_metadata.format = op_input_stream.format;
- input_metadata.quant_info = op_input_stream.quant_info;
- input_metadata.shape = op_input_stream.shape;
- input_metadata.padded_shape = op_input_stream.hw_shape;
- inputs_metadata.insert({op_input_stream.name, input_metadata});
- }
+Expected<net_flow::PostProcessOpMetadataPtr> create_yolov8_op_metadata(const ProtoHEFOp &op_proto,
+ const std::map<size_t, LayerInfo> &pad_index_to_streams_info, const std::map<size_t, size_t> &input_to_output_pads,
+ const std::string &network_name)
+{
+ auto nms_config = create_post_process_nms_config(op_proto);
+ TRY(auto yolov8_config, create_yolov8_config(op_proto, pad_index_to_streams_info));
+
+ std::unordered_map<std::string, net_flow::BufferMetaData> outputs_metadata;
+ net_flow::BufferMetaData output_metadata{};
+ output_metadata.format = net_flow::NmsOpMetadata::expand_output_format_autos_by_op_type(
+ { HAILO_FORMAT_TYPE_AUTO, HAILO_FORMAT_ORDER_AUTO, HAILO_FORMAT_FLAGS_NONE }, net_flow::OperationType::YOLOV8, nms_config.bbox_only);
+ outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata});
+
+ TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads));
return net_flow::Yolov8OpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolov8_config,
network_name);
}
+uint32_t compute_yolov8_bbox_only_num_of_proposals(const std::unordered_map<std::string, net_flow::BufferMetaData> &inputs_metadatas)
+{
+ static const uint32_t YOLOV8_NUM_OF_OUTPUTS_PER_SHAPE = 2;
+ uint32_t num_of_proposals = 0;
+ for (const auto &input_metadata_pair : inputs_metadatas) {
+ auto &input_metadata = input_metadata_pair.second;
+ num_of_proposals += input_metadata.shape.height * input_metadata.shape.width;
+ }
+ return num_of_proposals / YOLOV8_NUM_OF_OUTPUTS_PER_SHAPE; // we want to count the proposals from each unique shape only once
+}
+
+Expected<net_flow::PostProcessOpMetadataPtr> create_yolov8_bbox_only_op_metadata(const ProtoHEFOp &op_proto,
+ const std::map<size_t, LayerInfo> &pad_index_to_streams_info, const std::map<size_t, size_t> &input_to_output_pads,
+ const std::string &network_name)
+{
+ auto nms_config = create_post_process_nms_config(op_proto);
+ TRY(auto yolov8_config, create_yolov8_config(op_proto, pad_index_to_streams_info));
+
+ TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads));
+
+ uint32_t num_of_proposals = compute_yolov8_bbox_only_num_of_proposals(inputs_metadata);
+
+ std::unordered_map<std::string, net_flow::BufferMetaData> outputs_metadata;
+ net_flow::BufferMetaData output_metadata{};
+ output_metadata.format = net_flow::NmsOpMetadata::expand_output_format_autos_by_op_type(
+ { HAILO_FORMAT_TYPE_AUTO, HAILO_FORMAT_ORDER_AUTO, HAILO_FORMAT_FLAGS_NONE }, net_flow::OperationType::YOLOV8, nms_config.bbox_only);
+ auto bbox_num_of_coordinates = static_cast<uint32_t>(sizeof(hailo_rectangle_t) / sizeof(float32_t));
+ output_metadata.shape = {1, num_of_proposals, bbox_num_of_coordinates + op_proto.nms_op().classes()};
+ outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata});
+
+ return net_flow::Yolov8BboxOnlyOpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolov8_config,
+ network_name);
+}
+
Expected<net_flow::PostProcessOpMetadataPtr> create_yolox_op_metadata(const ProtoHEFOp &op_proto,
const std::map<size_t, LayerInfo> &pad_index_to_streams_info, const std::map<size_t, size_t> &input_to_output_pads,
const std::string &network_name)
// TODO: HRT-10603
const auto &op_input_layer_info = pad_index_to_streams_info.at(output_pad_index);
- auto max_periph_bytes_from_hef = HefConfigurator::max_periph_bytes_value(
- DeviceBase::hef_arch_to_device_arch(static_cast<HEFHwArch>(hef_arch)));
- CHECK_EXPECTED(max_periph_bytes_from_hef);
+ TRY(const auto max_periph_bytes_from_hef, HefConfigurator::max_periph_bytes_value(
+ DeviceBase::hef_arch_to_device_arch(static_cast<HEFHwArch>(hef_arch))));
// TODO HRT-12099 - return invalid hef error when remove support for hefs with no max_shmifo size
- const auto max_periph_bytes = (0 == op_input_layer_info.max_shmifo_size) ? max_periph_bytes_from_hef.value() :
- std::min(max_periph_bytes_from_hef.value(), op_input_layer_info.max_shmifo_size);
+ const auto max_periph_bytes = (0 == op_input_layer_info.max_shmifo_size) ? max_periph_bytes_from_hef :
+ std::min(max_periph_bytes_from_hef, op_input_layer_info.max_shmifo_size);
const auto is_core_hw_padding_supported = HefConfigurator::is_core_hw_padding_supported(op_input_layer_info,
max_periph_bytes, is_core_hw_padding_config_in_dfc);
switch (op_proto.nms_op().nms_op_case()) {
case ProtoHEFNmsOp::kYoloNmsOp: {
if (op_proto.nms_op().bbox_decoding_only()) {
- auto expected_post_process_op_metadata = create_yolov5_bbox_only_op_metadata(op_proto, pad_index_to_streams_info,
- input_to_output_pads, network_name);
- CHECK_EXPECTED(expected_post_process_op_metadata);
- post_process_op_metadata = expected_post_process_op_metadata.release();
+ TRY(post_process_op_metadata, create_yolov5_bbox_only_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, network_name));
break;
} else {
- auto expected_post_process_op_metadata = create_yolov5_op_metadata(op_proto, pad_index_to_streams_info,
- input_to_output_pads, network_name);
- CHECK_EXPECTED(expected_post_process_op_metadata);
- post_process_op_metadata = expected_post_process_op_metadata.release();
+ TRY(post_process_op_metadata, create_yolov5_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, network_name));
break;
}
}
case ProtoHEFNmsOp::kYoloxNmsOp: {
- auto expected_post_process_op_metadata = create_yolox_op_metadata(op_proto, pad_index_to_streams_info,
- input_to_output_pads, network_name);
- CHECK_EXPECTED(expected_post_process_op_metadata);
- post_process_op_metadata = expected_post_process_op_metadata.release();
+ TRY(post_process_op_metadata, create_yolox_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, network_name));
break;
}
case ProtoHEFNmsOp::kSsdNmsOp: {
- auto expected_post_process_op_metadata = create_ssd_op_metadata(op_proto, pad_index_to_streams_info,
- input_to_output_pads, network_name);
- CHECK_EXPECTED(expected_post_process_op_metadata);
- post_process_op_metadata = expected_post_process_op_metadata.release();
+ TRY(post_process_op_metadata, create_ssd_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, network_name));
break;
}
case ProtoHEFNmsOp::kIouOp: {
- auto expected_post_process_op_metadata = create_iou_op_metadata(op_proto, pad_index_to_streams_info,
- input_to_output_pads, network_name);
- CHECK_EXPECTED(expected_post_process_op_metadata);
- post_process_op_metadata = expected_post_process_op_metadata.release();
+ TRY(post_process_op_metadata, create_iou_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, network_name));
break;
}
case ProtoHEFNmsOp::kYoloSegOp: {
- auto expected_post_process_op_metadata = create_yolov5_seg_op_metadata(op_proto, pad_index_to_streams_info,
- input_to_output_pads, network_name);
- CHECK_EXPECTED(expected_post_process_op_metadata);
- post_process_op_metadata = expected_post_process_op_metadata.release();
+ TRY(post_process_op_metadata, create_yolov5_seg_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, network_name));
break;
}
case ProtoHEFNmsOp::kYolov8NmsOp: {
- auto expected_post_process_op_metadata = create_yolov8_op_metadata(op_proto, pad_index_to_streams_info,
- input_to_output_pads, network_name);
- CHECK_EXPECTED(expected_post_process_op_metadata);
- post_process_op_metadata = expected_post_process_op_metadata.release();
- break;
+ if (op_proto.nms_op().bbox_decoding_only()) {
+ TRY(post_process_op_metadata, create_yolov8_bbox_only_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, network_name));
+ break;
+ } else {
+ TRY(post_process_op_metadata, create_yolov8_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, network_name));
+ break;
+ }
}
default: {
LOGGER__ERROR("Unsupported Net-Flow NMS-Op");
break;
}
case ProtoHEFOp::kLogitsOp: {
- auto expected_logits_op_metadata = create_logits_op_metadata(op_proto, pad_index_to_streams_info,
- input_to_output_pads, hef_arch, network_name, m_supported_features.core_hw_padding_config_in_dfc);
- CHECK_EXPECTED(expected_logits_op_metadata);
- auto post_process_op_metadata = expected_logits_op_metadata.release();
-
+ TRY(auto post_process_op_metadata, create_logits_op_metadata(op_proto, pad_index_to_streams_info,
+ input_to_output_pads, hef_arch, network_name, m_supported_features.core_hw_padding_config_in_dfc));
result.push_back(post_process_op_metadata);
break;
}
hailo_status Hef::Impl::validate_boundary_streams_were_created(const std::string &network_group_name, std::shared_ptr<CoreOp> core_op)
{
- auto number_of_inputs = get_number_of_input_streams(network_group_name);
- CHECK_EXPECTED_AS_STATUS(number_of_inputs);
-
- auto size = core_op->get_input_streams().size();
- CHECK((number_of_inputs.value() == size),
+ TRY(const auto number_of_inputs, get_number_of_input_streams(network_group_name));
+ const auto size = core_op->get_input_streams().size();
+ CHECK((number_of_inputs == size),
HAILO_INVALID_ARGUMENT, "passed configure_params for network group {} did not contain all input streams", network_group_name);
- auto number_of_outputs = get_number_of_output_streams(network_group_name);
- CHECK_EXPECTED_AS_STATUS(number_of_inputs);
- CHECK((number_of_outputs.value() == core_op->get_output_streams().size()),
+ TRY(const auto number_of_outputs, get_number_of_output_streams(network_group_name));
+ CHECK((number_of_outputs == core_op->get_output_streams().size()),
HAILO_INVALID_ARGUMENT, "passed configure_params for network group {} did not contain all output streams", network_group_name);
return HAILO_SUCCESS;
case HAILO_ARCH_HAILO15H:
case HAILO_ARCH_HAILO15M:
case HAILO_ARCH_PLUTO:
+ case HAILO_ARCH_HAILO10H:
return HAILO1X_PERIPH_BYTES_PER_BUFFER_MAX_SIZE;
default:
LOGGER__ERROR("Unknown device architecture!");
case HAILO_ARCH_HAILO15H:
case HAILO_ARCH_HAILO15M:
case HAILO_ARCH_PLUTO:
+ case HAILO_ARCH_HAILO10H:
return HAILO1X_PERIPH_PAYLOAD_MAX_VALUE;
default:
LOGGER__ERROR("Unknown device architecture!");
Expected<std::vector<hailo_stream_info_t>> Hef::Impl::get_input_stream_infos(const std::string &net_group_name,
const std::string &network_name)
{
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
-
- return core_op_metadata.value()->get_input_stream_infos(network_name);
+ TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name));
+ return core_op_metadata->get_input_stream_infos(network_name);
}
Expected<std::vector<hailo_stream_info_t>> Hef::Impl::get_output_stream_infos(const std::string &net_group_name,
const std::string &network_name)
{
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
-
- return core_op_metadata.value()->get_output_stream_infos(network_name);
+ TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name));
+ return core_op_metadata->get_output_stream_infos(network_name);
}
Expected<std::vector<hailo_stream_info_t>> Hef::Impl::get_all_stream_infos(const std::string &net_group_name,
const std::string &network_name)
{
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
-
- return core_op_metadata.value()->get_all_stream_infos(network_name);
+ TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name));
+ return core_op_metadata->get_all_stream_infos(network_name);
}
Expected<std::vector<hailo_network_info_t>> Hef::Impl::get_network_infos(const std::string &net_group_name)
Expected<hailo_stream_info_t> Hef::Impl::get_stream_info_by_name(const std::string &stream_name,
hailo_stream_direction_t stream_direction, const std::string &net_group_name)
{
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
+ TRY(auto core_op_metadata, get_core_op_metadata(net_group_name));
if (HAILO_H2D_STREAM == stream_direction) {
- auto stream_infos = core_op_metadata.value()->get_input_stream_infos();
- CHECK_EXPECTED(stream_infos);
- for (auto &stream_info : stream_infos.value()) {
+ TRY(auto stream_infos, core_op_metadata->get_input_stream_infos());
+ for (auto &stream_info : stream_infos) {
if (stream_name == stream_info.name) {
return std::move(stream_info);
}
}
} else {
- auto stream_infos = core_op_metadata.value()->get_output_stream_infos();
- CHECK_EXPECTED(stream_infos);
- for (auto &stream_info : stream_infos.value()) {
+ TRY(auto stream_infos, core_op_metadata->get_output_stream_infos());
+ for (auto &stream_info : stream_infos) {
if (stream_name == stream_info.name) {
return std::move(stream_info);
}
Expected<size_t> Hef::Impl::get_number_of_input_streams(const std::string &net_group_name)
{
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
-
- auto input_stream_infos = core_op_metadata.value()->get_input_stream_infos();
- CHECK_EXPECTED(input_stream_infos);
- return input_stream_infos->size();
+ TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name));
+ TRY(const auto input_stream_infos, core_op_metadata->get_input_stream_infos());
+ return input_stream_infos.size();
}
Expected<size_t> Hef::Impl::get_number_of_output_streams(const std::string &net_group_name)
{
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
-
- auto output_stream_infos = core_op_metadata.value()->get_output_stream_infos();
- CHECK_EXPECTED(output_stream_infos);
- return output_stream_infos->size();
+ TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name));
+ TRY(const auto output_stream_infos, core_op_metadata->get_output_stream_infos());
+ return output_stream_infos.size();
}
static Expected<LayerType> get_layer_type(const ProtoHEFEdgeConnectionType &edge_connection_type)
bool transposed, const uint16_t context_index, const uint8_t network_index, LayerInfo &layer_info,
const SupportedFeatures &supported_features, const ProtoHEFHwArch &hef_arch, const bool is_part_of_mux_layer)
{
- auto format_order_exp = HailoRTDefaults::get_device_format_order(base_info.format());
- CHECK_EXPECTED_AS_STATUS(format_order_exp);
-
- auto format_oder = format_order_exp.release();
-
- auto layer_type = get_layer_type(edge_connection_type);
- CHECK_EXPECTED_AS_STATUS(layer_type);
- layer_info.type = layer_type.value();
+ TRY(layer_info.format.order, HailoRTDefaults::get_device_format_order(base_info.format()));
+ TRY(layer_info.type, get_layer_type(edge_connection_type));
// Parse host shape - parse hw shape after determining if core hw padding is supported
layer_info.shape = parse_layer_shape(base_info);
layer_info.hw_data_bytes = base_info.data_bytes();
// TODO: remove duplications with stream info parse
- layer_info.format.order = format_oder;
layer_info.format.flags = HAILO_FORMAT_FLAGS_NONE;
// The check network_group_proto.transposed_net() is for supporting backward compatability for old hefs
layer_info.shape.features = 1;
}
- auto type = HailoRTCommon::get_format_type(layer_info.hw_data_bytes);
- CHECK_EXPECTED_AS_STATUS(type);
- layer_info.format.type = type.value();
+ TRY(layer_info.format.type, HailoRTCommon::get_format_type(layer_info.hw_data_bytes));
- auto max_periph_bytes_from_hef =
- HefConfigurator::max_periph_bytes_value(DeviceBase::hef_arch_to_device_arch(static_cast<HEFHwArch>(hef_arch)));
- CHECK_EXPECTED_AS_STATUS(max_periph_bytes_from_hef);
+ TRY(const auto max_periph_bytes_from_hef,
+ HefConfigurator::max_periph_bytes_value(DeviceBase::hef_arch_to_device_arch(static_cast<HEFHwArch>(hef_arch))));
// TODO HRT-12099 - return invalid hef error when remove support for hefs with no max_shmifo size
- const auto max_periph_bytes = (0 == base_info.max_shmifo_size()) ? max_periph_bytes_from_hef.value() :
- std::min(max_periph_bytes_from_hef.value(), base_info.max_shmifo_size());
+ const auto max_periph_bytes = (0 == base_info.max_shmifo_size()) ? max_periph_bytes_from_hef :
+ std::min(max_periph_bytes_from_hef, base_info.max_shmifo_size());
// TODO HRT-12051: remove when is_core_hw_padding_supported function is removed
// Need to set layer_info.nn_stream_config.core_buffers_per_frame for condition in is_core_hw_padding_supported
const bool core_hw_padding_supported = is_part_of_mux_layer ? false :
HefConfigurator::is_core_hw_padding_supported(layer_info, max_periph_bytes,
supported_features.core_hw_padding_config_in_dfc);
- auto nn_stream_config = HefConfigurator::parse_nn_stream_config(base_info, core_hw_padding_supported,
- edge_connection_type);
- CHECK_EXPECTED_AS_STATUS(nn_stream_config, "Failed parse nn stream config");
- layer_info.nn_stream_config = nn_stream_config.release();
+ TRY(layer_info.nn_stream_config, HefConfigurator::parse_nn_stream_config(base_info, core_hw_padding_supported,
+ edge_connection_type), "Failed parse nn stream config");
layer_info.network_index = network_index;
layer_info.context_index = context_index;
layer_info.dma_engine_index = static_cast<uint8_t>(base_info.engine_id());
if (HAILO_FORMAT_ORDER_HAILO_NMS == layer_info.format.order) {
- auto expected_nms_info = parse_proto_nms_info(base_info.additional_info().nms_info(), supported_features.nms_burst_mode,
- hef_arch);
- CHECK_EXPECTED_AS_STATUS(expected_nms_info);
- layer_info.nms_info = expected_nms_info.release();
+ TRY(layer_info.nms_info, parse_proto_nms_info(base_info.additional_info().nms_info(), supported_features.nms_burst_mode,
+ hef_arch));
}
layer_info.max_shmifo_size = base_info.max_shmifo_size();
const ProtoHEFHwArch &hef_arch)
{
auto base_info = info.layer_info().edge_layer_base();
- auto format_order_exp = HailoRTDefaults::get_device_format_order(base_info.format());
- CHECK_EXPECTED_AS_STATUS(format_order_exp);
- layer_info.format.order = format_order_exp.release();
+ TRY(layer_info.format.order, HailoRTDefaults::get_device_format_order(base_info.format()));
layer_info.format.flags = HAILO_FORMAT_FLAGS_NONE;
layer_info.shape.height = static_cast<uint32_t>(info.nms_info().number_of_classes());
layer_info.hw_data_bytes = base_info.data_bytes();
- auto type = HailoRTCommon::get_format_type(layer_info.hw_data_bytes);
- CHECK_EXPECTED_AS_STATUS(type);
- layer_info.format.type = type.value();
+ TRY(layer_info.format.type, HailoRTCommon::get_format_type(layer_info.hw_data_bytes));
- auto expected_nms_info = parse_proto_nms_info(info.nms_info(), burst_mode_enabled, hef_arch);
- CHECK_EXPECTED_AS_STATUS(expected_nms_info);
- layer_info.nms_info = expected_nms_info.release();
+ TRY(layer_info.nms_info, parse_proto_nms_info(info.nms_info(), burst_mode_enabled, hef_arch));
if (HAILO_MAX_STREAM_NAME_SIZE < (info.layer_info().name().length() + 1)) {
LOGGER__ERROR("The edge layer '{}' has a too long name (max is HAILO_MAX_STREAM_NAME_SIZE)", info.layer_info().name());
hailo_stream_direction_t direction, const uint16_t context_index, const std::string &partial_network_name,
uint8_t network_index, LayerInfo &layer_info, const SupportedFeatures &supported_features, const ProtoHEFHwArch &hef_arch)
{
- auto layer_type = get_layer_type(edge_connection_type);
- CHECK_EXPECTED_AS_STATUS(layer_type);
- layer_info.type = layer_type.value();
+ TRY(layer_info.type, get_layer_type(edge_connection_type));
layer_info.direction = direction;
layer_info.shape.height = info.height();
layer_info.shape.features = info.features();
layer_info.hw_shape.features = info.features();
- auto format_order = convert_planes_format_to_hailo_format_order(info.planes_format());
- CHECK_EXPECTED_AS_STATUS(format_order);
- layer_info.format.order = format_order.release();
+ TRY(layer_info.format.order, convert_planes_format_to_hailo_format_order(info.planes_format()));
layer_info.format.flags = HAILO_FORMAT_FLAGS_NONE;
layer_info.quant_info = {}; // quant_info doesnt make any sense as this is a logical layer
layer_info.quant_infos = std::vector<hailo_quant_info_t>(1); // quant_info doesnt make any sense as this is a logical layer
return underlying_layer.hw_data_bytes == layer_info.hw_data_bytes;
}),
HAILO_INVALID_HEF, "Not all underlying layers of {} has the same format type", layer_info.name);
- auto type = HailoRTCommon::get_format_type(layer_info.hw_data_bytes);
- CHECK_EXPECTED_AS_STATUS(type);
- layer_info.format.type = type.value();
+ TRY(layer_info.format.type, HailoRTCommon::get_format_type(layer_info.hw_data_bytes));
return HAILO_SUCCESS;
}
ContextMetadata &context_metadata,
const ProtoHEFHwArch &hef_arch)
{
- auto layer_info = get_boundary_layer_info(core_op, context_index, layer, supported_features, hef_arch);
- CHECK_EXPECTED_AS_STATUS(layer_info);
-
- context_metadata.add_boundary_layer(layer_info.release());
+ TRY(auto layer_info, get_boundary_layer_info(core_op, context_index, layer, supported_features, hef_arch));
+ context_metadata.add_boundary_layer(layer_info);
return HAILO_SUCCESS;
}
const SupportedFeatures &supported_features,
ContextMetadata &context_metadata)
{
- auto layer_info = get_inter_context_layer_info(core_op, context_index, layer, supported_features);
- CHECK_EXPECTED_AS_STATUS(layer_info);
-
- context_metadata.add_inter_context_layer(layer_info.release());
+ TRY(auto layer_info, get_inter_context_layer_info(core_op, context_index, layer, supported_features));
+ context_metadata.add_inter_context_layer(layer_info);
return HAILO_SUCCESS;
}
const SupportedFeatures &supported_features,
ContextMetadata &context_metadata)
{
- auto layer_info = get_ddr_layer_info(core_op, context_index, layer, supported_features);
- CHECK_EXPECTED_AS_STATUS(layer_info);
+ TRY(auto layer_info, get_ddr_layer_info(core_op, context_index, layer, supported_features));
+ context_metadata.add_ddr_layer(layer_info);
+ return HAILO_SUCCESS;
+}
- context_metadata.add_ddr_layer(layer_info.release());
+hailo_status HefUtils::fill_cache_layers_info(
+ const ProtoHEFCoreOpMock &core_op,
+ const uint16_t context_index,
+ const ProtoHEFEdgeLayer &layer,
+ const SupportedFeatures &supported_features,
+ ContextMetadata &context_metadata)
+{
+ TRY(auto layer_info, get_cache_layer_info(core_op, context_index, layer, supported_features));
+ context_metadata.add_cache_layer(layer_info);
return HAILO_SUCCESS;
}
buffer_size += ccw_buffer.size();
}
- auto config_buffer = Buffer::create(buffer_size);
- CHECK_EXPECTED(config_buffer);
+ TRY(auto config_buffer, Buffer::create(buffer_size));
size_t current_offset = 0;
for (const auto &ccw_buffer : ccw_buffers) {
- assert(current_offset + ccw_buffer.size() <= config_buffer->size());
- memcpy(config_buffer->data() + current_offset, ccw_buffer.data(), ccw_buffer.size());
+ assert(current_offset + ccw_buffer.size() <= config_buffer.size());
+ memcpy(config_buffer.data() + current_offset, ccw_buffer.data(), ccw_buffer.size());
current_offset += ccw_buffer.size();
}
- return config_buffer.release();
+ return config_buffer;
}
static hailo_status merge_write_ccw_actions(
for (const auto &ccw_buffers_per_config_stream : ccw_buffers_per_config_streams) {
const auto config_stream_index = ccw_buffers_per_config_stream.first;
const auto &ccw_buffers = ccw_buffers_per_config_stream.second;
- auto config_buffer = build_config_buffer(ccw_buffers);
- CHECK_EXPECTED_AS_STATUS(config_buffer);
+ TRY(auto config_buffer, build_config_buffer(ccw_buffers));
- assert(config_buffer->size() < std::numeric_limits<uint32_t>::max());
- config_buffer_infos[config_stream_index].emplace_back(static_cast<uint32_t>(config_buffer->size()));
+ assert(config_buffer.size() < std::numeric_limits<uint32_t>::max());
+ config_buffer_infos[config_stream_index].bursts_sizes.emplace_back(static_cast<uint32_t>(config_buffer.size()));
const size_t total_ccw_burst = ccw_buffers.size();
- auto action = WriteDataCcwActionByBuffer::create(config_buffer.release(), config_stream_index, total_ccw_burst);
- CHECK_EXPECTED_AS_STATUS(action);
-
- actions.emplace_back(action.release());
+ TRY(auto action,
+ WriteDataCcwActionByBuffer::create(std::move(config_buffer), config_stream_index, total_ccw_burst));
+ actions.emplace_back(std::move(action));
}
return HAILO_SUCCESS;
static hailo_status build_write_ccw_actions(
std::vector<ContextSwitchConfigActionPtr> &actions,
ConfigBufferInfoMap &config_buffer_infos,
- const std::vector<const ProtoHEFActionWriteDataCcw *> &write_ccw_actions,
- std::shared_ptr<ShefFileHandle> shef_file_handle)
+ const std::vector<const ProtoHEFActionWriteDataCcwPtr*> &write_ccw_actions,
+ std::shared_ptr<SeekableBytesReader> hef_reader,
+ size_t ccws_offset)
{
- std::unordered_map<uint8_t, uint32_t> ccws_per_config_index;
+ std::unordered_map<uint8_t, std::vector<ccw_write_ptr_t>> ccw_write_ptrs_per_config_streams;
for (const auto *write_ccw_action : write_ccw_actions) {
CHECK(IS_FIT_IN_UINT8(write_ccw_action->cfg_channel_index()), HAILO_INVALID_HEF,
"Invalid cfg channel index");
const auto config_stream_index = static_cast<uint8_t>(write_ccw_action->cfg_channel_index());
- if (ccws_per_config_index.find(config_stream_index) == ccws_per_config_index.end()) {
- ccws_per_config_index[config_stream_index] = 0;
- }
- ccws_per_config_index[config_stream_index]++;
+ ccw_write_ptr_t ccw_write_ptr = {ccws_offset + write_ccw_action->offset(), write_ccw_action->size()}; // offset is relative to the start of the ccws
+ ccw_write_ptrs_per_config_streams[config_stream_index].emplace_back(ccw_write_ptr);
+ config_buffer_infos[config_stream_index].bursts_sizes.emplace_back(ccw_write_ptr.size);
}
- for (const auto *write_ccw_action : write_ccw_actions) {
- if (write_ccw_action->data().size() == 0) {
- continue;
- }
- const shef__ccw_offset_t *ccw_offset = reinterpret_cast<const shef__ccw_offset_t*>(write_ccw_action->data().data());
- const auto config_stream_index = static_cast<uint8_t>(write_ccw_action->cfg_channel_index());
- assert(BYTE_ORDER__htonl(ccw_offset->size) < std::numeric_limits<uint32_t>::max());
- config_buffer_infos[config_stream_index].emplace_back(static_cast<uint32_t>(BYTE_ORDER__htonl(ccw_offset->size)));
+ for (auto &ccw_write_ptrs_per_config_stream : ccw_write_ptrs_per_config_streams) {
+ const auto config_stream_index = ccw_write_ptrs_per_config_stream.first;
+ CHECK(IS_FIT_IN_UINT16(ccw_write_ptrs_per_config_stream.second.size()), HAILO_INVALID_HEF,
+ "Too many ccw burst {} (must fit in uint16)", ccw_write_ptrs_per_config_stream.second.size());
- const size_t total_ccw_burst = ccws_per_config_index[config_stream_index];
- auto action = WriteDataCcwAction::create(BYTE_ORDER__htonl(ccw_offset->offset), BYTE_ORDER__htonl(ccw_offset->size),
- config_stream_index, total_ccw_burst, shef_file_handle);
- CHECK_EXPECTED_AS_STATUS(action);
+ config_buffer_infos[config_stream_index].offset_from_hef_base = ccws_offset + ccw_write_ptrs_per_config_stream.second.begin()->offset;
- actions.emplace_back(action.release());
+ TRY(auto action,
+ WriteDataCcwAction::create(std::move(ccw_write_ptrs_per_config_stream.second), config_stream_index,
+ static_cast<uint16_t>(ccw_write_ptrs_per_config_stream.second.size()), hef_reader));
+ actions.emplace_back(std::move(action));
}
return HAILO_SUCCESS;
}
-static hailo_status parse_operation(std::vector<ContextSwitchConfigActionPtr> &actions,
- ConfigBufferInfoMap &config_buffer_infos,
- const ProtoHEFOperation &operation_proto,
- const SupportedFeatures &supported_features,
- std::shared_ptr<ShefFileHandle> shef_file_handle,
- bool &const_input_layer_found)
+static hailo_status parse_hef_v1_actions(const ProtoHEFOperation &operation_proto,
+ std::vector<ContextSwitchConfigActionPtr> &actions, ConfigBufferInfoMap &config_buffer_infos,
+ const SupportedFeatures &supported_features, bool &const_input_layer_found,
+ std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
{
- auto trigger_action = parse_trigger_action(operation_proto.trigger());
- CHECK_EXPECTED_AS_STATUS(trigger_action);
- actions.emplace_back(trigger_action.release());
+ std::vector<const ProtoHEFActionWriteDataCcwPtr*> current_write_ccw_ptr_actions;
+
+ for (int action_index = 0; action_index < operation_proto.actions_size(); action_index++) {
+ const auto &proto_action = operation_proto.actions(action_index);
+ CHECK(proto_action.action_case() != ProtoHEFAction::kWriteDataCcw, HAILO_INVALID_HEF, "WriteDataCcw action is not supported for hef version 1");
+
+ if (proto_action.action_case() == ProtoHEFAction::kWriteDataCcwPtr) {
+ // Keep in vector, parse later
+ current_write_ccw_ptr_actions.push_back(&proto_action.write_data_ccw_ptr());
- // If current_write_ccw_actions isn't empty, means we currently parsing a group of consecutive write ccw actions.
- // we will merge those actions into one write ccw per config channel.
+ const auto next_action_index = action_index + 1;
+ const bool is_last_ccw =
+ (next_action_index == operation_proto.actions_size()) ||
+ (operation_proto.actions(next_action_index).action_case() != ProtoHEFAction::kWriteDataCcwPtr);
+
+ if (is_last_ccw) {
+ assert(nullptr != hef_reader);
+ auto status = build_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_ptr_actions,
+ hef_reader, ccws_offset);
+ CHECK_SUCCESS(status);
+ current_write_ccw_ptr_actions.clear();
+ }
+ } else {
+ TRY(auto action, parse_action(proto_action, supported_features, const_input_layer_found));
+ actions.emplace_back(std::move(action));
+ }
+ }
+ assert(current_write_ccw_ptr_actions.empty());
+
+ return HAILO_SUCCESS;
+}
+
+static hailo_status parse_hef_v0_actions(const ProtoHEFOperation &operation_proto,
+ std::vector<ContextSwitchConfigActionPtr> &actions, ConfigBufferInfoMap &config_buffer_infos,
+ const SupportedFeatures &supported_features, bool &const_input_layer_found)
+{
std::vector<const ProtoHEFActionWriteDataCcw*> current_write_ccw_actions;
for (int action_index = 0; action_index < operation_proto.actions_size(); action_index++) {
const auto &proto_action = operation_proto.actions(action_index);
+ CHECK(proto_action.action_case() != ProtoHEFAction::kWriteDataCcwPtr, HAILO_INVALID_HEF, "kWriteDataCcwPtr action is not supported for hef version 0");
+
if (proto_action.action_case() == ProtoHEFAction::kWriteDataCcw) {
// Keep in vector, parse later
current_write_ccw_actions.push_back(&proto_action.write_data_ccw());
(next_action_index == operation_proto.actions_size()) ||
(operation_proto.actions(next_action_index).action_case() != ProtoHEFAction::kWriteDataCcw);
if (is_last_ccw) {
- if (nullptr != shef_file_handle) {
- auto status = build_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_actions, shef_file_handle);
- CHECK_SUCCESS(status);
- } else {
- auto status = merge_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_actions);
- CHECK_SUCCESS(status);
- }
+ auto status = merge_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_actions);
+ CHECK_SUCCESS(status);
current_write_ccw_actions.clear();
}
} else {
- auto action = parse_action(proto_action, supported_features, const_input_layer_found);
- CHECK_EXPECTED_AS_STATUS(action);
- actions.emplace_back(action.release());
+ TRY(auto action, parse_action(proto_action, supported_features, const_input_layer_found));
+ actions.emplace_back(std::move(action));
}
}
assert(current_write_ccw_actions.empty());
return HAILO_SUCCESS;
}
+static hailo_status parse_operation(std::vector<ContextSwitchConfigActionPtr> &actions,
+ ConfigBufferInfoMap &config_buffer_infos,
+ const ProtoHEFOperation &operation_proto,
+ const SupportedFeatures &supported_features,
+ bool &const_input_layer_found, uint32_t hef_version,
+ std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
+{
+ TRY(auto trigger_action, parse_trigger_action(operation_proto.trigger()));
+ actions.emplace_back(std::move(trigger_action));
+ hailo_status status = HAILO_UNINITIALIZED;
+
+ switch (hef_version)
+ {
+ case HEADER_VERSION_0:
+ status = parse_hef_v0_actions(operation_proto, actions, config_buffer_infos, supported_features, const_input_layer_found);
+ CHECK_SUCCESS(status);
+ break;
+ case HEADER_VERSION_1:
+ status = parse_hef_v1_actions(operation_proto, actions, config_buffer_infos, supported_features, const_input_layer_found, hef_reader, ccws_offset);
+ CHECK_SUCCESS(status);
+ break;
+ default:
+ LOGGER__ERROR("Unsupported hef version {}", hef_version);
+ return HAILO_INVALID_HEF;
+ }
+
+ return HAILO_SUCCESS;
+}
+
static Expected<ContextMetadata> parse_operations(
const google::protobuf::RepeatedPtrField<ProtoHEFOperation> &operations_proto,
- const SupportedFeatures &supported_features, std::shared_ptr<ShefFileHandle> shef_file_handle)
+ const SupportedFeatures &supported_features,
+ uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
{
std::vector<ContextSwitchConfigActionPtr> actions;
ConfigBufferInfoMap config_buffer_infos;
bool const_input_layer_found = false;
for (const auto &operation_proto : operations_proto) {
- auto status = parse_operation(actions, config_buffer_infos, operation_proto, supported_features, shef_file_handle,
- const_input_layer_found);
+ auto status = parse_operation(actions, config_buffer_infos, operation_proto, supported_features,
+ const_input_layer_found, hef_version, hef_reader, ccws_offset);
CHECK_SUCCESS_AS_EXPECTED(status);
}
}
Expected<ContextMetadata> HefUtils::parse_preliminary_context(const ProtoHEFPreliminaryConfig &preliminary_proto,
- const SupportedFeatures &supported_features, std::shared_ptr<ShefFileHandle> shef_file_handle)
+ const SupportedFeatures &supported_features, uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
{
- return parse_operations(preliminary_proto.operation(), supported_features, shef_file_handle);
+ return parse_operations(preliminary_proto.operation(), supported_features, hef_version, hef_reader, ccws_offset);
}
Expected<ContextMetadata> HefUtils::parse_single_dynamic_context(const ProtoHEFCoreOpMock &core_op,
const ProtoHEFContext &context_proto, uint16_t context_index, const SupportedFeatures &supported_features,
- const ProtoHEFHwArch &hef_arch, std::shared_ptr<ShefFileHandle> shef_file_handle)
+ const ProtoHEFHwArch &hef_arch, uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
{
- auto context_metadata_exp = parse_operations(context_proto.operations(), supported_features, shef_file_handle);
- CHECK_EXPECTED(context_metadata_exp);
- ContextMetadata context_metadata = context_metadata_exp.release();
+ TRY(auto context_metadata,
+ parse_operations(context_proto.operations(), supported_features, hef_version, hef_reader, ccws_offset));
for (const auto &edge_layer : context_proto.metadata().edge_layers()) {
if (ProtoHEFEdgeConnectionType::PROTO__EDGE_CONNECTION_TYPE__BOUNDARY ==
auto status = fill_ddr_layers_info(core_op, context_index, edge_layer,
supported_features, context_metadata);
CHECK_SUCCESS_AS_EXPECTED(status);
+ } else if (ProtoHEFEdgeConnectionType::PROTO__EDGE_CONNECTION_TYPE__CACHE ==
+ edge_layer.context_switch_info().edge_connection_type()) {
+ auto status = fill_cache_layers_info(core_op, context_index, edge_layer,
+ supported_features, context_metadata);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+ } else {
+ LOGGER__ERROR("Unsupported edge connection type given {}", edge_layer.context_switch_info().edge_connection_type());
+ return make_unexpected(HAILO_INVALID_HEF);
}
}
}
Expected<std::vector<ContextMetadata>> HefUtils::parse_dynamic_contexts(const ProtoHEFCoreOpMock &core_op, const SupportedFeatures &supported_features,
- const ProtoHEFHwArch &hef_arch, std::shared_ptr<ShefFileHandle> shef_file_handle)
+ const ProtoHEFHwArch &hef_arch, uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset)
{
std::vector<ContextMetadata> contexts_metadata;
for (uint16_t context_index = 0; context_index < core_op.contexts.size(); context_index++) {
auto &context_proto = core_op.contexts[context_index];
- auto context_metadata = parse_single_dynamic_context(core_op, context_proto, context_index, supported_features,
- hef_arch, shef_file_handle);
- CHECK_EXPECTED(context_metadata);
- contexts_metadata.emplace_back(context_metadata.release());
+ TRY(auto context_metadata, parse_single_dynamic_context(core_op, context_proto, context_index, supported_features,
+ hef_arch, hef_version, hef_reader, ccws_offset));
+ contexts_metadata.emplace_back(std::move(context_metadata));
}
const auto status = validate_unique_boundary_names(contexts_metadata);
}
case PROTO__HW_ARCH__HAILO15H:
case PROTO__HW_ARCH__HAILO15M:
+ case PROTO__HW_ARCH__HAILO10H:
case PROTO__HW_ARCH__GINGER:
case PROTO__HW_ARCH__LAVENDER:
case PROTO__HW_ARCH__PLUTO:
- // Second generation of hw NMS - included in hailo15 and pluto.
+ // Second generation of hw NMS - included in hailo15, hailo10 and pluto.
switch (nms_info.burst_type()) {
case PROTO__NMS_BURST_TYPE__H15_PER_CLASS:
return HAILO_BURST_TYPE_H15_PER_CLASS;
return HAILO_BURST_TYPE_H8_BBOX;
case PROTO__HW_ARCH__HAILO15H:
case PROTO__HW_ARCH__HAILO15M:
+ case PROTO__HW_ARCH__HAILO10H:
case PROTO__HW_ARCH__GINGER:
case PROTO__HW_ARCH__LAVENDER:
case PROTO__HW_ARCH__PLUTO:
if (burst_mode_enabled) {
nms_info.burst_size = static_cast<uint32_t>(proto_nms_info.burst_size());
-
- auto burst_type = get_nms_burst_mode(proto_nms_info, hef_arch);
- CHECK_EXPECTED(burst_type);
- nms_info.burst_type = *burst_type;
-
+ TRY(nms_info.burst_type, get_nms_burst_mode(proto_nms_info, hef_arch));
CHECK_AS_EXPECTED((nms_info.burst_size * nms_info.bbox_size) <= HailoRTCommon::MAX_NMS_BURST_SIZE,
HAILO_INVALID_HEF, "Invalid HEF, nms burst size {} larger than maximum burst size {}",
(nms_info.burst_size * nms_info.bbox_size), HailoRTCommon::MAX_NMS_BURST_SIZE);
} else {
// In case of bbox mode make burst size DEFAULT_NMS_NO_BURST_SIZE
nms_info.burst_size = DEFAULT_NMS_NO_BURST_SIZE;
- auto burst_type = get_nms_bbox_mode(proto_nms_info, hef_arch);
- CHECK_EXPECTED(burst_type);
- nms_info.burst_type = *burst_type;
+ TRY(nms_info.burst_type, get_nms_bbox_mode(proto_nms_info, hef_arch));
}
if (nms_info.chunks_per_frame == 0) {
HAILO_D2H_STREAM : HAILO_H2D_STREAM;
auto support_multi_networks = supported_features.multi_network_support;
auto network_index = static_cast<uint8_t>((support_multi_networks) ? layer.network_index() : 0);
- auto partial_network_name = HefUtils::get_partial_network_name_by_index(core_op, network_index, supported_features);
- CHECK_EXPECTED(partial_network_name);
+ TRY(const auto partial_network_name, HefUtils::get_partial_network_name_by_index(core_op, network_index, supported_features));
if (ProtoHEFEdgeLayerType::PROTO__EDGE_LAYER_TYPE__INFO == layer.edge_layer_type()) {
// TODO: return LayerInfo
auto status = fill_layer_info(layer.layer_info(), layer.context_switch_info().edge_connection_type(), core_op,
- direction, context_index, partial_network_name.value(), network_index, result, supported_features, hef_arch, false);
+ direction, context_index, partial_network_name, network_index, result, supported_features, hef_arch, false);
CHECK_SUCCESS_AS_EXPECTED(status);
} else if (ProtoHEFEdgeLayerType::PROTO__EDGE_LAYER_TYPE__MUX == layer.edge_layer_type()) {
// TODO: return LayerInfo
auto status = fill_mux_info(layer.layer_mux(), layer.context_switch_info().edge_connection_type(), core_op,
- direction, context_index, partial_network_name.value(), network_index, result, supported_features, hef_arch);
+ direction, context_index, partial_network_name, network_index, result, supported_features, hef_arch);
CHECK_SUCCESS_AS_EXPECTED(status);
} else if (ProtoHEFEdgeLayerType::PROTO__EDGE_LAYER_TYPE__PLANES == layer.edge_layer_type()) {
// TODO: return LayerInfo
auto status = fill_planes_info(layer.layer_planes(), layer.context_switch_info().edge_connection_type(), core_op,
- direction, context_index, partial_network_name.value(), network_index, result, supported_features, hef_arch);
+ direction, context_index, partial_network_name, network_index, result, supported_features, hef_arch);
CHECK_SUCCESS_AS_EXPECTED(status);
} else {
LOGGER__ERROR("Invalid layer type");
result.type = LayerType::INTER_CONTEXT;
auto support_multi_networks = supported_features.multi_network_support;
result.network_index = static_cast<uint8_t>((support_multi_networks) ? layer.network_index() : 0);
- auto partial_network_name = HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features);
- CHECK_EXPECTED(partial_network_name);
- result.network_name = HefUtils::get_network_name(core_op, partial_network_name.release());
+ TRY(const auto partial_network_name, HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features));
+ result.network_name = HefUtils::get_network_name(core_op, partial_network_name);
result.context_index = context_index;
result.name = layer.layer_info().name();
// Core hw padding is only supported on boundary layers
const bool CORE_HW_PADDING_NOT_SUPPORTED = false;
- auto nn_stream_config_exp = HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(),
- CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type());
- CHECK_EXPECTED(nn_stream_config_exp);
- result.nn_stream_config = nn_stream_config_exp.release();
+ TRY(result.nn_stream_config, HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(),
+ CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type()));
CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(layer.layer_info().edge_layer_base().sys_index()), HAILO_INVALID_HEF,
"Failed to parse HEF. Invalid sys_index: {}.", layer.layer_info().edge_layer_base().sys_index());
result.stream_index = static_cast<uint8_t>(layer.layer_info().edge_layer_base().sys_index());
result.hw_shape = parse_layer_hw_shape(layer.layer_info().edge_layer_base(), CORE_HW_PADDING_NOT_SUPPORTED);
result.hw_data_bytes = layer.layer_info().edge_layer_base().data_bytes();
- auto format_order_exp = HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format());
- CHECK_EXPECTED(format_order_exp);
- auto format_oder = format_order_exp.release();
- result.format.order = format_oder;
+ TRY(result.format.order,
+ HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format()));
result.format.flags = HAILO_FORMAT_FLAGS_NONE;
- auto type = HailoRTCommon::get_format_type(result.hw_data_bytes);
- CHECK_EXPECTED(type);
- result.format.type = type.value();
+ TRY(result.format.type, HailoRTCommon::get_format_type(result.hw_data_bytes));
result.direction = (ProtoHEFEdgeLayerDirection::PROTO__EDGE_LAYER_DIRECTION__DEVICE_TO_HOST ==
layer.direction()) ? HAILO_D2H_STREAM : HAILO_H2D_STREAM;
// HRT-7201 - The system supports one src and multiple dstinations. Right now we're saving only one dstination
CHECK_AS_EXPECTED(layer.context_switch_info().connected_contexts_size() >= 1, HAILO_INVALID_HEF,
"Inter context layer info must contain connected_context");
- auto connected_context = parse_connected_context_info(layer.context_switch_info().connected_contexts(0));
- CHECK_EXPECTED(connected_context);
- result.connected_context_info = connected_context.release();
+ TRY(result.connected_context_info,
+ parse_connected_context_info(layer.context_switch_info().connected_contexts(0)));
return result;
}
auto support_multi_networks = supported_features.multi_network_support;
result.network_index = static_cast<uint8_t>((support_multi_networks) ? layer.network_index() : 0);
- auto partial_network_name = HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features);
- CHECK_EXPECTED(partial_network_name);
- result.network_name = HefUtils::get_network_name(core_op, partial_network_name.release());
+ TRY(const auto partial_network_name,
+ HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features));
+ result.network_name = HefUtils::get_network_name(core_op, partial_network_name);
result.context_index = context_index;
result.name = layer.layer_info().name();
// Core hw padding is only supported on boundary layers
const bool CORE_HW_PADDING_NOT_SUPPORTED = false;
- auto nn_stream_config_exp = HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(),
- CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type());
- CHECK_EXPECTED(nn_stream_config_exp);
- result.nn_stream_config = nn_stream_config_exp.release();
+ TRY(result.nn_stream_config,
+ HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(),
+ CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type()));
CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(layer.layer_info().edge_layer_base().sys_index()), HAILO_INVALID_HEF,
"Failed to parse HEF. Invalid sys_index: {}.", layer.layer_info().edge_layer_base().sys_index());
result.stream_index = static_cast<uint8_t>(layer.layer_info().edge_layer_base().sys_index());
CHECK_AS_EXPECTED(layer.context_switch_info().connected_contexts_size() == 1, HAILO_INVALID_HEF,
"Only single connected context is supported on DDR channels");
- auto connected_context = parse_connected_context_info(layer.context_switch_info().connected_contexts(0));
- CHECK_EXPECTED(connected_context);
- CHECK_AS_EXPECTED(context_index == connected_context->context_index,
+ TRY(result.connected_context_info,
+ parse_connected_context_info(layer.context_switch_info().connected_contexts(0)));
+ CHECK_AS_EXPECTED(context_index == result.connected_context_info.context_index,
HAILO_INVALID_HEF, "for ddr layer, connected_context_index must be same to the edge layer's context");
- result.connected_context_info = connected_context.release();
result.direction = (ProtoHEFEdgeLayerDirection::PROTO__EDGE_LAYER_DIRECTION__DEVICE_TO_HOST ==
layer.direction()) ? HAILO_D2H_STREAM : HAILO_H2D_STREAM;
result.hw_shape = parse_layer_hw_shape(layer.layer_info().edge_layer_base(), CORE_HW_PADDING_NOT_SUPPORTED);
result.hw_data_bytes = layer.layer_info().edge_layer_base().data_bytes();
- auto format_order_exp = HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format());
- CHECK_EXPECTED(format_order_exp);
- auto format_oder = format_order_exp.release();
- result.format.order = format_oder;
+ TRY(result.format.order,
+ HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format()));
result.format.flags = HAILO_FORMAT_FLAGS_NONE;
- auto type = HailoRTCommon::get_format_type(result.hw_data_bytes);
- CHECK_EXPECTED(type);
- result.format.type = type.value();
+ TRY(result.format.type, HailoRTCommon::get_format_type(result.hw_data_bytes));
CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(layer.layer_info().edge_layer_base().core_buffers_per_frame()), HAILO_INVALID_HEF,
"Failed to parse HEF. Invalid core_buffers_per_frame: {}.", layer.layer_info().edge_layer_base().core_buffers_per_frame());
return result;
}
+Expected<LayerInfo> HefUtils::get_cache_layer_info(
+ const ProtoHEFCoreOpMock &core_op, const uint16_t context_index,
+ const ProtoHEFEdgeLayer &layer, const SupportedFeatures &supported_features)
+{
+ LayerInfo result{};
+ CHECK_AS_EXPECTED(PROTO__EDGE_LAYER_TYPE__INFO == layer.edge_layer_type(), HAILO_INVALID_HEF, "Inter-context layer can't be mux.");
+
+ result.type = LayerType::CACHE;
+ const auto support_multi_networks = supported_features.multi_network_support;
+ result.network_index = static_cast<uint8_t>((support_multi_networks) ? layer.network_index() : 0);
+ TRY(const auto partial_network_name, HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features));
+ result.network_name = HefUtils::get_network_name(core_op, partial_network_name);
+ result.context_index = context_index;
+ result.name = layer.layer_info().name();
+
+ // Core hw padding is only supported on boundary layers
+ const bool CORE_HW_PADDING_NOT_SUPPORTED = false;
+ TRY(result.nn_stream_config, HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(),
+ CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type()));
+ CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(layer.layer_info().edge_layer_base().sys_index()), HAILO_INVALID_HEF,
+ "Failed to parse HEF. Invalid sys_index: {}.", layer.layer_info().edge_layer_base().sys_index());
+ result.stream_index = static_cast<uint8_t>(layer.layer_info().edge_layer_base().sys_index());
+ CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(layer.layer_info().edge_layer_base().engine_id()), HAILO_INVALID_HEF,
+ "Failed to parse HEF. Invalid engine_id: {}.", layer.layer_info().edge_layer_base().engine_id());
+ result.dma_engine_index = static_cast<uint8_t>(layer.layer_info().edge_layer_base().engine_id());
+
+ result.max_shmifo_size = layer.layer_info().edge_layer_base().max_shmifo_size();
+
+ result.shape = parse_layer_shape(layer.layer_info().edge_layer_base());
+ result.hw_shape = parse_layer_hw_shape(layer.layer_info().edge_layer_base(), CORE_HW_PADDING_NOT_SUPPORTED);
+ result.hw_data_bytes = layer.layer_info().edge_layer_base().data_bytes();
+
+ TRY(result.format.order,
+ HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format()));
+ result.format.flags = HAILO_FORMAT_FLAGS_NONE;
+ TRY(result.format.type, HailoRTCommon::get_format_type(result.hw_data_bytes));
+
+ result.direction = (ProtoHEFEdgeLayerDirection::PROTO__EDGE_LAYER_DIRECTION__DEVICE_TO_HOST ==
+ layer.direction()) ? HAILO_D2H_STREAM : HAILO_H2D_STREAM;
+
+ // Negative cache_id means that the cache is not used
+ const int32_t cache_id = layer.context_switch_info().cache_id();
+ CHECK_AS_EXPECTED(cache_id >= 0, HAILO_INVALID_HEF, "Invalid cache_id: {}", cache_id);
+ result.cache_info.id = static_cast<uint32_t>(cache_id);
+
+ return result;
+}
+
Expected<std::string> HefUtils::get_partial_network_name_by_index(const ProtoHEFCoreOpMock &core_op, uint8_t network_index,
const SupportedFeatures &supported_features)
{
CHECK_AS_EXPECTED(data_length == expected_ccw_data_length, HAILO_INVALID_HEF,
"Invalid ccw buffer was parsed from HEF");
- auto data_buff = Buffer::create(reinterpret_cast<const uint8_t*>(ccw_buffer.data() + CCW_DATA_OFFSET), data_length);
- CHECK_EXPECTED(data_buff);
- write_memory_info.data = data_buff.release();
+ TRY(write_memory_info.data,
+ Buffer::create(reinterpret_cast<const uint8_t*>(ccw_buffer.data() + CCW_DATA_OFFSET), data_length));
return write_memory_info;
}
case ProtoHEFAction::kWriteData: {
WriteMemoryInfo write_memory_info = {};
write_memory_info.address = static_cast<uint32_t>(action.write_data().address());
- auto data_buff = Buffer::create(
- reinterpret_cast<const uint8_t*>(action.write_data().data().data()),
- action.write_data().data().length());
- CHECK_EXPECTED(data_buff);
- write_memory_info.data = data_buff.release();
+ TRY(write_memory_info.data,
+ Buffer::create(reinterpret_cast<const uint8_t*>(action.write_data().data().data()),
+ action.write_data().data().length()));
config_buffers.emplace_back(std::move(write_memory_info));
break;
}
case ProtoHEFAction::kWriteDataCcw: {
- auto config_buffer = parse_ccw_buffer(action.write_data_ccw().data()); // TODO: make this not supported in sHEF
- CHECK_EXPECTED(config_buffer);
- config_buffers.emplace_back(config_buffer.release());
+ TRY(auto config_buffer, parse_ccw_buffer(action.write_data_ccw().data())); // TODO: make this not supported in sHEF
+ config_buffers.emplace_back(std::move(config_buffer));
break;
}
case ProtoHEFAction::kDisableLcu: {
case ProtoHEFAction::kEnableLcu: {
WriteMemoryInfo write_memory_info = {};
write_memory_info.address = action.enable_lcu().lcu_enable_address();
- auto data_buff = Buffer::create(ENABLE_LCU_CONTROL_WORD, sizeof(ENABLE_LCU_CONTROL_WORD));
- CHECK_EXPECTED(data_buff);
- write_memory_info.data = data_buff.release();
+ TRY(write_memory_info.data,
+ Buffer::create(ENABLE_LCU_CONTROL_WORD, sizeof(ENABLE_LCU_CONTROL_WORD)));
config_buffers.emplace_back(std::move(write_memory_info));
break;
}
return m_header.hw_arch();
}
-std::shared_ptr<ShefFileHandle> Hef::Impl::get_shef_file_handle()
+std::shared_ptr<SeekableBytesReader> Hef::Impl::get_hef_reader()
{
- return m_shef_file_handle;
+ return m_hef_reader;
}
Expected<float64_t> Hef::Impl::get_bottleneck_fps(const std::string &net_group_name)
{
- auto core_op = get_core_op_by_net_group_name(net_group_name);
- CHECK_EXPECTED(core_op);
- return core_op.value()->network_group_metadata.bottleneck_fps();
+ TRY(const auto core_op, get_core_op_by_net_group_name(net_group_name));
+ return core_op->network_group_metadata.bottleneck_fps();
}
bool Hef::Impl::contains_ddr_layers(const ProtoHEFCoreOpMock& core_op)
Expected<std::string> Hef::Impl::get_vstream_name_from_original_name(const std::string &original_name,
const std::string &net_group_name)
{
- auto core_op = get_core_op_by_net_group_name(net_group_name);
- CHECK_EXPECTED(core_op);
+ TRY(const auto core_op, get_core_op_by_net_group_name(net_group_name));
std::string results;
- for (const auto &context : core_op.value()->contexts) {
+ for (const auto &context : core_op->contexts) {
for (const auto &layer_info : context.metadata().edge_layers()) {
if ((is_h2d_boundary_info_layer(layer_info)) || (is_d2h_boundary_info_layer(layer_info))) {
for (auto &name : layer_info.layer_info().original_names()) {
Expected<std::vector<std::string>> Hef::Impl::get_original_names_from_vstream_name(const std::string &vstream_name,
const std::string &net_group_name)
{
- auto copre_op = get_core_op_by_net_group_name(net_group_name);
- CHECK_EXPECTED(copre_op);
+ TRY(const auto copre_op, get_core_op_by_net_group_name(net_group_name));
std::vector<std::string> results;
- for (const auto &context : copre_op.value()->contexts) {
+ for (const auto &context : copre_op->contexts) {
for (const auto &layer_info : context.metadata().edge_layers()) {
if ((is_h2d_boundary_info_layer(layer_info)) || (is_d2h_boundary_info_layer(layer_info))) {
if (vstream_name == layer_info.layer_info().name()) {
{
NetworkGroupsParamsMap results;
for (const auto &name : pimpl->get_network_groups_names()) {
- auto params = create_configure_params(stream_interface, name);
- CHECK_EXPECTED(params);
- results.emplace(std::make_pair(name, params.release()));
+ TRY(auto params, create_configure_params(stream_interface, name));
+ results.emplace(std::make_pair(name, params));
}
return results;
}
{
NetworkGroupsParamsMap results;
for (const auto &name : pimpl->get_network_groups_names()) {
- auto params = create_configure_params_mipi_input(output_interface, mipi_params, name);
- CHECK_EXPECTED(params);
- results.emplace(std::make_pair(name, params.release()));
+ TRY(auto params, create_configure_params_mipi_input(output_interface, mipi_params, name));
+ results.emplace(std::make_pair(name, params));
}
return results;
}
std::string Hef::hash() const
{
- const auto &md5 = pimpl->md5();
+ const auto &hash = pimpl->get_hash_as_memview();
const bool LOWERCASE = false;
- return StringUtils::to_hex_string(md5, MD5_DIGEST_LENGTH, LOWERCASE);
+ return StringUtils::to_hex_string(hash.data(), hash.size(), LOWERCASE);
}
std::vector<std::string> Hef::Impl::get_network_groups_names()
Expected<std::vector<std::string>> Hef::Impl::get_stream_infos_description(const std::string &network_group_name, const std::string &network_name)
{
std::vector<std::string> infos_strings;
- auto input_stream_infos = get_input_stream_infos(network_group_name, network_name);
- CHECK_EXPECTED(input_stream_infos, "Failed to parse input stream infos");
- auto output_stream_infos = get_output_stream_infos(network_group_name, network_name);
- CHECK_EXPECTED(output_stream_infos, "Failed to parse output stream infos");
- infos_strings.reserve(input_stream_infos.value().size() + output_stream_infos.value().size());
+ TRY(const auto input_stream_infos,
+ get_input_stream_infos(network_group_name, network_name),
+ "Failed to parse input stream infos");
+ TRY(const auto output_stream_infos,
+ get_output_stream_infos(network_group_name, network_name),
+ "Failed to parse output stream infos");
+ infos_strings.reserve(input_stream_infos.size() + output_stream_infos.size());
std::string infos_string;
- for (const auto &stream_info : input_stream_infos.value()) {
+ for (const auto &stream_info : input_stream_infos) {
auto shape_str = get_shape_str(stream_info);
infos_string = "Input " + std::string(stream_info.name) + " " + shape_str + "\n";
infos_strings.emplace_back(infos_string);
}
- for (const auto &stream_info : output_stream_infos.value()) {
+ for (const auto &stream_info : output_stream_infos) {
auto shape_str = get_shape_str(stream_info);
infos_string = "Output " + std::string(stream_info.name) + " " + shape_str + "\n";
infos_strings.emplace_back(infos_string);
Expected<std::vector<std::string>> Hef::Impl::get_vstream_infos_description(const std::string &network_group_name, const std::string &network_name)
{
std::vector<std::string> infos_strings;
- auto input_vstream_infos = get_input_vstream_infos(network_group_name, network_name);
- CHECK_EXPECTED(input_vstream_infos, "Failed to parse input vstream infos");
- auto output_vstream_infos = get_output_vstream_infos(network_group_name, network_name);
- CHECK_EXPECTED(output_vstream_infos, "Failed to parse output stream infos");
- infos_strings.reserve(input_vstream_infos.value().size() + output_vstream_infos.value().size());
+ TRY(const auto input_vstream_infos,
+ get_input_vstream_infos(network_group_name, network_name),
+ "Failed to parse input vstream infos");
+ TRY(const auto output_vstream_infos,
+ get_output_vstream_infos(network_group_name, network_name),
+ "Failed to parse output vstream infos");
+ infos_strings.reserve(input_vstream_infos.size() + output_vstream_infos.size());
std::string infos_string;
- for (const auto &vstream_info : input_vstream_infos.value()) {
+ for (const auto &vstream_info : input_vstream_infos) {
auto shape_str = get_shape_str(vstream_info);
infos_string = "Input " + std::string(vstream_info.name) + " " + shape_str + "\n";
infos_strings.emplace_back(infos_string);
}
- for (const auto &vstream_info : output_vstream_infos.value()) {
+ for (const auto &vstream_info : output_vstream_infos) {
auto shape_str = get_shape_str(vstream_info);
infos_string = "Output " + std::string(vstream_info.name) + " " + shape_str + "\n";
infos_strings.emplace_back(infos_string);
Expected<std::string> Hef::get_description(bool stream_infos, bool vstream_infos) const
{
- auto arch = get_hef_device_arch();
- CHECK_EXPECTED(arch);
- return pimpl->get_description(stream_infos, vstream_infos, arch.value());
+ TRY(const auto arch, get_hef_device_arch());
+ return pimpl->get_description(stream_infos, vstream_infos, arch);
}
Expected<std::string> Hef::Impl::get_description(bool stream_infos, bool vstream_infos, hailo_device_architecture_t device_arch)
auto hef_arch_str = HailoRTCommon::get_device_arch_str(device_arch);
hef_infos += "Architecture HEF was compiled for: " + hef_arch_str + "\n";
- auto network_group_infos = get_network_groups_infos();
- CHECK_EXPECTED(network_group_infos);
- for (const auto &network_group_info : network_group_infos.release()) {
- auto core_op_metadata = get_core_op_metadata(network_group_info.name);
- CHECK_EXPECTED(core_op_metadata);
- const auto number_of_dynamic_contexts = core_op_metadata.value()->get_dynamic_contexts_count();
+ TRY(const auto network_group_infos, get_network_groups_infos());
+ for (const auto &network_group_info : network_group_infos) {
+ TRY(const auto core_op_metadata, get_core_op_metadata(network_group_info.name));
+ const auto number_of_dynamic_contexts = core_op_metadata->get_dynamic_contexts_count();
auto contexts_str = network_group_info.is_multi_context ?
"Multi Context - Number of contexts: " + std::to_string(number_of_dynamic_contexts) :
"Single Context";
hef_infos += "Network group name: " + std::string(network_group_info.name) + ", " + contexts_str + "\n";
- auto network_infos = get_network_infos(network_group_info.name);
- CHECK_EXPECTED(network_infos, "Failed to parse networks infos");
+ TRY(const auto network_infos, get_network_infos(network_group_info.name),
+ "Failed to parse networks infos");
- for (const auto &network_info : network_infos.value()) {
+ for (const auto &network_info : network_infos) {
hef_infos += add_tabs(1) + "Network name: " + network_info.name + "\n";
if (stream_infos) {
- auto stream_infos_strings = get_stream_infos_description(network_group_info.name, network_info.name);
- CHECK_EXPECTED(stream_infos_strings);
+ TRY(const auto stream_infos_strings,
+ get_stream_infos_description(network_group_info.name, network_info.name));
hef_infos += add_tabs(2) + "Stream infos:" + "\n";
- for (auto stream_info_string : stream_infos_strings.value()) {
+ for (auto stream_info_string : stream_infos_strings) {
hef_infos += add_tabs(3) + stream_info_string;
}
}
if (vstream_infos) {
- auto vstream_infos_strings = get_vstream_infos_description(network_group_info.name, network_info.name);
- CHECK_EXPECTED(vstream_infos_strings);
+ TRY(const auto vstream_infos_strings,
+ get_vstream_infos_description(network_group_info.name, network_info.name));
hef_infos += add_tabs(2) + "VStream infos:" + "\n";
- for (auto vstream_info_string : vstream_infos_strings.value()) {
+ for (auto vstream_info_string : vstream_infos_strings) {
hef_infos += add_tabs(3) + vstream_info_string;
}
- auto post_processes_infos_strings = get_post_processes_infos_description(network_group_info.name);
- CHECK_EXPECTED(post_processes_infos_strings);
+ TRY(const auto post_processes_infos_strings,
+ get_post_processes_infos_description(network_group_info.name));
/* Validating that there is a postprocess info. */
- if (post_processes_infos_strings->size() <= 0) {
+ if (post_processes_infos_strings.size() <= 0) {
continue;
}
hef_infos += add_tabs(3) + "Operation:" + "\n";
- for (auto post_process_info_string : post_processes_infos_strings.value()) {
+ for (auto post_process_info_string : post_processes_infos_strings) {
hef_infos += add_tabs(4) + post_process_info_string;
}
}
const std::string &name, bool /*unused*/, hailo_format_type_t format_type, uint32_t timeout_ms,
uint32_t queue_size)
{
- auto network_pair = pimpl->get_network_group_and_network_name(name);
- CHECK_EXPECTED(network_pair);
-
- return pimpl->make_input_vstream_params(network_pair.value().first, network_pair.value().second, format_type,
+ TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name));
+ return pimpl->make_input_vstream_params(network_pair.first, network_pair.second, format_type,
timeout_ms, queue_size);
}
const std::string &name, bool /*unused*/, hailo_format_type_t format_type, uint32_t timeout_ms,
uint32_t queue_size)
{
- auto network_pair = pimpl->get_network_group_and_network_name(name);
- CHECK_EXPECTED(network_pair);
-
- return pimpl->make_output_vstream_params(network_pair.value().first, network_pair.value().second, format_type,
+ TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name));
+ return pimpl->make_output_vstream_params(network_pair.first, network_pair.second, format_type,
timeout_ms, queue_size);
}
hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size)
{
CHECK(contains(m_network_group_metadata, net_group_name), HAILO_NOT_FOUND);
- auto input_vstream_infos = m_network_group_metadata.at(net_group_name).get_input_vstream_infos(network_name);
- CHECK_EXPECTED_AS_STATUS(input_vstream_infos);
+ TRY(const auto input_vstream_infos,
+ m_network_group_metadata.at(net_group_name).get_input_vstream_infos(network_name));
- return fill_missing_vstream_params_with_default(input_vstreams_params, input_vstream_infos.value(),
+ return fill_missing_vstream_params_with_default(input_vstreams_params, input_vstream_infos,
format_type, timeout_ms, queue_size);
}
hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size)
{
CHECK(contains(m_network_group_metadata, net_group_name), HAILO_NOT_FOUND);
- auto output_vstream_infos = m_network_group_metadata.at(net_group_name).get_output_vstream_infos(network_name);
- CHECK_EXPECTED_AS_STATUS(output_vstream_infos);
+ TRY(const auto output_vstream_infos,
+ m_network_group_metadata.at(net_group_name).get_output_vstream_infos(network_name));
- return fill_missing_vstream_params_with_default(output_vstream_params, output_vstream_infos.value(),
+ return fill_missing_vstream_params_with_default(output_vstream_params, output_vstream_infos,
format_type, timeout_ms, queue_size);
}
hailo_status Hef::Impl::fill_missing_vstream_params_with_default(std::map<std::string, hailo_vstream_params_t> &vstream_params,
- std::vector<hailo_vstream_info_t> &vstream_infos, hailo_format_type_t format_type, uint32_t timeout_ms,
+ const std::vector<hailo_vstream_info_t> &vstream_infos, hailo_format_type_t format_type, uint32_t timeout_ms,
uint32_t queue_size)
{
hailo_format_flags_t flags = HAILO_FORMAT_FLAGS_NONE;
Expected<ConfigureNetworkParams> Hef::Impl::create_configure_params(hailo_stream_interface_t stream_interface, const std::string &network_group_name)
{
auto params = HailoRTDefaults::get_configure_params();
- auto stream_params_by_name = create_stream_parameters_by_name(network_group_name, stream_interface);
- CHECK_EXPECTED(stream_params_by_name);
- params.stream_params_by_name = stream_params_by_name.release();
- auto network_params_by_name = create_network_parameters_by_name(network_group_name);
- CHECK_EXPECTED(network_params_by_name);
- params.network_params_by_name = network_params_by_name.release();
+ TRY(params.stream_params_by_name,
+ create_stream_parameters_by_name(network_group_name, stream_interface));
+ TRY(params.network_params_by_name,
+ create_network_parameters_by_name(network_group_name));
return params;
}
const hailo_mipi_input_stream_params_t &mipi_params, const std::string &network_group_name)
{
auto params = HailoRTDefaults::get_configure_params();
- auto stream_params_by_name = create_stream_parameters_by_name_mipi_input(network_group_name, output_interface, mipi_params);
- CHECK_EXPECTED(stream_params_by_name);
- params.stream_params_by_name = stream_params_by_name.release();
- auto network_params_by_name = create_network_parameters_by_name(network_group_name);
- CHECK_EXPECTED(network_params_by_name);
- params.network_params_by_name = network_params_by_name.release();
+ TRY(params.stream_params_by_name,
+ create_stream_parameters_by_name_mipi_input(network_group_name, output_interface, mipi_params));
+ TRY(params.network_params_by_name,
+ create_network_parameters_by_name(network_group_name));
return params;
}
Expected<std::map<std::string, hailo_stream_parameters_t>> Hef::create_stream_parameters_by_name(
const std::string &net_group_name, hailo_stream_interface_t stream_interface)
{
- auto network_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(network_group_name_pair);
- auto net_group_name_str = network_group_name_pair->first;
+ TRY(const auto network_group_name_pair,
+ pimpl->get_network_group_and_network_name(net_group_name));
+ const auto &net_group_name_str = network_group_name_pair.first;
return pimpl->create_stream_parameters_by_name(net_group_name_str, stream_interface);
}
Expected<std::map<std::string, hailo_stream_parameters_t>> Hef::Impl::create_stream_parameters_by_name(
const std::string &net_group_name, hailo_stream_interface_t stream_interface)
{
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
+ TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name));
std::map<std::string, hailo_stream_parameters_t> results;
- auto input_stream_infos = core_op_metadata.value()->get_input_stream_infos();
- CHECK_EXPECTED(input_stream_infos);
- for (auto &input_layer : input_stream_infos.value()) {
- auto params = HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_H2D_STREAM);
- CHECK_EXPECTED(params);
- results.emplace(std::make_pair(input_layer.name, params.release()));
+ TRY(const auto input_stream_infos,
+ core_op_metadata->get_input_stream_infos());
+ for (const auto &input_layer : input_stream_infos) {
+ TRY(auto params,
+ HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_H2D_STREAM));
+ results.emplace(std::make_pair(input_layer.name, params));
}
- auto output_stream_infos = core_op_metadata.value()->get_output_stream_infos();
- CHECK_EXPECTED(output_stream_infos);
- for (auto &output_layer : output_stream_infos.value()) {
- auto params = HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_D2H_STREAM);
- CHECK_EXPECTED(params);
- results.emplace(std::make_pair(output_layer.name, params.release()));
+ TRY(const auto output_stream_infos,
+ core_op_metadata->get_output_stream_infos());
+ for (const auto &output_layer : output_stream_infos) {
+ TRY(auto params,
+ HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_D2H_STREAM));
+ results.emplace(std::make_pair(output_layer.name, params));
}
return results;
Expected<std::map<std::string, hailo_network_parameters_t>> Hef::Impl::create_network_parameters_by_name(
const std::string &net_group_name)
{
- auto core_op = get_core_op_by_net_group_name(net_group_name);
- CHECK_EXPECTED(core_op);
-
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
+ TRY(const auto core_op, get_core_op_by_net_group_name(net_group_name));
+ TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name));
std::map<std::string, hailo_network_parameters_t> results;
- if (core_op_metadata.value()->supported_features().multi_network_support) {
- CHECK_AS_EXPECTED((core_op.value()->networks_names.size() != 0), HAILO_INTERNAL_FAILURE,
+ if (core_op_metadata->supported_features().multi_network_support) {
+ CHECK_AS_EXPECTED((core_op->networks_names.size() != 0), HAILO_INTERNAL_FAILURE,
"Hef support multiple networks, but no networks found in the proto");
- for (const auto &partial_network_name : core_op.value()->networks_names) {
+ for (const auto &partial_network_name : core_op->networks_names) {
auto network_name = HefUtils::get_network_name(net_group_name, partial_network_name);
auto params = HailoRTDefaults::get_network_parameters();
results.emplace(std::make_pair(network_name, params));
const std::string &net_group_name, hailo_stream_interface_t output_interface,
const hailo_mipi_input_stream_params_t &mipi_params)
{
- auto network_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name);
- CHECK_EXPECTED(network_group_name_pair);
- auto net_group_name_str = network_group_name_pair->first;
+ TRY(const auto network_group_name_pair,
+ pimpl->get_network_group_and_network_name(net_group_name));
+ const auto &net_group_name_str = network_group_name_pair.first;
return pimpl->create_stream_parameters_by_name_mipi_input(net_group_name_str, output_interface, mipi_params);
}
const std::string &net_group_name, hailo_stream_interface_t output_interface,
const hailo_mipi_input_stream_params_t &mipi_params)
{
- auto core_op_metadata = get_core_op_metadata(net_group_name);
- CHECK_EXPECTED(core_op_metadata);
+ TRY(const auto core_op_metadata,
+ get_core_op_metadata(net_group_name));
std::map<std::string, hailo_stream_parameters_t> results;
- auto input_stream_infos = core_op_metadata.value()->get_input_stream_infos();
- CHECK_EXPECTED(input_stream_infos);
- for (auto &input_layer : input_stream_infos.value()) {
+ TRY(const auto input_stream_infos,
+ core_op_metadata->get_input_stream_infos());
+ for (auto &input_layer : input_stream_infos) {
hailo_stream_parameters_t params = {};
params.direction = HAILO_H2D_STREAM;
params.stream_interface = HAILO_STREAM_INTERFACE_MIPI;
params.mipi_input_params = mipi_params;
results.emplace(std::make_pair(input_layer.name, params));
}
- auto output_stream_infos = core_op_metadata.value()->get_output_stream_infos();
- CHECK_EXPECTED(output_stream_infos);
- for (auto &output_layer : output_stream_infos.value()) {
- auto params = HailoRTDefaults::get_stream_parameters(output_interface, HAILO_D2H_STREAM);
- CHECK_EXPECTED(params);
- results.emplace(std::make_pair(output_layer.name, params.release()));
+ TRY(const auto output_stream_infos,
+ core_op_metadata->get_output_stream_infos());
+ for (auto &output_layer : output_stream_infos) {
+ TRY(auto params, HailoRTDefaults::get_stream_parameters(output_interface, HAILO_D2H_STREAM));
+ results.emplace(std::make_pair(output_layer.name, params));
}
return results;
#include "net_flow/ops/op.hpp"
#include "device_common/control_protocol.hpp"
+#include "common/file_utils.hpp"
+
#include "control_protocol.h"
#include <functional>
#include <bitset>
};
#pragma pack(push, 1)
+// TODO HRT-13921: change structure of hef header types
+
+typedef union {
+ struct {
+ uint32_t reserved;
+ MD5_SUM_t expected_md5;
+ } v0;
+ struct {
+ uint32_t crc;
+ uint64_t ccws_size;
+ uint32_t reserved;
+ } v1;
+} hef__header_distinct_t;
+
typedef struct {
uint32_t magic;
uint32_t version;
uint32_t hef_proto_size;
- uint32_t ccws_size;
- MD5_SUM_t expected_md5;
+ hef__header_distinct_t distinct;
} hef__header_t;
#pragma pack(pop)
-#pragma pack(push, 1)
-typedef struct {
- uint32_t offset;
- uint32_t size;
-} shef__ccw_offset_t;
-#pragma pack(pop)
+static const size_t HEF_COMMON_SIZE = sizeof(hef__header_t) - sizeof(hef__header_distinct_t);
+static const size_t HEF_HEADER_SIZE_V0 = HEF_COMMON_SIZE + sizeof(hef__header_distinct_t::v0);
+static const size_t HEF_HEADER_SIZE_V1 = HEF_COMMON_SIZE + sizeof(hef__header_distinct_t::v1);
typedef enum {
HEF__FORMAT__TF_RGB = 0,
HAILO_NET_FLOW_OP_TYPE_MAX_ENUM = HAILO_MAX_ENUM
} hailo_net_flow_op_type_t;
+#define HEADER_MAGIC (0x01484546)
+#define HEADER_VERSION_0 (0)
+#define HEADER_VERSION_1 (1)
+
const static uint32_t SUPPORTED_EXTENSIONS_BITSET_SIZE = 1000;
static const std::vector<ProtoHEFExtensionType> SUPPORTED_EXTENSIONS = {
ABBALE,
class VdmaDevice;
class HailoRTDriver;
-class ShefFileHandle final
-{
-public:
- ShefFileHandle(const std::string &hef_path, uint32_t ccws_buffer_offset);
- hailo_status open();
- Expected<Buffer> read(uint32_t offset, size_t size);
- hailo_status close();
-
-private:
- std::string m_hef_path;
- std::ifstream m_hef_file;
- uint32_t m_ccws_buffer_offset;
-};
-
class Hef::Impl final
{
public:
- static const uint32_t HEADER_MAGIC = 0x01484546;
- static const uint32_t HEADER_VERSION_0 = 0; // Old HEF
- static const uint32_t HEADER_VERSION_1 = 1; // New HEF (SHEF)
static Expected<Impl> create(const std::string &hef_path);
static Expected<Impl> create(const MemoryView &hef_buffer);
Expected<size_t> get_number_of_input_streams(const std::string &net_group_name="");
Expected<size_t> get_number_of_output_streams(const std::string &net_group_name="");
ProtoHEFHwArch get_device_arch();
- std::shared_ptr<ShefFileHandle> get_shef_file_handle();
+ std::shared_ptr<SeekableBytesReader> get_hef_reader();
Expected<float64_t> get_bottleneck_fps(const std::string &net_group_name="");
static bool contains_ddr_layers(const ProtoHEFCoreOpMock &core_op);
static hailo_status validate_core_op_unique_layer_names(const ProtoHEFCoreOpMock &core_op);
const std::string &network_name, std::map<std::string, hailo_vstream_params_t> &output_vstream_params,
hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size);
static hailo_status fill_missing_vstream_params_with_default(std::map<std::string, hailo_vstream_params_t> &vstream_params,
- std::vector<hailo_vstream_info_t> &name_to_format_info, hailo_format_type_t format_type, uint32_t timeout_ms,
+ const std::vector<hailo_vstream_info_t> &vstream_infos, hailo_format_type_t format_type, uint32_t timeout_ms,
uint32_t queue_size);
// Also adds information to CoreOpMetadata
// TODO: When supporting multiple core ops in same netflow - Change metadata param to a map of core_ops_metadata.
Expected<std::string> get_description(bool stream_infos, bool vstream_infos, hailo_device_architecture_t device_arch);
- const MD5_SUM_t &md5() const
+ const MemoryView get_hash_as_memview() const
{
- return m_md5;
+ if (HEADER_VERSION_0 == m_hef_version){
+ return MemoryView::create_const(m_md5, sizeof(m_md5));
+ } else { // HEADER_VERSION_1
+ return MemoryView::create_const(&m_crc, sizeof(m_crc));
+ }
}
static hailo_status update_network_batch_size(ConfigureNetworkParams &network_group_config_params)
hailo_status parse_hef_file(const std::string &hef_path);
hailo_status parse_hef_memview(const MemoryView &hef_memview);
+ hailo_status parse_hef_memview_internal(const size_t proto_size, const uint8_t *proto_buffer, const uint32_t hef_version,
+ std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset);
+ Expected<hef__header_t> parse_hef_header_before_distinct(std::shared_ptr<SeekableBytesReader> hef_reader);
+ hailo_status fill_v1_hef_header(hef__header_t &hef_header, std::shared_ptr<SeekableBytesReader> hef_reader);
+ hailo_status fill_core_ops_and_networks_metadata(uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset);
hailo_status transfer_protobuf_field_ownership(ProtoHEFHef &hef_message);
void fill_core_ops();
- hailo_status fill_networks_metadata();
+ hailo_status fill_networks_metadata(uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset);
void fill_extensions_bitset();
void init_md5(MD5_SUM_t &calculated_md5);
+ void init_crc(uint32_t crc_32);
+ void init_hef_version(uint32_t version);
static bool check_hef_extension(const ProtoHEFExtensionType &extension, const ProtoHEFHeader &header,
const std::vector<ProtoHEFExtension> &hef_extensions, const ProtoHEFIncludedFeatures &included_features);
hailo_status validate_hef_extensions();
static hailo_status validate_hef_header(const hef__header_t &header, MD5_SUM_t &calculated_md5, size_t proto_size);
+ static hailo_status validate_hef_header(const hef__header_t &header, const uint32_t &crc_32, size_t hef_file_residue_size);
+
Expected<std::map<std::string, hailo_format_t>> get_inputs_vstream_names_and_format_info(
const std::string &net_group_name, const std::string &network_name);
static Expected<std::string> get_vstream_name_from_original_name_mux(const std::string &original_name, const ProtoHefEdge &layer);
static Expected<std::vector<std::string>> get_original_names_from_vstream_name_mux(const std::string &vstream_name, const ProtoHefEdge &layer);
- Expected<CoreOpMetadataPtr> create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op, const std::vector<std::string> &sorted_network_names); // TODO: Remove sorted_network_names
+ Expected<CoreOpMetadataPtr> create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op, const std::vector<std::string> &sorted_network_names,
+ uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset); // TODO: Remove sorted_network_names
Expected<std::vector<std::string>> get_stream_infos_description(const std::string &network_group_name, const std::string &network_name);
Expected<std::vector<std::string>> get_vstream_infos_description(const std::string &network_group_name, const std::string &network_name);
Expected<std::vector<std::string>> get_post_processes_infos_description(const std::string &network_group_name);
std::vector<ProtoHEFExtension> m_hef_extensions;
std::vector<ProtoHEFOptionalExtension> m_hef_optional_extensions;
std::bitset<SUPPORTED_EXTENSIONS_BITSET_SIZE> m_supported_extensions_bitset;
+ uint32_t m_hef_version;
MD5_SUM_t m_md5;
- std::shared_ptr<ShefFileHandle> m_shef_file_handle;
+ uint32_t m_crc;
+ std::shared_ptr<SeekableBytesReader> m_hef_reader;
#ifdef HAILO_SUPPORT_MULTI_PROCESS
Buffer m_hef_buffer;
const std::vector<LayerInfo> &context_ddr_input_layers,
const std::vector<LayerInfo> &context_ddr_output_layers,
const uint16_t context_index);
+ static hailo_status fill_cache_layers_info(
+ const ProtoHEFCoreOpMock &core_op,
+ const uint16_t context_index,
+ const ProtoHEFEdgeLayer &layer,
+ const SupportedFeatures &supported_features,
+ ContextMetadata &context_metadata);
+ static Expected<LayerInfo> get_cache_layer_info(
+ const ProtoHEFCoreOpMock &core_op, const uint16_t context_index,
+ const ProtoHEFEdgeLayer &layer, const SupportedFeatures &supported_features);
static Expected<ContextMetadata> parse_preliminary_context(const ProtoHEFPreliminaryConfig &preliminary_proto,
- const SupportedFeatures &supported_features, std::shared_ptr<ShefFileHandle> shef_file_handle);
+ const SupportedFeatures &supported_features, const uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset);
static Expected<ContextMetadata> parse_single_dynamic_context(const ProtoHEFCoreOpMock &core_op,
const ProtoHEFContext &context_proto, uint16_t context_index, const SupportedFeatures &supported_features,
- const ProtoHEFHwArch &hef_arch, std::shared_ptr<ShefFileHandle> shef_file_handle);
+ const ProtoHEFHwArch &hef_arch, const uint32_t hef_version, std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset);
static Expected<std::vector<ContextMetadata>> parse_dynamic_contexts(const ProtoHEFCoreOpMock &core_op,
- const SupportedFeatures &supported_features, const ProtoHEFHwArch &hef_arch, std::shared_ptr<ShefFileHandle> shef_file_handle);
+ const SupportedFeatures &supported_features, const ProtoHEFHwArch &hef_arch, const uint32_t hef_version,
+ std::shared_ptr<SeekableBytesReader> hef_reader, size_t ccws_offset);
static Expected<hailo_nms_info_t> parse_proto_nms_info(const ProtoHEFNmsInfo &proto_nms_info,
const bool burst_mode_enabled, const ProtoHEFHwArch &hef_arch);
static Expected<LayerInfo> get_boundary_layer_info(const ProtoHEFCoreOpMock &core_op,
BOUNDARY = 1,
INTER_CONTEXT = 2,
DDR = 3,
- CFG = 4
+ CFG = 4,
+ CACHE = 5
};
struct BufferIndices {
uint16_t min_buffered_rows;
};
+struct CacheInfo {
+ uint32_t id;
+ uint32_t size;
+};
+
struct LayerInfo {
LayerType type = LayerType::NOT_SET;
hailo_stream_direction_t direction;
// Context switch info TODO: we should use std::optional for this structures (or implement our self).
ConnectedContextInfo connected_context_info;
DdrInfo ddr_info;
+ CacheInfo cache_info;
};
// LayerIdentifier = <LayerType, hailo_stream_direction_t, layer_name, stream_index>
${CMAKE_CURRENT_SOURCE_DIR}/ops/softmax_post_process.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ops/yolov5_seg_post_process.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ops/yolov8_post_process.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/ops/yolov8_bbox_only_post_process.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pipeline/pipeline.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pipeline/pipeline_internal.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pipeline/async_pipeline_builder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pipeline/async_infer_runner.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pipeline/infer_model.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/infer_model_hrpc_client.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/configured_infer_model_hrpc_client.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pipeline/vstream_builder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pipeline/vstream.cpp
#include "hailo/hailort_common.hpp"
#include "hailo/hailort_defaults.hpp"
+#if defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4244 4267 4127)
+#else
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wconversion"
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#pragma GCC diagnostic ignored "-Wclass-memaccess"
+#endif
+#include <Eigen/Dense>
+#if defined(_MSC_VER)
+#pragma warning(pop)
+#else
+#pragma GCC diagnostic pop
+#endif
+
#include "common/utils.hpp"
#include <limits>
hailo_status SoftmaxPostProcessOp::softmax(float32_t *src, float32_t *dst, size_t num_of_elements)
{
- // In order to avoid overflows, we will perform the following:
- // We find the maximal value and then we substract it from all of the values.
- // This will preserve the original softmax values + prevent overflows
- float32_t max_val = *std::max_element(src, src + num_of_elements);
- float32_t sum_exp = 0; // denominator
- for (uint32_t c = 0; c < num_of_elements; c++) {
- auto ¤t_value = *(src + c);
- current_value -= max_val; // This step preserves the original softmax values + prevent overflows
- current_value = std::exp(static_cast<float32_t>(current_value)); // Set src[c] to e^(src[c]) so that we only calculate it once
- sum_exp += current_value;
- }
- for (uint32_t c = 0; c < num_of_elements; c++) {
- const auto ¤t_value = *(src + c);
- dst[c] = static_cast<float32_t>(current_value / sum_exp);
- }
+ // Create an Eigen Matrix view for src and dst
+ Eigen::Map<Eigen::Matrix<float, Eigen::Dynamic, 1>> src_eigen(src, num_of_elements);
+ Eigen::Map<Eigen::Matrix<float, Eigen::Dynamic, 1>> dst_eigen(dst, num_of_elements);
+
+ src_eigen = src_eigen.array().exp(); // Compute exponentials in place
+
+ assert(!src_eigen.hasNaN()); // Checks if any element in the array is NaN (Not a Number)
+
+ float sum_exp = src_eigen.sum(); // Compute the sum of exponentials
+
+ assert(0.0f != sum_exp); // Checks for division by zero
+ dst_eigen = src_eigen / sum_exp; // Perform softmax operation
+
return HAILO_SUCCESS;
}
Expected<hailo_vstream_info_t> Yolov5BboxOnlyOpMetadata::get_output_vstream_info()
{
- auto vstream_info = NmsOpMetadata::get_output_vstream_info();
- CHECK_EXPECTED(vstream_info);
-
- vstream_info->shape = m_outputs_metadata.begin()->second.shape;
- return vstream_info.release();
+ TRY(auto vstream_info, NmsOpMetadata::get_output_vstream_info());
+ vstream_info.shape = m_outputs_metadata.begin()->second.shape;
+ return vstream_info;
}
hailo_status Yolov5BboxOnlyOpMetadata::validate_format_info()
const uint32_t W_OFFSET, const uint32_t H_OFFSET, hailo_quant_info_t quant_info, uint32_t anchor,
const std::vector<int> &layer_anchors, uint32_t col, uint32_t row, hailo_3d_image_shape_t shape)
{
- auto tx = dequantize_and_sigmoid<DstType, SrcType>(data[entry_idx + X_OFFSET], quant_info);
- auto ty = dequantize_and_sigmoid<DstType, SrcType>(data[entry_idx + Y_OFFSET], quant_info);
- auto tw = dequantize_and_sigmoid<DstType, SrcType>(data[entry_idx + W_OFFSET], quant_info);
- auto th = dequantize_and_sigmoid<DstType, SrcType>(data[entry_idx + H_OFFSET], quant_info);
- return decode(tx, ty, tw, th, layer_anchors[anchor * 2], layer_anchors[anchor * 2 + 1], col, row,
- shape.width, shape.height);
+ auto tx = dequantize_and_sigmoid<DstType, SrcType>(data[entry_idx + X_OFFSET], quant_info);
+ auto ty = dequantize_and_sigmoid<DstType, SrcType>(data[entry_idx + Y_OFFSET], quant_info);
+ auto tw = dequantize_and_sigmoid<DstType, SrcType>(data[entry_idx + W_OFFSET], quant_info);
+ auto th = dequantize_and_sigmoid<DstType, SrcType>(data[entry_idx + H_OFFSET], quant_info);
+ return decode(tx, ty, tw, th, layer_anchors[anchor * 2], layer_anchors[anchor * 2 + 1], col, row,
+ shape.width, shape.height);
}
template<typename DstType = float32_t, typename SrcType>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
#pragma GCC diagnostic ignored "-Wunused-parameter"
+#pragma GCC diagnostic ignored "-Wclass-memaccess"
#endif
#define STB_IMAGE_RESIZE_IMPLEMENTATION
#include "stb_image_resize.h"
Expected<hailo_vstream_info_t> Yolov5SegOpMetadata::get_output_vstream_info()
{
- auto vstream_info = NmsOpMetadata::get_output_vstream_info();
- CHECK_EXPECTED(vstream_info);
-
- vstream_info->nms_shape.max_accumulated_mask_size = m_yolo_seg_config.max_accumulated_mask_size;
- return vstream_info.release();
+ TRY(auto vstream_info, NmsOpMetadata::get_output_vstream_info());
+ vstream_info.nms_shape.max_accumulated_mask_size = m_yolo_seg_config.max_accumulated_mask_size;
+ return vstream_info;
}
Expected<std::shared_ptr<Op>> Yolov5SegPostProcess::create(std::shared_ptr<Yolov5SegOpMetadata> metadata)
assert(contains(metadata->inputs_metadata(), metadata->yolov5seg_config().proto_layer_name));
auto proto_layer_metadata = metadata->inputs_metadata().at(metadata->yolov5seg_config().proto_layer_name);
auto transformed_proto_layer_frame_size = HailoRTCommon::get_shape_size(proto_layer_metadata.shape) * sizeof(float32_t);
- auto transformed_proto_buffer = Buffer::create(transformed_proto_layer_frame_size);
- CHECK_EXPECTED(transformed_proto_buffer);
- auto mask_mult_result_buffer = Buffer::create(proto_layer_metadata.shape.height * proto_layer_metadata.shape.width * sizeof(float32_t));
- CHECK_EXPECTED(mask_mult_result_buffer);
+ TRY(auto transformed_proto_buffer,
+ Buffer::create(transformed_proto_layer_frame_size));
+ TRY(auto mask_mult_result_buffer,
+ Buffer::create(proto_layer_metadata.shape.height * proto_layer_metadata.shape.width * sizeof(float32_t)));
- auto image_size = static_cast<uint32_t>(metadata->yolov5_config().image_width) * static_cast<uint32_t>(metadata->yolov5_config().image_height);
- auto resized_buffer = Buffer::create(image_size * sizeof(float32_t));
- CHECK_EXPECTED(resized_buffer);
+ const auto image_size = static_cast<uint32_t>(metadata->yolov5_config().image_width) * static_cast<uint32_t>(metadata->yolov5_config().image_height);
+ TRY(auto resized_buffer, Buffer::create(image_size * sizeof(float32_t)));
auto op = std::shared_ptr<Yolov5SegPostProcess>(new (std::nothrow) Yolov5SegPostProcess(std::move(metadata),
- mask_mult_result_buffer.release(), resized_buffer.release(), transformed_proto_buffer.release()));
+ std::move(mask_mult_result_buffer), std::move(resized_buffer), std::move(transformed_proto_buffer)));
CHECK_NOT_NULL_AS_EXPECTED(op, HAILO_OUT_OF_HOST_MEMORY);
return std::shared_ptr<Op>(std::move(op));
status = copied_bytes_amount.status();
break;
}
- CHECK_EXPECTED_AS_STATUS(copied_bytes_amount);
+ CHECK_EXPECTED_AS_STATUS(copied_bytes_amount); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
buffer_offset += copied_bytes_amount.release();
detections_count++;
}
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file yolov8_bbox_only_post_process.cpp
+ * @brief YOLOv8 bbox only post process
+ *
+ **/
+
+#include "net_flow/ops/yolov8_bbox_only_post_process.hpp"
+
+namespace hailort
+{
+namespace net_flow
+{
+
+Expected<std::shared_ptr<Op>> YOLOv8BboxOnlyPostProcessOp::create(std::shared_ptr<Yolov8BboxOnlyOpMetadata> metadata)
+{
+ auto status = metadata->validate_format_info();
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ auto op = std::shared_ptr<YOLOv8BboxOnlyPostProcessOp>(new (std::nothrow) YOLOv8BboxOnlyPostProcessOp(metadata));
+ CHECK_AS_EXPECTED(op != nullptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return std::shared_ptr<Op>(std::move(op));
+}
+
+Expected<hailo_vstream_info_t> Yolov8BboxOnlyOpMetadata::get_output_vstream_info()
+{
+ TRY(auto vstream_info, NmsOpMetadata::get_output_vstream_info());
+
+ vstream_info.shape = m_outputs_metadata.begin()->second.shape;
+ return vstream_info;
+}
+
+hailo_status Yolov8BboxOnlyOpMetadata::validate_format_info()
+{
+ for (const auto& output_metadata : m_outputs_metadata) {
+
+ CHECK(HAILO_FORMAT_TYPE_FLOAT32 == output_metadata.second.format.type, HAILO_INVALID_ARGUMENT, "The given output format type {} is not supported, "
+ "should be HAILO_FORMAT_TYPE_FLOAT32", HailoRTCommon::get_format_type_str(output_metadata.second.format.type));
+
+ CHECK(HAILO_FORMAT_ORDER_NHWC == output_metadata.second.format.order, HAILO_INVALID_ARGUMENT, "The given output format order {} is not supported, "
+ "should be HAILO_FORMAT_ORDER_NHWC", HailoRTCommon::get_format_order_str(output_metadata.second.format.order));
+
+ CHECK(!(HAILO_FORMAT_FLAGS_TRANSPOSED & output_metadata.second.format.flags), HAILO_INVALID_ARGUMENT, "Output {} is marked as transposed, which is not supported for this model.",
+ output_metadata.first);
+ CHECK(!(HAILO_FORMAT_FLAGS_HOST_ARGMAX & output_metadata.second.format.flags), HAILO_INVALID_ARGUMENT, "Output {} is marked as argmax, which is not supported for this model.",
+ output_metadata.first);
+ }
+
+ assert(1 <= m_inputs_metadata.size());
+ const hailo_format_type_t& first_input_type = m_inputs_metadata.begin()->second.format.type;
+ for (const auto& input_metadata : m_inputs_metadata) {
+ CHECK(HAILO_FORMAT_ORDER_NHCW == input_metadata.second.format.order, HAILO_INVALID_ARGUMENT, "The given input format order {} is not supported, "
+ "should be HAILO_FORMAT_ORDER_NHCW", HailoRTCommon::get_format_order_str(input_metadata.second.format.order));
+
+ CHECK((HAILO_FORMAT_TYPE_UINT8 == input_metadata.second.format.type) ||
+ (HAILO_FORMAT_TYPE_UINT16 == input_metadata.second.format.type),
+ HAILO_INVALID_ARGUMENT, "The given input format type {} is not supported, should be HAILO_FORMAT_TYPE_UINT8 or HAILO_FORMAT_TYPE_UINT16",
+ HailoRTCommon::get_format_type_str(input_metadata.second.format.type));
+
+ CHECK(input_metadata.second.format.type == first_input_type, HAILO_INVALID_ARGUMENT,"All inputs format type should be the same");
+ }
+
+ return HAILO_SUCCESS;
+}
+
+std::string Yolov8BboxOnlyOpMetadata::get_op_description()
+{
+ auto nms_config_info = fmt::format("Classes: {}",
+ nms_config().number_of_classes);
+ auto config_info = fmt::format("Op {}, Name: {}, {}, Image height: {:d}, Image width: {:d}",
+ OpMetadata::get_operation_type_str(m_type), m_name, nms_config_info,
+ static_cast<int>(m_yolov8_config.image_height), static_cast<int>(m_yolov8_config.image_width));
+ return config_info;
+}
+
+Expected<std::shared_ptr<OpMetadata>> Yolov8BboxOnlyOpMetadata::create(const std::unordered_map<std::string, BufferMetaData> &inputs_metadata,
+ const std::unordered_map<std::string, BufferMetaData> &outputs_metadata,
+ const NmsPostProcessConfig &nms_post_process_config,
+ const Yolov8PostProcessConfig &yolov8_post_process_config,
+ const std::string &network_name)
+{
+ auto op_metadata = std::shared_ptr<Yolov8BboxOnlyOpMetadata>(new (std::nothrow) Yolov8BboxOnlyOpMetadata(inputs_metadata, outputs_metadata,
+ nms_post_process_config, yolov8_post_process_config, network_name));
+ CHECK_AS_EXPECTED(op_metadata != nullptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ auto status = op_metadata->validate_params();
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ return std::shared_ptr<OpMetadata>(std::move(op_metadata));
+}
+
+hailo_status YOLOv8BboxOnlyPostProcessOp::execute(const std::map<std::string, MemoryView> &inputs, std::map<std::string, MemoryView> &outputs)
+{
+ const auto &yolov8_config = m_metadata->yolov8_config();
+ const auto &inputs_metadata = m_metadata->inputs_metadata();
+
+ auto dst_ptr = (float32_t*)outputs.begin()->second.data();
+
+ size_t next_bbox_output_offset = 0;
+ for (const auto ®_to_cls_name : yolov8_config.reg_to_cls_inputs) {
+ hailo_status status = HAILO_UNINITIALIZED;
+ assert(contains(inputs, reg_to_cls_name.cls));
+ assert(contains(inputs, reg_to_cls_name.reg));
+
+ auto &input_metadata = inputs_metadata.at(reg_to_cls_name.reg);
+
+ if (HAILO_FORMAT_TYPE_UINT8 == input_metadata.format.type) {
+ status = add_bboxes<float32_t, uint8_t>(dst_ptr, next_bbox_output_offset, reg_to_cls_name,
+ inputs.at(reg_to_cls_name.reg), inputs.at(reg_to_cls_name.cls), reg_to_cls_name.stride);
+ } else if (HAILO_FORMAT_TYPE_UINT16 == input_metadata.format.type) {
+ status = add_bboxes<float32_t, uint16_t>(dst_ptr, next_bbox_output_offset, reg_to_cls_name,
+ inputs.at(reg_to_cls_name.reg), inputs.at(reg_to_cls_name.cls), reg_to_cls_name.stride);
+ } else {
+ CHECK_SUCCESS(HAILO_INVALID_ARGUMENT, "YOLOV8 bbox only post-process received invalid input type {}", input_metadata.format.type);
+ }
+
+ CHECK_SUCCESS(status);
+ }
+ return HAILO_SUCCESS;
+}
+
+} // namespace net_flow
+} // namespace hailort
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file yolov8_bbox_only_post_process.hpp
+ * @brief YOLOV8 bbox only post process
+ * Output format of yolov8_bbox_only is NHWC - [1, total_proposals, 4 + number_of_classes]
+ * 1 is the batch size and 4 is the number of coordinates for each proposal
+ * The bboxes entry in the output of yolov8_bbox_only is a list of bboxes, such that each of them looks like this:
+ * (y_min, x_min, y_max, x_max, score_per_class)
+ **/
+
+#ifndef _HAILO_YOLOV8_BBOX_ONLY_POST_PROCESS_HPP_
+#define _HAILO_YOLOV8_BBOX_ONLY_POST_PROCESS_HPP_
+
+#include "net_flow/ops/yolov8_post_process.hpp"
+#include "net_flow/ops_metadata/yolov8_bbox_only_op_metadata.hpp"
+
+namespace hailort
+{
+namespace net_flow
+{
+
+class YOLOv8BboxOnlyPostProcessOp : public YOLOV8PostProcessOp
+{
+public:
+ static Expected<std::shared_ptr<Op>> create(std::shared_ptr<Yolov8BboxOnlyOpMetadata> metadata);
+
+ hailo_status execute(const std::map<std::string, MemoryView> &inputs, std::map<std::string, MemoryView> &outputs) override;
+
+private:
+
+ YOLOv8BboxOnlyPostProcessOp(std::shared_ptr<Yolov8BboxOnlyOpMetadata> metadata) :
+ YOLOV8PostProcessOp(static_cast<std::shared_ptr<Yolov8OpMetadata>>(metadata))
+ {}
+
+ template<typename DstType = float32_t, typename SrcType>
+ hailo_status add_bboxes(DstType *dst_ptr, size_t &next_bbox_output_offset, const Yolov8MatchingLayersNames &layers_names,
+ const MemoryView ®_buffer, const MemoryView &cls_buffer, uint32_t stride)
+ {
+ const auto &inputs_metadata = m_metadata->inputs_metadata();
+ const auto &nms_config = m_metadata->nms_config();
+
+ assert(contains(inputs_metadata, layers_names.reg));
+ assert(contains(inputs_metadata, layers_names.cls));
+ const auto ®_shape = inputs_metadata.at(layers_names.reg).shape;
+ const auto &cls_shape = inputs_metadata.at(layers_names.cls).shape;
+ const auto ®_padded_shape = inputs_metadata.at(layers_names.reg).padded_shape;
+ const auto &cls_padded_shape = inputs_metadata.at(layers_names.cls).padded_shape;
+ const auto ®_quant_info = inputs_metadata.at(layers_names.reg).quant_info;
+ const auto &cls_quant_info = inputs_metadata.at(layers_names.cls).quant_info;
+
+ CHECK_SUCCESS(validate_regression_buffer_size<SrcType>(reg_padded_shape, reg_buffer, layers_names));
+ CHECK_SUCCESS(validate_classes_buffer_size<SrcType>(cls_padded_shape, cls_buffer, layers_names, nms_config));
+
+ // Format is NHCW -> each row size is (padded C size) * (padded W size)
+ auto cls_row_size = cls_padded_shape.features * cls_padded_shape.width;
+
+ SrcType *reg_data = (SrcType*)reg_buffer.data();
+ SrcType *cls_data = (SrcType*)cls_buffer.data();
+
+ for (uint32_t row = 0; row < cls_shape.height; row++) {
+ for (uint32_t col = 0; col < cls_shape.width; col++) {
+ auto cls_idx = (cls_row_size * row) + col;
+
+ assert(contains(m_d_matrix, layers_names.reg));
+ auto &d_matrix = m_d_matrix.at(layers_names.reg);
+ auto bbox = get_bbox<DstType, SrcType>(row, col, stride, reg_padded_shape, reg_shape, reg_quant_info,
+ (SrcType*)reg_data, d_matrix); // we don't pass confidence here
+ memcpy(&dst_ptr[next_bbox_output_offset], &bbox, sizeof(hailo_rectangle_t)); // copy y_min, x_min, y_max, x_max
+ next_bbox_output_offset += sizeof(hailo_rectangle_t) / sizeof(float32_t);
+
+ // copy class confidence for each of the classes
+ for (uint32_t curr_class_idx = 0; curr_class_idx < nms_config.number_of_classes; curr_class_idx++) {
+ auto class_entry_idx = cls_idx + (curr_class_idx * cls_padded_shape.width);
+ auto class_confidence = Quantization::dequantize_output<DstType, SrcType>(
+ cls_data[class_entry_idx], cls_quant_info);
+ dst_ptr[next_bbox_output_offset++] = class_confidence;
+ }
+ }
+ }
+ return HAILO_SUCCESS;
+ }
+};
+
+} /* namespace net_flow */
+} /* namespace hailort */
+
+#endif /* _HAILO_YOLOV5_BBOX_ONLY_POST_PROCESS_HPP_ */
\ No newline at end of file
{
CHECK_SUCCESS(NmsOpMetadata::validate_params());
- CHECK(!nms_config().bbox_only, HAILO_INVALID_ARGUMENT, "YOLOV8PostProcessOp: bbox_only is not supported for YOLOV8 model");
-
// We go over the inputs metadata and check that it includes all of the regs and clss
for (const auto &layer_names : m_yolov8_config.reg_to_cls_inputs) {
CHECK(contains(m_inputs_metadata, layer_names.reg), HAILO_INVALID_ARGUMENT,
#include "net_flow/ops/nms_post_process.hpp"
#include "net_flow/ops/softmax_post_process.hpp"
#include "net_flow/ops_metadata/yolov8_op_metadata.hpp"
+#include "net_flow/ops/softmax_post_process.hpp"
+
namespace hailort
{
namespace net_flow
hailo_status execute(const std::map<std::string, MemoryView> &inputs, std::map<std::string, MemoryView> &outputs) override;
-private:
+protected:
std::shared_ptr<Yolov8OpMetadata> m_metadata;
- std::vector<float32_t> m_d_values_matrix; // Holds the values of the bbox boundaries distances from the stride's center
std::unordered_map<std::string, std::vector<std::vector<float32_t>>> m_d_matrix; // Holds the values from which we compute those distances
YOLOV8PostProcessOp(std::shared_ptr<Yolov8OpMetadata> metadata)
: NmsPostProcessOp(static_cast<std::shared_ptr<NmsOpMetadata>>(metadata))
{
for (const auto &input_metadata : m_metadata->inputs_metadata()) {
m_d_matrix[input_metadata.first] = std::vector<std::vector<float32_t>>(NUM_OF_D_VALUES,
- std::vector<float32_t>(input_metadata.second.shape.features / NUM_OF_D_VALUES));
+ std::vector<float32_t>(input_metadata.second.padded_shape.features / NUM_OF_D_VALUES));
}
}
auto &tmp_vector = d_matrix.at(vector_index);
SoftmaxPostProcessOp::softmax(tmp_vector.data(), tmp_vector.data(), tmp_vector.size());
}
+
// Performing dot product on each vector
// (A, B, C, ..., F, G) -> 0*A + 1*B + 2*C + ... + 14*F + 15*G
for (uint32_t vector_index = 0; vector_index < NUM_OF_D_VALUES; vector_index++) {
m_d_values_matrix[vector_index] = dot_product(d_matrix.at(vector_index));
}
+
// The decode function extract x_min, y_min, x_max, y_max from d1, d2, d3, d4
const auto &d1 = m_d_values_matrix.at(0);
const auto &d2 = m_d_values_matrix.at(1);
return bbox;
}
+ template<typename SrcType>
+ hailo_status validate_regression_buffer_size(const hailo_3d_image_shape_t ®_padded_shape, const MemoryView ®_buffer,
+ const Yolov8MatchingLayersNames &layers_names)
+ {
+ auto number_of_entries = reg_padded_shape.height * reg_padded_shape.width;
+ auto buffer_size = number_of_entries * reg_padded_shape.features * sizeof(SrcType);
+ CHECK(buffer_size == reg_buffer.size(), HAILO_INVALID_ARGUMENT,
+ "Failed to extract_detections, reg {} buffer_size should be {}, but is {}", layers_names.reg, buffer_size, reg_buffer.size());
+ return HAILO_SUCCESS;
+ }
+
+ template<typename SrcType>
+ hailo_status validate_classes_buffer_size(const hailo_3d_image_shape_t &cls_padded_shape, const MemoryView &cls_buffer,
+ const Yolov8MatchingLayersNames &layers_names, const NmsPostProcessConfig &nms_config)
+ {
+ const uint32_t cls_entry_size = nms_config.number_of_classes;
+ auto number_of_entries = cls_padded_shape.height * cls_padded_shape.width;
+ auto buffer_size = number_of_entries * cls_entry_size * sizeof(SrcType);
+ CHECK(buffer_size == cls_buffer.size(), HAILO_INVALID_ARGUMENT,
+ "Failed to extract_detections, cls {} buffer_size should be {}, but is {}", layers_names.cls, buffer_size, cls_buffer.size());
+ return HAILO_SUCCESS;
+ }
+
+private:
+ std::vector<float32_t> m_d_values_matrix; // Holds the values of the bbox boundaries distances from the stride's center
+
static const uint32_t CLASSES_START_INDEX = 0;
static const uint32_t NO_OBJECTNESS = 1;
static const uint32_t NUM_OF_D_VALUES = 4;
const auto ®_quant_info = inputs_metadata.at(layers_names.reg).quant_info;
const auto &cls_quant_info = inputs_metadata.at(layers_names.cls).quant_info;
- // Validate regression buffer size
- auto number_of_entries = reg_padded_shape.height * reg_padded_shape.width;
- auto buffer_size = number_of_entries * reg_padded_shape.features * sizeof(SrcType);
- CHECK(buffer_size == reg_buffer.size(), HAILO_INVALID_ARGUMENT,
- "Failed to extract_detections, reg {} buffer_size should be {}, but is {}", layers_names.reg, buffer_size, reg_buffer.size());
-
- // Validate classes buffer size
- const uint32_t cls_entry_size = nms_config.number_of_classes;
- number_of_entries = cls_padded_shape.height * cls_padded_shape.width;
- buffer_size = number_of_entries * cls_entry_size * sizeof(SrcType);
- CHECK(buffer_size == cls_buffer.size(), HAILO_INVALID_ARGUMENT,
- "Failed to extract_detections, cls {} buffer_size should be {}, but is {}", layers_names.cls, buffer_size, cls_buffer.size());
+ CHECK_SUCCESS(validate_regression_buffer_size<SrcType>(reg_padded_shape, reg_buffer, layers_names));
+ CHECK_SUCCESS(validate_classes_buffer_size<SrcType>(cls_padded_shape, cls_buffer, layers_names, nms_config));
// Format is NHCW -> each row size is (padded C size) * (padded W size)
auto cls_row_size = cls_padded_shape.features * cls_padded_shape.width;
return HAILO_SUCCESS;
}
- template<typename DstType = float32_t, typename SrcType>
- hailo_bbox_float32_t get_bbox(uint32_t row, uint32_t col, uint32_t stride, const hailo_3d_image_shape_t ®_padded_shape,
- const hailo_quant_info_t ®_quant_info, SrcType *reg_data, std::vector<std::vector<DstType>> &d_matrix, DstType class_confidence);
-
virtual hailo_bbox_float32_t decode(float32_t tx, float32_t ty, float32_t tw, float32_t th,
uint32_t col, uint32_t row, uint32_t stride) const;
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file yolov8_bbox_only_op_metadata.hpp
+ * @brief YOLOv8 Bbox Only Post-Process op metadata
+ **/
+
+#ifndef _HAILO_YOLOV8_BBOX_ONLY_OP_METADATA_HPP_
+#define _HAILO_YOLOV8_BBOX_ONLY_OP_METADATA_HPP_
+
+#include "hailo/hailort.h"
+#include "net_flow/ops_metadata/yolov8_op_metadata.hpp"
+
+namespace hailort
+{
+namespace net_flow
+{
+
+class Yolov8BboxOnlyOpMetadata : public Yolov8OpMetadata
+{
+public:
+ static Expected<std::shared_ptr<OpMetadata>> create(const std::unordered_map<std::string, BufferMetaData> &inputs_metadata,
+ const std::unordered_map<std::string, BufferMetaData> &outputs_metadata,
+ const NmsPostProcessConfig &nms_post_process_config,
+ const Yolov8PostProcessConfig &yolov8_post_process_config,
+ const std::string &network_name);
+ hailo_status validate_format_info() override;
+ std::string get_op_description() override;
+ virtual Expected<hailo_vstream_info_t> get_output_vstream_info() override;
+
+private:
+ Yolov8BboxOnlyOpMetadata(const std::unordered_map<std::string, BufferMetaData> &inputs_metadata,
+ const std::unordered_map<std::string, BufferMetaData> &outputs_metadata,
+ const NmsPostProcessConfig &nms_post_process_config,
+ const Yolov8PostProcessConfig &yolov8_post_process_config,
+ const std::string &network_name)
+ : Yolov8OpMetadata(inputs_metadata, outputs_metadata, nms_post_process_config, yolov8_post_process_config, network_name,
+ "YOLOv8Bbox-Only-Post-Process")
+ {}
+
+};
+
+} /* namespace hailort */
+} /* namespace net_flow */
+
+#endif /* _HAILO_YOLOV8_BBOX_ONLY_OP_METADATA_HPP_ */
\ No newline at end of file
namespace net_flow
{
- struct Yolov8MatchingLayersNames
+struct Yolov8MatchingLayersNames
{
// Regression layer
std::string reg;
std::string get_op_description() override;
Yolov8PostProcessConfig &yolov8_config() { return m_yolov8_config;};
-private:
- Yolov8PostProcessConfig m_yolov8_config;
+protected:
Yolov8OpMetadata(const std::unordered_map<std::string, BufferMetaData> &inputs_metadata,
const std::unordered_map<std::string, BufferMetaData> &outputs_metadata,
const NmsPostProcessConfig &nms_post_process_config,
const Yolov8PostProcessConfig &yolov8_post_process_config,
- const std::string &network_name)
- : NmsOpMetadata(inputs_metadata, outputs_metadata, nms_post_process_config, "YOLOV8-Post-Process", network_name, OperationType::YOLOV8)
+ const std::string &network_name,
+ const std::string &name = "YOLOV8-Post-Process")
+ : NmsOpMetadata(inputs_metadata, outputs_metadata, nms_post_process_config, name, network_name, OperationType::YOLOV8)
, m_yolov8_config(yolov8_post_process_config)
{}
-
+ Yolov8PostProcessConfig m_yolov8_config;
hailo_status validate_params() override;
};
auto pipeline_status = make_shared_nothrow<std::atomic<hailo_status>>(HAILO_SUCCESS);
CHECK_AS_EXPECTED(nullptr != pipeline_status, HAILO_OUT_OF_HOST_MEMORY);
- auto async_pipeline_expected = AsyncPipelineBuilder::create_pipeline(net_group, inputs_formats, outputs_formats, timeout, pipeline_status);
- CHECK_EXPECTED(async_pipeline_expected);
+ TRY(auto async_pipeline,
+ AsyncPipelineBuilder::create_pipeline(net_group, inputs_formats, outputs_formats, timeout, pipeline_status));
- auto async_infer_runner_ptr = make_shared_nothrow<AsyncInferRunnerImpl>(async_pipeline_expected.release(), pipeline_status);
+ auto async_infer_runner_ptr = make_shared_nothrow<AsyncInferRunnerImpl>(std::move(async_pipeline), pipeline_status);
CHECK_NOT_NULL_AS_EXPECTED(async_infer_runner_ptr, HAILO_OUT_OF_HOST_MEMORY);
auto status = async_infer_runner_ptr->start_pipeline();
return;
}
-Expected<bool> AsyncInferRunnerImpl::can_push_buffers()
+Expected<bool> AsyncInferRunnerImpl::can_push_buffers(uint32_t frames_count)
{
for (auto &last_element : m_async_pipeline->get_last_elements()) {
- auto can_push_buffer = last_element.second->can_push_buffer_upstream();
- CHECK_EXPECTED(can_push_buffer);
- if (!can_push_buffer.release()) {
+ TRY(const auto can_push_buffer, last_element.second->can_push_buffer_upstream(frames_count));
+ if (!can_push_buffer) {
return false;
}
}
for (auto &entry_element : m_async_pipeline->get_entry_elements()) {
- auto can_push_buffer = entry_element.second->can_push_buffer_downstream();
- CHECK_EXPECTED(can_push_buffer);
- if (!can_push_buffer.release()) {
+ TRY(const auto can_push_buffer, entry_element.second->can_push_buffer_downstream(frames_count));
+ if (!can_push_buffer) {
return false;
}
}
}
hailo_status AsyncInferRunnerImpl::set_buffers(std::unordered_map<std::string, PipelineBuffer> &inputs,
- std::unordered_map<std::string, std::pair<MemoryView, TransferDoneCallbackAsyncInfer>> &outputs)
+ std::unordered_map<std::string, PipelineBuffer> &outputs)
{
for (auto &last_element : m_async_pipeline->get_last_elements()) {
// TODO: handle the non-recoverable case where one buffer is enqueued successfully and the second isn't (HRT-11783)
- auto status = last_element.second->enqueue_execution_buffer(outputs.at(last_element.first).first,
- outputs.at(last_element.first).second);
+ auto status = last_element.second->enqueue_execution_buffer(std::move(outputs.at(last_element.first)));
CHECK_SUCCESS(status);
}
return HAILO_SUCCESS;
}
-void AsyncInferRunnerImpl::set_pix_buffer_inputs(std::unordered_map<std::string, PipelineBuffer> &inputs, hailo_pix_buffer_t userptr_pix_buffer,
+void AsyncInferRunnerImpl::set_pix_buffer_inputs(std::unordered_map<std::string, PipelineBuffer> &inputs, hailo_pix_buffer_t pix_buffer,
TransferDoneCallbackAsyncInfer input_done, const std::string &input_name)
{
- if (1 == userptr_pix_buffer.number_of_planes) {
- inputs[input_name] = PipelineBuffer(MemoryView(userptr_pix_buffer.planes[0].user_ptr, userptr_pix_buffer.planes[0].bytes_used), input_done);
+ if (1 == pix_buffer.number_of_planes) {
+ if (HAILO_PIX_BUFFER_MEMORY_TYPE_DMABUF == pix_buffer.memory_type) {
+ hailo_dma_buffer_t dma_buffer = {pix_buffer.planes[0].fd, pix_buffer.planes[0].plane_size};
+ inputs[input_name] = PipelineBuffer(dma_buffer, input_done);
+ } else if (HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR == pix_buffer.memory_type) {
+ inputs[input_name] = PipelineBuffer(MemoryView(pix_buffer.planes[0].user_ptr, pix_buffer.planes[0].bytes_used), input_done);
+ } else {
+ LOGGER__ERROR("Buffer type Pix buffer supports only memory of types USERPTR or DMABUF.");
+ inputs[input_name] = PipelineBuffer(HAILO_INVALID_OPERATION, input_done);
+ }
} else if (m_async_pipeline->is_multi_planar()) {
// If model is multi-planar
- inputs[input_name] = PipelineBuffer(userptr_pix_buffer, input_done);
+ inputs[input_name] = PipelineBuffer(pix_buffer, input_done);
} else {
// Other cases - return error, as on async flow we do not support copy to new buffer
LOGGER__ERROR("HEF was compiled for single input layer, while trying to pass non-contiguous planes buffers.");
inputs[input_name] = PipelineBuffer(HAILO_INVALID_OPERATION, input_done);
}
-
-}
-
-Expected<hailo_pix_buffer_t> AsyncInferRunnerImpl::convert_dma_pix_buffer_to_userptr_pix_buffer(const hailo_pix_buffer_t &dma_pix_buffer)
-{
- hailo_pix_buffer_t userptr_pix_buffer;
- userptr_pix_buffer.index = dma_pix_buffer.index;
- userptr_pix_buffer.number_of_planes = dma_pix_buffer.number_of_planes;
- userptr_pix_buffer.memory_type = HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR;
-
- for (uint32_t i = 0; i < dma_pix_buffer.number_of_planes; i++ ) {
- auto current_plane = dma_pix_buffer.planes[i];
- hailo_dma_buffer_t dma_buffer = {current_plane.fd, current_plane.bytes_used};
-
- auto dma_buffer_memview_expected = DmaBufferUtils::mmap_dma_buffer_read(dma_buffer);
- CHECK_EXPECTED_AS_STATUS(dma_buffer_memview_expected);
- auto dma_buffer_memview = dma_buffer_memview_expected.release();
-
- hailo_pix_buffer_plane_t new_plane;
- new_plane.bytes_used = current_plane.bytes_used;
- new_plane.plane_size = current_plane.plane_size;
- new_plane.user_ptr = dma_buffer_memview.data();
-
- userptr_pix_buffer.planes[i] = new_plane;
- }
-
- return userptr_pix_buffer;
}
hailo_status AsyncInferRunnerImpl::run(ConfiguredInferModel::Bindings &bindings, TransferDoneCallbackAsyncInfer transfer_done)
hailo_status status = m_async_pipeline->get_pipeline_status()->load();
CHECK_SUCCESS(status, "Can't handle infer request since Pipeline status is {}.", status);
- TRY(auto are_pools_ready, can_push_buffers());
+ TRY(auto are_pools_ready, can_push_buffers(1));
CHECK(are_pools_ready, HAILO_QUEUE_IS_FULL, "Can't handle infer request since a queue in the pipeline is full.");
- std::unordered_map<std::string, std::pair<MemoryView, TransferDoneCallbackAsyncInfer>> outputs;
+ std::unordered_map<std::string, PipelineBuffer> outputs;
+
for (auto &last_element : m_async_pipeline->get_last_elements()) {
auto buff_type = bindings.output(last_element.first)->m_pimpl->get_type();
+ bool is_user_buffer = true;
if (BufferType::DMA_BUFFER == buff_type) {
TRY(auto dma_buffer, bindings.output(last_element.first)->get_dma_buffer(), "Couldnt find output buffer for '{}'", last_element.first);
-
- TRY(auto dma_buffer_memview, DmaBufferUtils::mmap_dma_buffer_write(dma_buffer));
-
- auto output_done = [dma_buffer_memview, dma_buffer=dma_buffer, transfer_done](hailo_status status) {
- auto mumap_status = DmaBufferUtils::munmap_dma_buffer_write(dma_buffer, dma_buffer_memview);
- if (HAILO_SUCCESS != mumap_status) {
- LOGGER__ERROR("Failed to unmap dma buffer");
- status = HAILO_FILE_OPERATION_FAILURE;
- }
- transfer_done(status);
- };
- std::pair<MemoryView, TransferDoneCallbackAsyncInfer> buffer_cb_pair (dma_buffer_memview, output_done);
- outputs[last_element.first] = buffer_cb_pair;
-
+ outputs[last_element.first] = PipelineBuffer(dma_buffer, transfer_done, HAILO_SUCCESS, is_user_buffer);
} else {
TRY(auto buffer, bindings.output(last_element.first)->get_buffer(), "Couldnt find output buffer for '{}'", last_element.first);
-
- std::pair<MemoryView, TransferDoneCallbackAsyncInfer> buffer_cb_pair (buffer, transfer_done);
- outputs[last_element.first] = buffer_cb_pair;
+ outputs[last_element.first] = PipelineBuffer(buffer, transfer_done, HAILO_SUCCESS, is_user_buffer);
}
}
case BufferType::DMA_BUFFER:
{
TRY(auto dma_buffer, bindings.input(entry_element.first)->get_dma_buffer(), "Couldnt find input buffer for '{}'", entry_element.first);
-
- TRY(auto dma_buffer_memview, DmaBufferUtils::mmap_dma_buffer_read(dma_buffer));
-
- auto input_done = [dma_buffer_memview, dma_buffer, transfer_done](hailo_status status) {
- auto mumap_status = DmaBufferUtils::munmap_dma_buffer_read(dma_buffer, dma_buffer_memview);
- if (HAILO_SUCCESS != mumap_status) {
- // Note: we overide the status even if it was not success before (but either way it's set to non-success)
- LOGGER__ERROR("Failed to unmap dma buffer");
- status = mumap_status;
- }
- transfer_done(status);
- };
- inputs[entry_element.first] = PipelineBuffer(dma_buffer_memview, input_done);
+ inputs[entry_element.first] = PipelineBuffer(dma_buffer, transfer_done);
break;
}
case BufferType::PIX_BUFFER:
{
- // TODO: handle a case in which the pix_buffer is DMA buffers (HRT-12771)
TRY(auto pix_buffer, bindings.input(entry_element.first)->get_pix_buffer(), "Couldnt find input buffer for '{}'", entry_element.first);
- if (HAILO_PIX_BUFFER_MEMORY_TYPE_DMABUF == pix_buffer.memory_type) {
- TRY(auto userptr_pix_buffer, convert_dma_pix_buffer_to_userptr_pix_buffer(pix_buffer));
-
- auto input_done = [userptr_pix_buffer, transfer_done, dma_pix_buffer=pix_buffer](hailo_status status) {
- for (uint32_t i = 0; i < dma_pix_buffer.number_of_planes; i++ ) {
- auto plane_in_dma_buffer = dma_pix_buffer.planes[i];
- hailo_dma_buffer_t dma_buffer = {plane_in_dma_buffer.fd, plane_in_dma_buffer.bytes_used};
-
- auto dma_buffer_memview = MemoryView(userptr_pix_buffer.planes[i].user_ptr, userptr_pix_buffer.planes[i].bytes_used);
-
- auto mumap_status = DmaBufferUtils::munmap_dma_buffer_read(dma_buffer, dma_buffer_memview);
- if (HAILO_SUCCESS != mumap_status) {
- // Note: we overide the status even if it was not success before (but either way it's set to non-success)
- LOGGER__ERROR("Failed to unmap dma buffer");
- status = mumap_status;
- }
- }
- transfer_done(status);
- };
-
- set_pix_buffer_inputs(inputs, userptr_pix_buffer, input_done, entry_element.first);
- } else {
- set_pix_buffer_inputs(inputs, pix_buffer, transfer_done, entry_element.first);
- }
+ set_pix_buffer_inputs(inputs, pix_buffer, transfer_done, entry_element.first);
break;
}
}
status = set_buffers(inputs, outputs);
+ // TODO: (HRT-14283) If set_buffers fails after a buffer is enqueued, the buffer's CB will be called - and might call user's CB
CHECK_SUCCESS(status);
-
return HAILO_SUCCESS;
}
hailo_status run(ConfiguredInferModel::Bindings &bindings, TransferDoneCallbackAsyncInfer transfer_done);
hailo_status set_buffers(std::unordered_map<std::string, PipelineBuffer> &inputs,
- std::unordered_map<std::string, std::pair<MemoryView, TransferDoneCallbackAsyncInfer>> &outputs);
+ std::unordered_map<std::string, PipelineBuffer> &outputs);
void abort();
- Expected<bool> can_push_buffers();
+ Expected<bool> can_push_buffers(uint32_t frames_count);
void add_element_to_pipeline(std::shared_ptr<PipelineElement> pipeline_element);
void add_entry_element(std::shared_ptr<PipelineElement> pipeline_element, const std::string &input_name);
hailo_status start_pipeline();
hailo_status stop_pipeline();
- static Expected<hailo_pix_buffer_t> convert_dma_pix_buffer_to_userptr_pix_buffer(const hailo_pix_buffer_t &dma_pix_buffer);
void set_pix_buffer_inputs(std::unordered_map<std::string, PipelineBuffer> &inputs, hailo_pix_buffer_t userptr_pix_buffer,
TransferDoneCallbackAsyncInfer input_done, const std::string &input_name);
#include "net_flow/ops/yolov5_seg_post_process.hpp"
#include "net_flow/ops/yolov5_bbox_only_post_process.hpp"
#include "net_flow/ops/yolov8_post_process.hpp"
+#include "net_flow/ops/yolov8_bbox_only_post_process.hpp"
#include "net_flow/ops/argmax_post_process.hpp"
#include "net_flow/ops/softmax_post_process.hpp"
#include "net_flow/ops/yolox_post_process.hpp"
{
std::unordered_map<std::string, hailo_format_t> expanded_input_format;
for (auto &input_format : inputs_formats) {
- auto input_streams_names = net_group->get_stream_names_from_vstream_name(input_format.first);
- CHECK_EXPECTED(input_streams_names);
+ TRY(const auto input_streams_names, net_group->get_stream_names_from_vstream_name(input_format.first));
- auto is_multi_planar = (input_streams_names.value().size() > 1);
+ auto is_multi_planar = (input_streams_names.size() > 1);
if(is_multi_planar) {
- auto vstream_info_exp = net_group->get_input_vstream_infos();
- CHECK_EXPECTED(vstream_info_exp);
- auto vstream_infos = vstream_info_exp.release();
+ TRY(const auto vstream_infos, net_group->get_input_vstream_infos());
auto matching_vstream_info = std::find_if(vstream_infos.begin(), vstream_infos.end(), [&](const auto &item)
{ return item.name == input_format.first; } );
CHECK_AS_EXPECTED(vstream_infos.end() != matching_vstream_info, HAILO_NOT_FOUND,
expanded_input_format[input_format.first] =
VStreamsBuilderUtils::expand_user_buffer_format_autos_multi_planar(*matching_vstream_info, input_format.second);
} else {
- const auto &stream_name = input_streams_names.value()[0];
+ const auto &stream_name = input_streams_names[0];
CHECK_AS_EXPECTED(contains(named_stream_infos, stream_name), HAILO_INTERNAL_FAILURE);
const auto &stream_info = named_stream_infos.at(stream_name);
{
std::unordered_map<std::string, hailo_format_t> expanded_output_format;
for (auto &output_format : outputs_formats) {
- auto output_streams_names = net_group->get_stream_names_from_vstream_name(output_format.first);
- CHECK_EXPECTED(output_streams_names);
+ TRY(const auto output_streams_names, net_group->get_stream_names_from_vstream_name(output_format.first));
// TODO: Taking data from the first ll stream will not work in multi-planar work
- const auto &stream_name = output_streams_names.value()[0];
+ const auto &stream_name = output_streams_names[0];
CHECK_AS_EXPECTED(contains(named_stream_infos, stream_name), HAILO_INTERNAL_FAILURE);
const auto &stream_info = named_stream_infos.at(stream_name);
const std::vector<std::string> &stream_names, const std::unordered_map<std::string, hailo_format_t> &inputs_formats,
const std::unordered_map<std::string, hailo_stream_info_t> &named_stream_infos, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto vstream_names = net_group->get_vstream_names_from_stream_name(*stream_names.begin());
- CHECK_EXPECTED_AS_STATUS(vstream_names);
- CHECK(vstream_names.value().size() == 1, HAILO_NOT_SUPPORTED, "low level stream must have exactly 1 user input");
- const auto &vstream_name = vstream_names.value()[0];
+ TRY(const auto vstream_names, net_group->get_vstream_names_from_stream_name(*stream_names.begin()));
+ CHECK(vstream_names.size() == 1, HAILO_NOT_SUPPORTED, "low level stream must have exactly 1 user input");
+ const auto &vstream_name = vstream_names[0];
std::shared_ptr<PixBufferElement> multi_plane_splitter = nullptr;
std::shared_ptr<PipelineElement> last_element_connected_to_pipeline = nullptr;
auto is_empty = true;
auto interacts_with_hw = true; // We want the entry queue size to be the size of queues interacts with HW
auto is_entry = true;
- auto entry_queue_elem_expected = add_push_queue_element(PipelineObject::create_element_name("EntryPushQEl", vstream_name, 0),
- async_pipeline, 0, is_empty, interacts_with_hw, nullptr, 0, is_entry);
- CHECK_EXPECTED_AS_STATUS(entry_queue_elem_expected);
- auto entry_queue_elem = entry_queue_elem_expected.release();
+ TRY(auto entry_queue_elem,
+ add_push_queue_element(PipelineObject::create_element_name("EntryPushQEl", vstream_name, 0),
+ async_pipeline, 0, is_empty, interacts_with_hw, nullptr, 0, is_entry));
async_pipeline->add_entry_element(entry_queue_elem, vstream_name);
last_element_connected_to_pipeline = entry_queue_elem;
async_pipeline->set_as_multi_planar();
const auto &vstream_order = inputs_formats.at(vstream_name).order;
- auto multi_plane_splitter_expected = create_multi_plane_splitter_element(vstream_name, vstream_order,
- async_pipeline->get_build_params().pipeline_status, async_pipeline);
- CHECK_EXPECTED_AS_STATUS(multi_plane_splitter_expected);
- multi_plane_splitter = multi_plane_splitter_expected.release();
+ TRY(multi_plane_splitter, create_multi_plane_splitter_element(vstream_name, vstream_order,
+ async_pipeline->get_build_params().pipeline_status, async_pipeline));
async_pipeline->add_element_to_pipeline(multi_plane_splitter);
CHECK_SUCCESS(PipelinePad::link_pads(entry_queue_elem, multi_plane_splitter));
const auto &input_stream_info = named_stream_infos.at(stream_name);
auto src_format = inputs_formats.at(vstream_name);
- auto sink_index_expected = async_pipeline->get_async_hw_element()->get_sink_index_from_input_stream_name(stream_name);
- CHECK_EXPECTED_AS_STATUS(sink_index_expected);
- auto sink_index = static_cast<uint8_t>(sink_index_expected.release());
+ TRY(const auto sink_index, async_pipeline->get_async_hw_element()->get_sink_index_from_input_stream_name(stream_name));
if(is_multi_planar) {
is_empty = true;
interacts_with_hw = false;
- auto post_split_push_queue = add_push_queue_element(
+ TRY(auto post_split_push_queue, add_push_queue_element(
PipelineObject::create_element_name("PostSplitPushQEl", stream_name, sink_index),
- async_pipeline, 0, is_empty, interacts_with_hw, nullptr);
- CHECK_EXPECTED_AS_STATUS(post_split_push_queue);
- CHECK_SUCCESS(PipelinePad::link_pads(multi_plane_splitter, post_split_push_queue.value(), plane_index++));
+ async_pipeline, 0, is_empty, interacts_with_hw, nullptr));
+ CHECK_SUCCESS(PipelinePad::link_pads(multi_plane_splitter, post_split_push_queue, plane_index++));
- last_element_connected_to_pipeline = post_split_push_queue.value();
+ last_element_connected_to_pipeline = post_split_push_queue;
/* In multi-planar case, the format order of each plane (stream) is determined by the ll-stream's order.
Type and flags are determined by the vstream params */
src_format.order = input_stream_info.format.order;
}
- auto should_transform = InputTransformContext::is_transformation_required(input_stream_info.shape,
+ TRY(const auto should_transform, InputTransformContext::is_transformation_required(input_stream_info.shape,
src_format, input_stream_info.hw_shape, input_stream_info.format,
- std::vector<hailo_quant_info_t>(1, input_stream_info.quant_info)); // Inputs always have single quant_info
- CHECK_EXPECTED_AS_STATUS(should_transform);
+ std::vector<hailo_quant_info_t>(1, input_stream_info.quant_info))); // Inputs always have single quant_info
- if (should_transform.value()) {
- auto pre_infer_elem = PreInferElement::create(input_stream_info.shape, src_format,
+ if (should_transform) {
+ TRY(auto pre_infer_elem, PreInferElement::create(input_stream_info.shape, src_format,
input_stream_info.hw_shape, input_stream_info.format, { input_stream_info.quant_info },
PipelineObject::create_element_name("PreInferEl", stream_name, input_stream_info.index),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED_AS_STATUS(pre_infer_elem);
- async_pipeline->add_element_to_pipeline(pre_infer_elem.value());
- CHECK_SUCCESS(PipelinePad::link_pads(last_element_connected_to_pipeline, pre_infer_elem.value()));
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
+ async_pipeline->add_element_to_pipeline(pre_infer_elem);
+ CHECK_SUCCESS(PipelinePad::link_pads(last_element_connected_to_pipeline, pre_infer_elem));
is_empty = false;
interacts_with_hw = true;
- auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl", stream_name, input_stream_info.index),
- async_pipeline, input_stream_info.hw_frame_size, is_empty, interacts_with_hw, pre_infer_elem.value());
- CHECK_EXPECTED_AS_STATUS(queue_elem);
- CHECK_SUCCESS(PipelinePad::link_pads(pre_infer_elem.value(), queue_elem.value()));
- CHECK_SUCCESS(PipelinePad::link_pads(queue_elem.value(), async_pipeline->get_async_hw_element(), 0, sink_index));
+ TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl", stream_name, input_stream_info.index),
+ async_pipeline, input_stream_info.hw_frame_size, is_empty, interacts_with_hw, pre_infer_elem));
+ CHECK_SUCCESS(PipelinePad::link_pads(pre_infer_elem, queue_elem));
+ CHECK_SUCCESS(PipelinePad::link_pads(queue_elem, async_pipeline->get_async_hw_element(), 0, sink_index));
} else {
CHECK_SUCCESS(PipelinePad::link_pads(last_element_connected_to_pipeline, async_pipeline->get_async_hw_element(), 0, sink_index));
}
std::shared_ptr<AsyncPipeline> async_pipeline)
{
for(const auto &input : inputs_formats) {
- auto stream_names_under_vstream = net_group->get_stream_names_from_vstream_name(input.first);
- CHECK_EXPECTED_AS_STATUS(stream_names_under_vstream);
+ TRY(const auto stream_names_under_vstream,
+ net_group->get_stream_names_from_vstream_name(input.first));
- auto status = create_pre_async_hw_elements_per_input(net_group, stream_names_under_vstream.release(), inputs_formats,
+ auto status = create_pre_async_hw_elements_per_input(net_group, stream_names_under_vstream, inputs_formats,
named_stream_infos, async_pipeline);
CHECK_SUCCESS(status);
}
HailoRTCommon::get_nms_hw_frame_size(nms_info) : HailoRTCommon::get_periph_frame_size(src_image_shape, src_format);
auto is_empty = false;
auto interacts_with_hw = true;
- auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl", final_elem->name(),
+ TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl", final_elem->name(),
static_cast<uint8_t>(final_elem_source_index)), async_pipeline, pre_transform_frame_size, is_empty, interacts_with_hw,
- final_elem, final_elem_source_index);
- CHECK_EXPECTED(queue_elem);
+ final_elem, final_elem_source_index));
- auto post_infer_elem = PostInferElement::create(src_image_shape, src_format, dst_image_shape, output_format,
+ TRY(auto post_infer_elem, PostInferElement::create(src_image_shape, src_format, dst_image_shape, output_format,
dst_quant_infos, nms_info, PipelineObject::create_element_name("PostInferEl",
final_elem->name(), static_cast<uint8_t>(final_elem_source_index)), async_pipeline->get_build_params(),
- PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED(post_infer_elem);
+ PipelineDirection::PUSH, async_pipeline));
- async_pipeline->add_element_to_pipeline(post_infer_elem.value());
+ async_pipeline->add_element_to_pipeline(post_infer_elem);
- CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(queue_elem.value(), post_infer_elem.value()));
- return post_infer_elem.release();
+ CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(queue_elem, post_infer_elem));
+ return post_infer_elem;
}
Expected<std::shared_ptr<AsyncPushQueueElement>> AsyncPipelineBuilder::add_push_queue_element(const std::string &queue_name, std::shared_ptr<AsyncPipeline> async_pipeline,
size_t frame_size, bool is_empty, bool interacts_with_hw, std::shared_ptr<PipelineElement> final_elem, const uint32_t final_elem_source_index, bool is_entry)
{
- auto push_queue_elem = AsyncPushQueueElement::create(queue_name, async_pipeline->get_build_params(), frame_size,
- is_empty, interacts_with_hw, async_pipeline, is_entry);
- CHECK_EXPECTED(push_queue_elem);
+ TRY(auto push_queue_elem, AsyncPushQueueElement::create(queue_name, async_pipeline->get_build_params(), frame_size,
+ is_empty, interacts_with_hw, async_pipeline, is_entry));
- async_pipeline->add_element_to_pipeline(push_queue_elem.value());
+ async_pipeline->add_element_to_pipeline(push_queue_elem);
// final elem will be nullptr in case it's the first element in pipeline
if (final_elem) {
- CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, push_queue_elem.value(), final_elem_source_index, 0));
+ CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, push_queue_elem, final_elem_source_index, 0));
}
- return push_queue_elem.release();
+ return push_queue_elem;
}
Expected<std::shared_ptr<ConvertNmsToDetectionsElement>> AsyncPipelineBuilder::add_nms_to_detections_convert_element(std::shared_ptr<AsyncPipeline> async_pipeline,
auto metadata = std::dynamic_pointer_cast<net_flow::NmsOpMetadata>(op_metadata);
assert(nullptr != metadata);
- auto nms_to_detections_element = ConvertNmsToDetectionsElement::create(metadata->nms_info(),
+ TRY(auto nms_to_detections_element, ConvertNmsToDetectionsElement::create(metadata->nms_info(),
PipelineObject::create_element_name(element_name, output_stream_name, stream_index),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED(nms_to_detections_element);
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
- async_pipeline->add_element_to_pipeline(nms_to_detections_element.value());
+ async_pipeline->add_element_to_pipeline(nms_to_detections_element);
- CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, nms_to_detections_element.value(), final_elem_index, 0));
- return nms_to_detections_element.release();
+ CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, nms_to_detections_element, final_elem_index, 0));
+ return nms_to_detections_element;
}
Expected<std::shared_ptr<RemoveOverlappingBboxesElement>> AsyncPipelineBuilder::add_remove_overlapping_bboxes_element(std::shared_ptr<AsyncPipeline> async_pipeline,
auto metadata = std::dynamic_pointer_cast<net_flow::NmsOpMetadata>(op_metadata);
assert(nullptr != metadata);
- auto remove_overlapping_bboxes_element = RemoveOverlappingBboxesElement::create(metadata->nms_config(),
+ TRY(auto remove_overlapping_bboxes_element, RemoveOverlappingBboxesElement::create(metadata->nms_config(),
PipelineObject::create_element_name(element_name, output_stream_name, stream_index),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED(remove_overlapping_bboxes_element);
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
- async_pipeline->add_element_to_pipeline(remove_overlapping_bboxes_element.value());
+ async_pipeline->add_element_to_pipeline(remove_overlapping_bboxes_element);
- CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, remove_overlapping_bboxes_element.value(), final_elem_index, 0));
+ CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, remove_overlapping_bboxes_element, final_elem_index, 0));
return remove_overlapping_bboxes_element;
}
auto metadata = std::dynamic_pointer_cast<net_flow::NmsOpMetadata>(op_metadata);
assert(nullptr != metadata);
- auto fill_nms_format_element = FillNmsFormatElement::create(metadata->nms_config(),
+ TRY(auto fill_nms_format_element, FillNmsFormatElement::create(metadata->nms_config(),
PipelineObject::create_element_name(element_name, output_stream_name, stream_index),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED(fill_nms_format_element);
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
- async_pipeline->add_element_to_pipeline(fill_nms_format_element.value());
+ async_pipeline->add_element_to_pipeline(fill_nms_format_element);
- CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, fill_nms_format_element.value(), final_elem_index, 0));
+ CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, fill_nms_format_element, final_elem_index, 0));
return fill_nms_format_element;
}
Expected<std::shared_ptr<LastAsyncElement>> AsyncPipelineBuilder::add_last_async_element(std::shared_ptr<AsyncPipeline> async_pipeline,
const std::string &output_format_name, size_t frame_size, std::shared_ptr<PipelineElement> final_elem, const uint32_t final_elem_source_index)
{
- auto last_async_element = LastAsyncElement::create(PipelineObject::create_element_name("LastAsyncEl",
- final_elem->name(), static_cast<uint8_t>(final_elem_source_index)), async_pipeline->get_build_params(), frame_size, async_pipeline);
- CHECK_EXPECTED(last_async_element);
+ TRY(auto last_async_element, LastAsyncElement::create(PipelineObject::create_element_name("LastAsyncEl",
+ final_elem->name(), static_cast<uint8_t>(final_elem_source_index)), async_pipeline->get_build_params(), frame_size, async_pipeline));
- async_pipeline->add_element_to_pipeline(last_async_element.value());
- CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, last_async_element.value(), final_elem_source_index, 0));
+ async_pipeline->add_element_to_pipeline(last_async_element);
+ CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, last_async_element, final_elem_source_index, 0));
- async_pipeline->add_last_element(last_async_element.value(), output_format_name);
+ async_pipeline->add_last_element(last_async_element, output_format_name);
- return last_async_element.release();
+ return last_async_element;
}
Expected<std::pair<std::string, hailo_format_t>> AsyncPipelineBuilder::get_output_format_from_edge_info_name(const std::string &edge_info_name,
CHECK(contains(named_stream_infos, output_stream_name), HAILO_INTERNAL_FAILURE);
const auto &stream_info = named_stream_infos.at(output_stream_name);
- auto source_index = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(output_stream_name);
- CHECK_EXPECTED_AS_STATUS(source_index);
+ TRY(const auto source_index,
+ async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(output_stream_name));
auto is_empty = false;
auto interacts_with_hw = true;
- auto hw_queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQueueElement_post_hw", stream_info.name, stream_info.index),
- async_pipeline, stream_info.hw_frame_size, is_empty, interacts_with_hw, async_pipeline->get_async_hw_element(), *source_index);
- CHECK_EXPECTED_AS_STATUS(hw_queue_elem);
+ TRY_V(auto hw_queue_elem,
+ add_push_queue_element(PipelineObject::create_element_name("PushQueueElement_post_hw", stream_info.name, stream_info.index),
+ async_pipeline, stream_info.hw_frame_size, is_empty, interacts_with_hw, async_pipeline->get_async_hw_element(), source_index));
- auto layer_info = net_group->get_layer_info(output_stream_name);
- CHECK_EXPECTED_AS_STATUS(layer_info);
+ TRY(const auto layer_info, net_group->get_layer_info(output_stream_name));
- auto expected_demuxer = OutputDemuxerBase::create(stream_info.hw_frame_size, *layer_info.value());
- CHECK_EXPECTED_AS_STATUS(expected_demuxer);
+ TRY(auto demuxer, OutputDemuxerBase::create(stream_info.hw_frame_size, *layer_info));
- auto demuxer_ptr = make_shared_nothrow<OutputDemuxerBase>(expected_demuxer.release());
+ auto demuxer_ptr = make_shared_nothrow<OutputDemuxerBase>(std::move(demuxer));
CHECK_ARG_NOT_NULL(demuxer_ptr);
- auto demux_elem = TransformDemuxElement::create(demuxer_ptr,
+ TRY(auto demux_elem, TransformDemuxElement::create(demuxer_ptr,
PipelineObject::create_element_name("TransformDemuxEl", output_stream_name, stream_info.index),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED_AS_STATUS(demux_elem);
- async_pipeline->add_element_to_pipeline(demux_elem.value());
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
+ async_pipeline->add_element_to_pipeline(demux_elem);
- CHECK_SUCCESS(PipelinePad::link_pads(hw_queue_elem.value(), demux_elem.value()));
+ CHECK_SUCCESS(PipelinePad::link_pads(hw_queue_elem, demux_elem));
uint8_t i = 0;
for (auto &edge_info : demuxer_ptr->get_edges_stream_info()) {
- auto output_format_expected = get_output_format_from_edge_info_name(edge_info.name, outputs_formats);
- CHECK_EXPECTED_AS_STATUS(output_format_expected);
+ TRY(const auto output_format, get_output_format_from_edge_info_name(edge_info.name, outputs_formats));
- is_empty = false;
- interacts_with_hw = false;
- auto demux_queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_demux", edge_info.name, i), async_pipeline,
- edge_info.hw_frame_size, is_empty, interacts_with_hw, demux_elem.value(), i);
- CHECK_EXPECTED_AS_STATUS(demux_queue_elem);
+ TRY(const auto should_transform, OutputTransformContext::is_transformation_required(edge_info.hw_shape,
+ edge_info.format, edge_info.shape, output_format.second, std::vector<hailo_quant_info_t>{edge_info.quant_info})); // TODO: Get quant vector (HRT-11077)
- auto should_transform = OutputTransformContext::is_transformation_required(edge_info.hw_shape,
- edge_info.format, edge_info.shape, output_format_expected.value().second, std::vector<hailo_quant_info_t>{edge_info.quant_info}); // TODO: Get quant vector (HRT-11077)
- CHECK_EXPECTED_AS_STATUS(should_transform);
+ if (should_transform) {
+ is_empty = false;
+ interacts_with_hw = false;
+ TRY(auto demux_queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_demux", edge_info.name, i), async_pipeline,
+ edge_info.hw_frame_size, is_empty, interacts_with_hw, demux_elem, i));
- if (should_transform.value()) {
- auto post_infer_elem = add_post_infer_element(output_format_expected.value().second, edge_info.nms_info,
- async_pipeline, edge_info.hw_shape, edge_info.format, edge_info.shape, {edge_info.quant_info}, demux_queue_elem.value());
- CHECK_EXPECTED_AS_STATUS(post_infer_elem);
+ TRY(auto post_infer_elem, add_post_infer_element(output_format.second, edge_info.nms_info,
+ async_pipeline, edge_info.hw_shape, edge_info.format, edge_info.shape, {edge_info.quant_info}, demux_queue_elem));
auto post_transform_frame_size = (HailoRTCommon::is_nms(edge_info.format.order)) ?
- HailoRTCommon::get_nms_host_frame_size(edge_info.nms_info, output_format_expected.value().second) :
- HailoRTCommon::get_frame_size(edge_info.shape, output_format_expected.value().second);
+ HailoRTCommon::get_nms_host_frame_size(edge_info.nms_info, output_format.second) :
+ HailoRTCommon::get_frame_size(edge_info.shape, output_format.second);
- auto last_async_element = add_last_async_element(async_pipeline, output_format_expected.value().first, post_transform_frame_size,
- post_infer_elem.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size,
+ post_infer_elem));
} else {
- auto last_async_element = add_last_async_element(async_pipeline, output_format_expected.value().first, edge_info.hw_frame_size,
- demux_queue_elem.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, edge_info.hw_frame_size,
+ demux_elem));
}
i++;
}
Expected<bool> AsyncPipelineBuilder::should_transform(const hailo_stream_info_t &stream_info, const std::vector<hailo_quant_info_t> &stream_quant_infos,
const hailo_format_t &output_format)
{
- auto should_transform = OutputTransformContext::is_transformation_required(stream_info.hw_shape,
+ return OutputTransformContext::is_transformation_required(stream_info.hw_shape,
stream_info.format, stream_info.shape, output_format, stream_quant_infos);
- CHECK_EXPECTED(should_transform);
- return should_transform.release();
}
hailo_status AsyncPipelineBuilder::add_nms_fuse_flow(const std::vector<std::string> &output_streams_names,
// To get the fused layer name and src stream format, we use the stream info of one of the defuses
auto fused_layer_name = first_defused_stream_info.nms_info.defuse_info.original_name;
- auto nms_elem = NmsMuxElement::create(nms_infos, PipelineObject::create_element_name("NmsMuxEl", fused_layer_name, 0),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED_AS_STATUS(nms_elem);
-
- async_pipeline->add_element_to_pipeline(nms_elem.value());
+ TRY(auto nms_elem, NmsMuxElement::create(nms_infos, PipelineObject::create_element_name("NmsMuxEl", fused_layer_name, 0),
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
+ async_pipeline->add_element_to_pipeline(nms_elem);
uint32_t i = 0;
for (const auto &stream_name : output_streams_names) {
CHECK(contains(named_stream_infos, stream_name), HAILO_INTERNAL_FAILURE);
const auto &curr_stream_info = named_stream_infos.at(stream_name);
- auto output_index = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name);
- CHECK_EXPECTED_AS_STATUS(output_index);
+ TRY(const auto output_index, async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name));
auto is_empty = false;
auto interacts_with_hw = true;
- auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_nms", curr_stream_info.name, curr_stream_info.index),
+ TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_nms", curr_stream_info.name, curr_stream_info.index),
async_pipeline, curr_stream_info.hw_frame_size, is_empty, interacts_with_hw,
- async_pipeline->get_async_hw_element(), output_index.value());
- CHECK_EXPECTED_AS_STATUS(queue_elem);
+ async_pipeline->get_async_hw_element(), output_index));
- CHECK_SUCCESS(PipelinePad::link_pads(queue_elem.value(), nms_elem.value(), 0, i));
+ CHECK_SUCCESS(PipelinePad::link_pads(queue_elem, nms_elem, 0, i));
i++;
}
auto stream_quant_infos = std::vector<hailo_quant_info_t>(1, first_defused_stream_info.quant_info);
// On NMS models we always need tp post-infer
- auto fused_layer_nms_info = nms_elem.value()->get_fused_nms_info();
+ const auto &fused_layer_nms_info = nms_elem->get_fused_nms_info();
- auto post_infer_elem = add_post_infer_element(output_format.second, fused_layer_nms_info, async_pipeline,
- first_defused_stream_info.hw_shape, first_defused_stream_info.format, first_defused_stream_info.shape, stream_quant_infos, nms_elem.value());
- CHECK_EXPECTED_AS_STATUS(post_infer_elem);
+ TRY(auto post_infer_elem, add_post_infer_element(output_format.second, fused_layer_nms_info, async_pipeline,
+ first_defused_stream_info.hw_shape, first_defused_stream_info.format, first_defused_stream_info.shape, stream_quant_infos, nms_elem));
- auto post_transform_frame_size = HailoRTCommon::get_nms_host_frame_size(fused_layer_nms_info, output_format.second);
+ const auto post_transform_frame_size = HailoRTCommon::get_nms_host_frame_size(fused_layer_nms_info, output_format.second);
- auto last_async_element = add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size,
- post_infer_elem.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size,
+ post_infer_elem));
return HAILO_SUCCESS;
}
auto updated_output_format = output_format;
- auto hw_async_elem_index = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name);
- CHECK_EXPECTED_AS_STATUS(hw_async_elem_index);
+ TRY(const auto hw_async_elem_index, async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name));
- auto op_input_format = softmax_op_metadata->inputs_metadata().begin()->second.format;
- auto output_format_expanded = net_flow::SoftmaxOpMetadata::expand_output_format_autos(updated_output_format.second, op_input_format);
+ const auto op_input_format = softmax_op_metadata->inputs_metadata().begin()->second.format;
+ const auto output_format_expanded = net_flow::SoftmaxOpMetadata::expand_output_format_autos(updated_output_format.second, op_input_format);
// TODO (HRT-11078): Fix multi qp for PP
auto stream_quant_infos = std::vector<hailo_quant_info_t>(1, stream_info.quant_info);
- auto post_infer_elem = add_post_infer_element(output_format_expanded, {}, async_pipeline, stream_info.hw_shape, stream_info.format,
- stream_info.shape, stream_quant_infos, async_pipeline->get_async_hw_element(), hw_async_elem_index.value());
- CHECK_EXPECTED_AS_STATUS(post_infer_elem);
+ TRY(auto post_infer_elem, add_post_infer_element(output_format_expanded, {}, async_pipeline, stream_info.hw_shape, stream_info.format,
+ stream_info.shape, stream_quant_infos, async_pipeline->get_async_hw_element(), hw_async_elem_index));
auto is_empty = false;
auto interacts_with_hw = false;
const auto post_transform_frame_size = HailoRTCommon::get_frame_size(stream_info.shape, output_format_expanded);
- auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_softmax", async_pipeline->get_async_hw_element()->name(),
- static_cast<uint8_t>(hw_async_elem_index.value())), async_pipeline, post_transform_frame_size, is_empty, interacts_with_hw, post_infer_elem.value());
- CHECK_EXPECTED_AS_STATUS(queue_elem);
+ TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_softmax", async_pipeline->get_async_hw_element()->name(),
+ static_cast<uint8_t>(hw_async_elem_index)), async_pipeline, post_transform_frame_size, is_empty, interacts_with_hw, post_infer_elem));
// Updating metadata according to user request
// Currently softmax only supports inputs to be float32 and order NHWC or NC
metadata->set_inputs_metadata(updated_inputs_metadata);
CHECK_SUCCESS(metadata->validate_format_info());
- auto op_expected = net_flow::SoftmaxPostProcessOp::create(metadata);
- CHECK_EXPECTED_AS_STATUS(op_expected);
-
- auto softmax_op = op_expected.release();
- auto softmax_element = SoftmaxPostProcessElement::create(softmax_op,
+ TRY(auto softmax_op, net_flow::SoftmaxPostProcessOp::create(metadata));
+ TRY(auto softmax_element, SoftmaxPostProcessElement::create(softmax_op,
PipelineObject::create_element_name("SoftmaxPPEl", stream_name, stream_info.index),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED_AS_STATUS(softmax_element);
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
- async_pipeline->add_element_to_pipeline(softmax_element.value());
- CHECK_SUCCESS(PipelinePad::link_pads(queue_elem.value(), softmax_element.value()));
+ async_pipeline->add_element_to_pipeline(softmax_element);
+ CHECK_SUCCESS(PipelinePad::link_pads(queue_elem, softmax_element));
- auto last_async_element = add_last_async_element(async_pipeline, updated_output_format.first, post_transform_frame_size,
- softmax_element.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element, add_last_async_element(async_pipeline, updated_output_format.first, post_transform_frame_size,
+ softmax_element));
return HAILO_SUCCESS;
}
CHECK(contains(named_stream_infos, stream_name), HAILO_INTERNAL_FAILURE);
const auto &stream_info = named_stream_infos.at(stream_name);
- auto hw_async_elem_index = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name);
- CHECK_EXPECTED_AS_STATUS(hw_async_elem_index);
+ TRY(const auto hw_async_elem_index, async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name));
auto is_empty = false;
auto interacts_with_hw = true;
- auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_argmax", async_pipeline->get_async_hw_element()->name(),
- static_cast<uint8_t>(hw_async_elem_index.value())), async_pipeline, stream_info.hw_frame_size, is_empty, interacts_with_hw,
- async_pipeline->get_async_hw_element(), hw_async_elem_index.value());
- CHECK_EXPECTED_AS_STATUS(queue_elem);
+ TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_argmax", async_pipeline->get_async_hw_element()->name(),
+ static_cast<uint8_t>(hw_async_elem_index)), async_pipeline, stream_info.hw_frame_size, is_empty, interacts_with_hw,
+ async_pipeline->get_async_hw_element(), hw_async_elem_index));
// Updating metadata according to user request
- auto op_input_format = argmax_op_metadata->inputs_metadata().begin()->second.format;
+ const auto op_input_format = argmax_op_metadata->inputs_metadata().begin()->second.format;
auto updated_outputs_metadata = argmax_op_metadata.get()->outputs_metadata();
updated_outputs_metadata.begin()->second.format = net_flow::ArgmaxOpMetadata::expand_output_format_autos(output_format.second, op_input_format);
auto metadata = std::dynamic_pointer_cast<net_flow::ArgmaxOpMetadata>(argmax_op_metadata);
metadata->set_outputs_metadata(updated_outputs_metadata);
CHECK_SUCCESS(metadata->validate_format_info());
- auto op_expected = net_flow::ArgmaxPostProcessOp::create(metadata);
- CHECK_EXPECTED_AS_STATUS(op_expected);
- auto argmax_op = op_expected.release();
-
- auto argmax_element = ArgmaxPostProcessElement::create(argmax_op,
+ TRY(auto argmax_op, net_flow::ArgmaxPostProcessOp::create(metadata));
+ TRY(auto argmax_element, ArgmaxPostProcessElement::create(argmax_op,
PipelineObject::create_element_name("ArgmaxPPEl", stream_name, stream_info.index),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED_AS_STATUS(argmax_element);
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
- async_pipeline->add_element_to_pipeline(argmax_element.value());
- CHECK_SUCCESS(PipelinePad::link_pads(queue_elem.value(), argmax_element.value()));
+ async_pipeline->add_element_to_pipeline(argmax_element);
+ CHECK_SUCCESS(PipelinePad::link_pads(queue_elem, argmax_element));
const auto post_transform_frame_size = HailoRTCommon::get_frame_size(updated_outputs_metadata.begin()->second.shape,
updated_outputs_metadata.begin()->second.format);
- auto last_async_element = add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size,
- argmax_element.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size,
+ argmax_element));
return HAILO_SUCCESS;
}
};
outputs_metadata.insert({nms_op->outputs_metadata().begin()->first, output_metadata});
- auto nms_elem = NmsPostProcessMuxElement::create(nms_op, PipelineObject::create_element_name("NmsPPMuxEl", nms_op->get_name(), 0),
- async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED_AS_STATUS(nms_elem);
+ TRY(auto nms_elem,
+ NmsPostProcessMuxElement::create(nms_op, PipelineObject::create_element_name("NmsPPMuxEl", nms_op->get_name(), 0),
+ async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline));
- async_pipeline->add_element_to_pipeline(nms_elem.value());
+ async_pipeline->add_element_to_pipeline(nms_elem);
hailo_format_t nms_src_format = {};
nms_src_format.flags = HAILO_FORMAT_FLAGS_NONE;
// TODO (HRT-11052): Fix multi qp for NMS
auto stream_quant_infos = std::vector<hailo_quant_info_t>(1, curr_stream_info.quant_info); //output_stream_base->get_quant_infos();
- auto should_transform = OutputTransformContext::is_transformation_required(curr_stream_info.hw_shape, curr_stream_info.format,
- curr_stream_info.hw_shape, nms_src_format, stream_quant_infos);
- CHECK_EXPECTED_AS_STATUS(should_transform);
+ TRY(const auto should_transform,
+ OutputTransformContext::is_transformation_required(curr_stream_info.hw_shape, curr_stream_info.format,
+ curr_stream_info.hw_shape, nms_src_format, stream_quant_infos));
- CHECK(!(should_transform.value()), HAILO_INVALID_ARGUMENT, "Unexpected transformation required for {}", curr_stream_name);
+ CHECK(!(should_transform), HAILO_INVALID_ARGUMENT, "Unexpected transformation required for {}", curr_stream_name);
- auto source_id = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(curr_stream_name);
- CHECK_EXPECTED_AS_STATUS(source_id);
+ TRY(const auto source_id,
+ async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(curr_stream_name));
auto is_empty = false;
auto interacts_with_hw = true;
- auto nms_source_queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_nms", curr_stream_info.name,
+ TRY(auto nms_source_queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_nms", curr_stream_info.name,
curr_stream_info.index), async_pipeline, curr_stream_info.hw_frame_size, is_empty, interacts_with_hw,
- async_pipeline->get_async_hw_element(), source_id.value());
- CHECK_EXPECTED_AS_STATUS(nms_source_queue_elem);
+ async_pipeline->get_async_hw_element(), source_id));
- CHECK_SUCCESS(PipelinePad::link_pads(nms_source_queue_elem.value(), nms_elem.value(), 0, i));
- nms_elem.value()->add_sink_name(curr_stream_name);
+ CHECK_SUCCESS(PipelinePad::link_pads(nms_source_queue_elem, nms_elem, 0, i));
+ nms_elem->add_sink_name(curr_stream_name);
}
uint32_t post_transform_frame_size;
if(nms_op_metadata->nms_config().bbox_only){
} else {
post_transform_frame_size = HailoRTCommon::get_nms_host_frame_size(vstream_info.nms_shape, output_format.second);
}
- auto last_async_element = add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size,
- nms_elem.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element,
+ add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, nms_elem));
return HAILO_SUCCESS;
}
// TODO (HRT-11078): Fix multi qp for PP
auto stream_quant_infos = std::vector<hailo_quant_info_t>(1, output_stream_info.quant_info); //output_stream_base->get_quant_infos();
- auto post_infer_element = add_post_infer_element(output_format.second, output_stream_info.nms_info,
+ TRY(auto post_infer_element, add_post_infer_element(output_format.second, output_stream_info.nms_info,
async_pipeline, output_stream_info.hw_shape, output_stream_info.format, output_stream_info.shape, stream_quant_infos,
- async_pipeline->get_async_hw_element());
- CHECK_EXPECTED_AS_STATUS(post_infer_element);
+ async_pipeline->get_async_hw_element()));
auto is_empty = false;
auto interacts_with_hw = false;
const auto post_transform_frame_size = HailoRTCommon::get_nms_host_frame_size(output_stream_info.nms_info, output_format.second);
- auto pre_nms_convert_queue_element = add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_nms_convert", output_stream_name,
- output_stream_info.index), async_pipeline, post_transform_frame_size, is_empty, interacts_with_hw, post_infer_element.value());
- CHECK_EXPECTED_AS_STATUS(pre_nms_convert_queue_element);
+ TRY(auto pre_nms_convert_queue_element,
+ add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_nms_convert", output_stream_name,
+ output_stream_info.index), async_pipeline, post_transform_frame_size, is_empty, interacts_with_hw, post_infer_element));
- auto nms_to_detections_element = add_nms_to_detections_convert_element(async_pipeline, output_stream_name, output_stream_info.index,
- "NmsFormatToDetectionsEl", iou_op_metadata, pre_nms_convert_queue_element.value());
- CHECK_EXPECTED_AS_STATUS(nms_to_detections_element);
+ TRY(auto nms_to_detections_element,
+ add_nms_to_detections_convert_element(async_pipeline, output_stream_name, output_stream_info.index,
+ "NmsFormatToDetectionsEl", iou_op_metadata, pre_nms_convert_queue_element));
- auto pre_remove_overlapping_bboxes_element_queue_element = add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_bboxes_removing",
- output_stream_name, output_stream_info.index), async_pipeline, 0, is_empty, interacts_with_hw, nms_to_detections_element.value());
- CHECK_EXPECTED_AS_STATUS(pre_remove_overlapping_bboxes_element_queue_element);
+ TRY(auto pre_remove_overlapping_bboxes_element_queue_element,
+ add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_bboxes_removing",
+ output_stream_name, output_stream_info.index), async_pipeline, 0, is_empty, interacts_with_hw, nms_to_detections_element));
+
+ TRY(auto remove_overlapping_bboxes_element,
+ add_remove_overlapping_bboxes_element(async_pipeline, output_stream_name, output_stream_info.index,
+ "RemoveOverlappingBboxesEl", iou_op_metadata, pre_remove_overlapping_bboxes_element_queue_element));
- auto remove_overlapping_bboxes_element = add_remove_overlapping_bboxes_element(async_pipeline, output_stream_name, output_stream_info.index,
- "RemoveOverlappingBboxesEl", iou_op_metadata, pre_remove_overlapping_bboxes_element_queue_element.value());
- CHECK_EXPECTED_AS_STATUS(remove_overlapping_bboxes_element);
+ TRY(auto pre_fill_nms_format_element_queue_element,
+ add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_fill_nms_format",
+ output_stream_name, output_stream_info.index), async_pipeline, 0, is_empty, interacts_with_hw, remove_overlapping_bboxes_element));
- auto pre_fill_nms_format_element_queue_element = add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_fill_nms_format",
- output_stream_name, output_stream_info.index), async_pipeline, 0, is_empty, interacts_with_hw, remove_overlapping_bboxes_element.value());
- CHECK_EXPECTED_AS_STATUS(pre_fill_nms_format_element_queue_element);
+ TRY(auto fill_nms_format_element,
+ add_fill_nms_format_element(async_pipeline, output_stream_name, output_stream_info.index,
+ "FillNmsFormatEl", iou_op_metadata, pre_fill_nms_format_element_queue_element));
- auto fill_nms_format_element = add_fill_nms_format_element(async_pipeline, output_stream_name, output_stream_info.index,
- "FillNmsFormatEl", iou_op_metadata, pre_fill_nms_format_element_queue_element.value());
- CHECK_EXPECTED_AS_STATUS(fill_nms_format_element);
+ TRY(const auto output_vstream_info, iou_op_metadata->get_output_vstream_info());
+ const auto final_frame_size = HailoRTCommon::get_frame_size(output_vstream_info, output_format.second);
- auto output_vstream_info = iou_op_metadata->get_output_vstream_info();
- CHECK_EXPECTED_AS_STATUS(output_vstream_info);
- const auto final_frame_size = HailoRTCommon::get_frame_size(*output_vstream_info, output_format.second);
-
- auto last_async_element = add_last_async_element(async_pipeline, output_format.first, final_frame_size, fill_nms_format_element.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element,
+ add_last_async_element(async_pipeline, output_format.first, final_frame_size, fill_nms_format_element));
return HAILO_SUCCESS;
}
{
auto metadata = std::dynamic_pointer_cast<net_flow::YoloxOpMetadata>(op_metadata);
assert(nullptr != metadata);
- auto op_expected = net_flow::YOLOXPostProcessOp::create(metadata);
- CHECK_EXPECTED_AS_STATUS(op_expected);
- op = op_expected.release();
+ TRY(op, net_flow::YOLOXPostProcessOp::create(metadata));
break;
}
case net_flow::OperationType::YOLOV8:
{
auto metadata = std::dynamic_pointer_cast<net_flow::Yolov8OpMetadata>(op_metadata);
assert(nullptr != metadata);
- auto op_expected = net_flow::YOLOV8PostProcessOp::create(metadata);
- CHECK_EXPECTED_AS_STATUS(op_expected);
- op = op_expected.release();
- break;
+ if (metadata->nms_config().bbox_only) {
+ auto bbox_only_metadata = std::dynamic_pointer_cast<net_flow::Yolov8BboxOnlyOpMetadata>(op_metadata);
+ assert(nullptr != bbox_only_metadata);
+ TRY(op, net_flow::YOLOv8BboxOnlyPostProcessOp::create(bbox_only_metadata));
+ break;
+ } else {
+ TRY(op, net_flow::YOLOV8PostProcessOp::create(metadata));
+ break;
+ }
}
case net_flow::OperationType::YOLOV5:
{
if (metadata->nms_config().bbox_only) {
auto bbox_only_metadata = std::dynamic_pointer_cast<net_flow::Yolov5BboxOnlyOpMetadata>(op_metadata);
assert(nullptr != bbox_only_metadata);
- auto op_expected = net_flow::YOLOv5BboxOnlyPostProcessOp::create(bbox_only_metadata);
- CHECK_EXPECTED(op_expected);
- op = op_expected.release();
+ TRY(op, net_flow::YOLOv5BboxOnlyPostProcessOp::create(bbox_only_metadata));
break;
} else {
- auto op_expected = net_flow::YOLOv5PostProcessOp::create(metadata);
- CHECK_EXPECTED_AS_STATUS(op_expected);
- op = op_expected.release();
+ TRY(op, net_flow::YOLOv5PostProcessOp::create(metadata));
break;
}
}
{
auto metadata = std::dynamic_pointer_cast<net_flow::Yolov5SegOpMetadata>(op_metadata);
assert(nullptr != metadata);
- auto op_expected = net_flow::Yolov5SegPostProcess::create(metadata);
- CHECK_EXPECTED_AS_STATUS(op_expected);
- op = op_expected.release();
+ TRY(op, net_flow::Yolov5SegPostProcess::create(metadata));
break;
}
case net_flow::OperationType::SSD:
{
auto metadata = std::dynamic_pointer_cast<net_flow::SSDOpMetadata>(op_metadata);
assert(nullptr != metadata);
- auto op_expected = net_flow::SSDPostProcessOp::create(metadata);
- CHECK_EXPECTED_AS_STATUS(op_expected);
- op = op_expected.release();
+ TRY(op, net_flow::SSDPostProcessOp::create(metadata));
break;
}
default:
std::unordered_map<stream_name_t, op_name_t> op_inputs_to_op_name;
for (auto &metadata : net_group->get_ops_metadata().release()) {
post_process_metadata.insert({metadata->get_name(), metadata});
- for (auto &input_name : metadata->get_input_names()) {
+ for (const auto &input_name : metadata->get_input_names()) {
op_inputs_to_op_name.insert({input_name, metadata->get_name()});
}
}
for (auto &output_format : expanded_outputs_formats) {
- auto stream_names = net_group->get_stream_names_from_vstream_name(output_format.first);
- CHECK_EXPECTED_AS_STATUS(stream_names);
+ TRY(const auto stream_names, net_group->get_stream_names_from_vstream_name(output_format.first));
- if (contains(streams_added, *stream_names->begin())) {
+ if (contains(streams_added, *stream_names.begin())) {
continue;
}
- for (auto &output_name : stream_names.value()) {
+ for (auto &output_name : stream_names) {
streams_added.push_back(output_name);
}
- CHECK(contains(named_stream_infos, *stream_names->begin()), HAILO_INTERNAL_FAILURE);
- const auto &first_stream_info = named_stream_infos.at(*stream_names->begin());
+ CHECK(contains(named_stream_infos, *stream_names.begin()), HAILO_INTERNAL_FAILURE);
+ const auto &first_stream_info = named_stream_infos.at(*stream_names.begin());
- if (contains(op_inputs_to_op_name, *stream_names->begin())) {
- auto &op_name = op_inputs_to_op_name.at(*stream_names->begin());
+ if (contains(op_inputs_to_op_name, *stream_names.begin())) {
+ const auto &op_name = op_inputs_to_op_name.at(*stream_names.begin());
auto &op_metadata = post_process_metadata.at(op_name);
- auto output_vstreams_infos = net_group->get_output_vstream_infos();
- CHECK_EXPECTED_AS_STATUS(output_vstreams_infos);
+ TRY(const auto output_vstreams_infos, net_group->get_output_vstream_infos());
std::pair<std::string, hailo_format_t> original_output_format = {output_format.first, original_outputs_formats.at(output_format.first)};
hailo_status status = add_ops_flows(async_pipeline, original_output_format,
- op_metadata, stream_names.value(), output_vstreams_infos.value(), named_stream_infos);
+ op_metadata, stream_names, output_vstreams_infos, named_stream_infos);
CHECK_SUCCESS(status);
} else if ((HAILO_FORMAT_ORDER_HAILO_NMS == first_stream_info.format.order) &&
(first_stream_info.nms_info.is_defused)) {
// Case defuse NMS
- hailo_status status = add_nms_fuse_flow(stream_names.value(), output_format, async_pipeline, named_stream_infos);
+ hailo_status status = add_nms_fuse_flow(stream_names, output_format, async_pipeline, named_stream_infos);
CHECK_SUCCESS(status);
} else if (first_stream_info.is_mux) {
// case demux in output from NN core (only one output stream is currently supported)
- hailo_status status = add_output_demux_flow(*stream_names->begin(), async_pipeline, expanded_outputs_formats, net_group, named_stream_infos);
+ hailo_status status = add_output_demux_flow(*stream_names.begin(), async_pipeline, expanded_outputs_formats, net_group, named_stream_infos);
CHECK_SUCCESS(status);
} else {
// case simple and single output from NN core to user (and transformation at best)
- auto final_elem_source_index = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(*stream_names->begin());
- CHECK_EXPECTED_AS_STATUS(final_elem_source_index);
+ TRY(const auto final_elem_source_index,
+ async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(*stream_names.begin()));
- auto layer_info = net_group->get_layer_info(first_stream_info.name);
- CHECK_EXPECTED_AS_STATUS(layer_info);
- auto stream_quant_infos = layer_info.value()->quant_infos;
+ TRY(const auto layer_info, net_group->get_layer_info(first_stream_info.name));
+ auto stream_quant_infos = layer_info->quant_infos;
- auto should_transform_expected = should_transform(first_stream_info, stream_quant_infos, output_format.second);
- CHECK_EXPECTED_AS_STATUS(should_transform_expected);
+ TRY(auto should_transform,
+ should_transform(first_stream_info, stream_quant_infos, output_format.second));
- if (should_transform_expected.value()) {
- auto post_infer_elem = add_post_infer_element(output_format.second, first_stream_info.nms_info, async_pipeline, first_stream_info.hw_shape,
- first_stream_info.format, first_stream_info.shape, stream_quant_infos, async_pipeline->get_async_hw_element(), final_elem_source_index.value());
- CHECK_EXPECTED_AS_STATUS(post_infer_elem);
+ if (should_transform) {
+ TRY(auto post_infer_elem,
+ add_post_infer_element(output_format.second, first_stream_info.nms_info, async_pipeline, first_stream_info.hw_shape,
+ first_stream_info.format, first_stream_info.shape, stream_quant_infos, async_pipeline->get_async_hw_element(),
+ final_elem_source_index));
- auto post_transform_frame_size = (HailoRTCommon::is_nms(first_stream_info.format.order)) ?
+ const auto post_transform_frame_size = (HailoRTCommon::is_nms(first_stream_info.format.order)) ?
HailoRTCommon::get_nms_host_frame_size(first_stream_info.nms_info, output_format.second) :
HailoRTCommon::get_frame_size(first_stream_info.shape, output_format.second);
- auto last_async_element = add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size,
- post_infer_elem.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size,
+ post_infer_elem));
} else {
- auto last_async_element = add_last_async_element(async_pipeline, output_format.first, first_stream_info.hw_frame_size,
- async_pipeline->get_async_hw_element(), final_elem_source_index.value());
- CHECK_EXPECTED_AS_STATUS(last_async_element);
+ TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, first_stream_info.hw_frame_size,
+ async_pipeline->get_async_hw_element(), final_elem_source_index));
}
}
}
// Buffer pool sizes for pipeline elements should be:
// * The minimum of the maximum queue size of all LL streams (input and output) - for edge elements
// * HAILO_DEFAULT_ASYNC_INFER_QUEUE_SIZE - for internal elements
- auto buffer_pool_size_expected = net_group->get_min_buffer_pool_size();
- CHECK_EXPECTED(buffer_pool_size_expected);
- build_params.buffer_pool_size_edges = buffer_pool_size_expected.release();
- build_params.buffer_pool_size_internal = std::min(static_cast<uint32_t>(build_params.buffer_pool_size_edges), static_cast<uint32_t>(HAILO_DEFAULT_ASYNC_INFER_QUEUE_SIZE));
+ TRY(build_params.buffer_pool_size_edges, net_group->get_min_buffer_pool_size());
+ build_params.buffer_pool_size_internal = std::min(static_cast<uint32_t>(build_params.buffer_pool_size_edges),
+ static_cast<uint32_t>(HAILO_DEFAULT_ASYNC_INFER_QUEUE_SIZE));
build_params.elem_stats_flags = HAILO_PIPELINE_ELEM_STATS_NONE;
build_params.vstream_stats_flags = HAILO_VSTREAM_STATS_NONE;
- auto async_pipeline_expected = AsyncPipeline::create_shared();
- CHECK_EXPECTED(async_pipeline_expected);
- auto async_pipeline = async_pipeline_expected.release();
-
- auto all_stream_infos = net_group->get_all_stream_infos();
- CHECK_EXPECTED(all_stream_infos);
+ TRY(auto async_pipeline, AsyncPipeline::create_shared());
+ TRY(const auto all_stream_infos, net_group->get_all_stream_infos());
std::unordered_map<std::string, hailo_stream_info_t> named_stream_infos;
- for (const auto &info : all_stream_infos.value()) {
+ for (const auto &info : all_stream_infos) {
named_stream_infos.emplace(info.name, info);
}
- auto input_expanded_format = expand_auto_input_formats(net_group, inputs_formats, named_stream_infos);
- CHECK_EXPECTED(input_expanded_format);
-
- auto output_expanded_format = expand_auto_output_formats(net_group, outputs_formats, named_stream_infos);
- CHECK_EXPECTED(output_expanded_format);
-
+ TRY(const auto input_expanded_format, expand_auto_input_formats(net_group, inputs_formats, named_stream_infos));
+ TRY(const auto output_expanded_format, expand_auto_output_formats(net_group, outputs_formats, named_stream_infos));
auto outputs_original_formats = outputs_formats; // The original formats is needed for specific format expanding (required for PP OPs, like argmax)
- auto shutdown_event_expected = Event::create_shared(Event::State::not_signalled);
- CHECK_EXPECTED(shutdown_event_expected);
-
- build_params.shutdown_event = shutdown_event_expected.release();
+ TRY(build_params.shutdown_event, Event::create_shared(Event::State::not_signalled));
build_params.pipeline_status = pipeline_status;
build_params.timeout = std::chrono::milliseconds(timeout);
async_pipeline->set_build_params(build_params);
- auto async_hw_elem = AsyncHwElement::create(named_stream_infos, build_params.timeout,
+ TRY(auto async_hw_elem, AsyncHwElement::create(named_stream_infos, build_params.timeout,
build_params.elem_stats_flags, "AsyncHwEl", build_params.pipeline_status, net_group,
- PipelineDirection::PUSH, async_pipeline);
- CHECK_EXPECTED(async_hw_elem);
- async_pipeline->add_element_to_pipeline(async_hw_elem.value());
- async_pipeline->set_async_hw_element(async_hw_elem.release());
+ PipelineDirection::PUSH, async_pipeline));
+ async_pipeline->add_element_to_pipeline(async_hw_elem);
+ async_pipeline->set_async_hw_element(async_hw_elem);
- hailo_status status = create_pre_async_hw_elements(net_group, input_expanded_format.value(), named_stream_infos,
+ hailo_status status = create_pre_async_hw_elements(net_group, input_expanded_format, named_stream_infos,
async_pipeline);
CHECK_SUCCESS_AS_EXPECTED(status);
- status = create_post_async_hw_elements(net_group, output_expanded_format.value(), outputs_original_formats, named_stream_infos,
+ status = create_post_async_hw_elements(net_group, output_expanded_format, outputs_original_formats, named_stream_infos,
async_pipeline);
CHECK_SUCCESS_AS_EXPECTED(status);
HAILO_INVALID_ARGUMENT, "The given order ({}) is not a multi-planar order", HailoRTCommon::get_format_order_str(order));
// TODO: Support fps/latency collection for queue elems (HRT-7711)
- auto duration_collector_expected = DurationCollector::create(HAILO_PIPELINE_ELEM_STATS_NONE);
- CHECK_EXPECTED(duration_collector_expected);
+ TRY(auto duration_collector, DurationCollector::create(HAILO_PIPELINE_ELEM_STATS_NONE));
- auto planes_splitter = PixBufferElement::create(PipelineObject::create_element_name("PixBufEl",
- input_name, 0), std::chrono::milliseconds(HAILO_INFINITE), duration_collector_expected.release(), pipeline_status, order,
- async_pipeline);
- CHECK_EXPECTED(planes_splitter);
+ TRY(auto planes_splitter, PixBufferElement::create(PipelineObject::create_element_name("PixBufEl",
+ input_name, 0), std::chrono::milliseconds(HAILO_INFINITE), std::move(duration_collector), pipeline_status, order,
+ async_pipeline));
- return planes_splitter.release();
+ return planes_splitter;
}
} /* namespace hailort */
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file configured_infer_model_hrpc_client.cpp
+ * @brief ConfiguredInferModel HRPC client implementation
+ **/
+
+#include "configured_infer_model_hrpc_client.hpp"
+#include "hailo/hailort.h"
+#include <iostream>
+
+namespace hailort
+{
+
+Expected<MemoryView> InferStreamOnStack::get_buffer()
+{
+ return MemoryView(m_buffer);
+}
+
+Expected<OutputBindingsOnStack> OutputBindingsOnStack::create(ConfiguredInferModel::Bindings bindings,
+ const std::vector<std::string> &outputs_names)
+{
+ std::unordered_map<std::string, InferStreamOnStack> output_streams;
+ for (const auto &output_name : outputs_names) {
+ TRY(auto output, bindings.output(output_name));
+ TRY(auto buffer, output.get_buffer());
+ output_streams.emplace(output_name, InferStreamOnStack(buffer));
+ }
+ return OutputBindingsOnStack(std::move(output_streams));
+}
+
+Expected<InferStreamOnStack> OutputBindingsOnStack::output()
+{
+ CHECK_AS_EXPECTED(1 == m_output_streams.size(), HAILO_INVALID_OPERATION, "Model has more than one output!");
+ auto copy = m_output_streams.begin()->second;
+ return copy;
+}
+
+Expected<InferStreamOnStack> OutputBindingsOnStack::output(const std::string &name)
+{
+ CHECK_AS_EXPECTED(contains(m_output_streams, name), HAILO_NOT_FOUND, "Output {}, not found!", name);
+ auto copy = m_output_streams.at(name);
+ return copy;
+}
+
+AsyncInferJobHrpcClient::AsyncInferJobHrpcClient(EventPtr event) : m_event(event)
+{
+}
+
+hailo_status AsyncInferJobHrpcClient::wait(std::chrono::milliseconds timeout)
+{
+ return m_event->wait(timeout);
+}
+
+CallbacksQueue::CallbacksQueue(std::shared_ptr<hrpc::Client> client, const std::vector<std::string> &outputs_names) :
+ m_outputs_names(outputs_names)
+{
+ client->register_custom_reply(HailoRpcActionID::CALLBACK_CALLED,
+ [this, &outputs_names] (const MemoryView &serialized_reply, hrpc::RpcConnection connection) -> hailo_status {
+ TRY(auto tuple, CallbackCalledSerializer::deserialize_reply(serialized_reply));
+
+ auto callback_status = std::get<0>(tuple);
+ auto callback_handle_id = std::get<1>(tuple);
+
+ {
+ std::unique_lock<std::mutex> lock(m_mutex);
+ CHECK(contains(m_callbacks, callback_handle_id), HAILO_NOT_FOUND, "Callback handle not found!");
+ m_callbacks_status[callback_handle_id] = callback_status;
+
+ if (HAILO_SUCCESS == callback_status) {
+ CHECK(contains(m_bindings, callback_handle_id), HAILO_NOT_FOUND, "Callback handle not found!");
+ for (const auto &output_name : outputs_names) {
+ TRY(auto buffer, m_bindings.at(callback_handle_id).output(output_name)->get_buffer());
+ auto status = connection.read_buffer(buffer);
+ // TODO: Errors here should be unrecoverable (HRT-14275)
+ CHECK_SUCCESS(status);
+ }
+ }
+ m_callbacks_queue.push(callback_handle_id);
+ }
+
+ m_cv.notify_one();
+ return HAILO_SUCCESS;
+ });
+
+ m_is_running = true;
+ m_callback_thread = std::thread([this] {
+ while (true) {
+ callback_id_t callback_id;
+ hailo_status info_status = HAILO_UNINITIALIZED;
+ std::function<void(const AsyncInferCompletionInfo&)> cb;
+ {
+ std::unique_lock<std::mutex> lock(m_mutex);
+ m_cv.wait(lock, [this] { return !m_is_running || !m_callbacks_queue.empty(); });
+ if (!m_is_running) {
+ break;
+ }
+
+ callback_id = m_callbacks_queue.front();
+ m_callbacks_queue.pop();
+
+ m_cv.wait(lock, [this, callback_id] { return !m_is_running || (m_callbacks.find(callback_id) != m_callbacks.end()); });
+ if (!m_is_running) {
+ break;
+ }
+
+ info_status = m_callbacks_status[callback_id];
+ cb = m_callbacks[callback_id];
+ m_callbacks.erase(callback_id);
+ m_callbacks_status.erase(callback_id);
+ m_bindings.erase(callback_id);
+ }
+ AsyncInferCompletionInfo info(info_status);
+ cb(info);
+ }
+ });
+}
+
+CallbacksQueue::~CallbacksQueue()
+{
+ {
+ std::unique_lock<std::mutex> lock(m_mutex);
+ m_is_running = false;
+ }
+ m_cv.notify_one();
+ m_callback_thread.join();
+}
+Expected<std::shared_ptr<AsyncInferJobHrpcClient>> CallbacksQueue::register_callback(callback_id_t id,
+ ConfiguredInferModel::Bindings bindings,
+ std::function<void(const AsyncInferCompletionInfo&)> callback)
+{
+ TRY(auto event_ptr, Event::create_shared(Event::State::not_signalled));
+
+ {
+ std::unique_lock<std::mutex> lock(m_mutex);
+ TRY(auto output_bindings, OutputBindingsOnStack::create(bindings, m_outputs_names));
+ m_bindings.emplace(id, output_bindings);
+ m_callbacks_status[id] = HAILO_SUCCESS;
+ m_callbacks[id] = [callback, event_ptr] (const AsyncInferCompletionInfo &info) {
+ auto status = event_ptr->signal();
+ if (HAILO_SUCCESS != status) {
+ LOGGER__CRITICAL("Could not signal event! status = {}", status);
+ }
+ callback(info);
+ };
+ }
+ m_cv.notify_one();
+
+ auto ptr = make_shared_nothrow<AsyncInferJobHrpcClient>(event_ptr);
+ CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return ptr;
+}
+
+Expected<std::shared_ptr<ConfiguredInferModelHrpcClient>> ConfiguredInferModelHrpcClient::create(std::shared_ptr<hrpc::Client> client,
+ rpc_object_handle_t handle_id, std::vector<hailo_vstream_info_t> &&input_vstream_infos,
+ std::vector<hailo_vstream_info_t> &&output_vstream_infos, uint32_t max_ongoing_transfers,
+ std::unique_ptr<CallbacksQueue> &&callbacks_queue, rpc_object_handle_t infer_model_id,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes,
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes)
+{
+ // TODO: consider create a separate client object here - HRT-13687
+ auto ptr = make_shared_nothrow<ConfiguredInferModelHrpcClient>(client, handle_id, std::move(input_vstream_infos),
+ std::move(output_vstream_infos), max_ongoing_transfers, std::move(callbacks_queue), infer_model_id, inputs_frame_sizes,
+ outputs_frame_sizes);
+ CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return ptr;
+}
+
+ConfiguredInferModelHrpcClient::~ConfiguredInferModelHrpcClient()
+{
+ if (INVALID_HANDLE_ID == m_handle_id) {
+ return;
+ }
+
+ auto request = DestroyConfiguredInferModelSerializer::serialize_request(m_handle_id);
+ if (!request) {
+ LOGGER__CRITICAL("Failed to serialize ConfiguredInferModel_release request");
+ return;
+ }
+
+ auto client = m_client.lock();
+ if (client) {
+ auto result = client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__DESTROY, MemoryView(*request));
+ if (!result) {
+ LOGGER__CRITICAL("Failed to destroy configured infer model! status = {}", result.status());
+ }
+
+ if (HAILO_SUCCESS != DestroyConfiguredInferModelSerializer::deserialize_reply(MemoryView(*result))) {
+ LOGGER__CRITICAL("Failed to destroy configured infer model! status = {}", result.status());
+ }
+ }
+}
+
+Expected<ConfiguredInferModel::Bindings> ConfiguredInferModelHrpcClient::create_bindings()
+{
+ std::unordered_map<std::string, ConfiguredInferModel::Bindings::InferStream> inputs;
+ std::unordered_map<std::string, ConfiguredInferModel::Bindings::InferStream> outputs;
+
+ for (const auto &vstream_info : m_input_vstream_infos) {
+ TRY(auto stream, ConfiguredInferModelBase::create_infer_stream(vstream_info));
+ inputs.emplace(vstream_info.name, std::move(stream));
+ }
+
+ for (const auto &vstream_info : m_output_vstream_infos) {
+ TRY(auto stream, ConfiguredInferModelBase::create_infer_stream(vstream_info));
+ outputs.emplace(vstream_info.name, std::move(stream));
+ }
+
+ TRY(auto bindings, ConfiguredInferModelBase::create_bindings(std::move(inputs), std::move(outputs)));
+ return bindings;
+}
+
+hailo_status ConfiguredInferModelHrpcClient::wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count)
+{
+ std::unique_lock<std::mutex> lock(m_ongoing_transfers_mutex);
+ bool done = m_cv.wait_for(lock, timeout, [this, frames_count] () {
+ return (m_max_ongoing_transfers - m_ongoing_transfers.load()) >= frames_count;
+ });
+ CHECK(done, HAILO_TIMEOUT);
+
+ return HAILO_SUCCESS;
+}
+
+Expected<AsyncInferJob> ConfiguredInferModelHrpcClient::run_async(ConfiguredInferModel::Bindings bindings,
+ std::function<void(const AsyncInferCompletionInfo &)> callback)
+{
+ auto async_job = run_async_impl(bindings, callback);
+ if (HAILO_SUCCESS != async_job.status()) {
+ shutdown();
+ return make_unexpected(async_job.status());
+ }
+ return async_job.release();
+}
+
+Expected<AsyncInferJob> ConfiguredInferModelHrpcClient::run_async_impl(ConfiguredInferModel::Bindings bindings,
+ std::function<void(const AsyncInferCompletionInfo &)> callback)
+{
+ CHECK_SUCCESS_AS_EXPECTED(validate_bindings(bindings));
+ std::unique_lock<std::mutex> lock(m_infer_mutex);
+ m_callbacks_counter++;
+ auto callback_wrapper = [this, callback] (const AsyncInferCompletionInfo &info) {
+ {
+ std::unique_lock<std::mutex> transfers_lock(m_ongoing_transfers_mutex);
+ m_ongoing_transfers--;
+ }
+ m_cv.notify_one();
+ if (callback) {
+ callback(info);
+ }
+ };
+
+ TRY(auto job_ptr, m_callbacks_queue->register_callback(m_callbacks_counter, bindings, callback_wrapper));
+
+ TRY(auto request, RunAsyncSerializer::serialize_request(m_handle_id, m_infer_model_handle_id,
+ m_callbacks_counter));
+
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use.");
+ TRY(auto serialized_result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__RUN_ASYNC,
+ MemoryView(request), [this, &bindings] (hrpc::RpcConnection connection) -> hailo_status {
+ for (const auto &input_vstream : m_input_vstream_infos) {
+ TRY(auto input, bindings.input(input_vstream.name));
+ auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(input);
+ switch(buffer_type) {
+ case BufferType::VIEW:
+ {
+ TRY(auto buffer, input.get_buffer());
+ auto status = connection.write_buffer(MemoryView(buffer));
+ CHECK_SUCCESS(status);
+ break;
+ }
+ case BufferType::PIX_BUFFER:
+ {
+ TRY(auto pix_buffer, input.get_pix_buffer());
+ for (uint32_t i = 0; i < pix_buffer.number_of_planes; i++) {
+ auto status = connection.write_buffer(MemoryView(pix_buffer.planes[i].user_ptr, pix_buffer.planes[i].bytes_used));
+ CHECK_SUCCESS(status);
+ }
+ break;
+ }
+ case BufferType::DMA_BUFFER:
+ LOGGER__CRITICAL("DMA_BUFFER is not supported in HRPC");
+ return HAILO_NOT_IMPLEMENTED;
+ default:
+ LOGGER__CRITICAL("Unknown buffer type");
+ return HAILO_INTERNAL_FAILURE;
+ }
+ }
+ return HAILO_SUCCESS;
+ }));
+ auto status = RunAsyncSerializer::deserialize_reply(MemoryView(serialized_result));
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ {
+ std::unique_lock<std::mutex> transfers_lock(m_ongoing_transfers_mutex);
+ m_ongoing_transfers++;
+ }
+
+ return AsyncInferJobBase::create(job_ptr);
+}
+
+hailo_status ConfiguredInferModelHrpcClient::set_scheduler_timeout(const std::chrono::milliseconds &timeout)
+{
+ TRY(auto serialized_request, SetSchedulerTimeoutSerializer::serialize_request(m_handle_id, timeout));
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use.");
+ TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_TIMEOUT, MemoryView(serialized_request)));
+ CHECK_SUCCESS(SetSchedulerTimeoutSerializer::deserialize_reply(MemoryView(result)));
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status ConfiguredInferModelHrpcClient::set_scheduler_threshold(uint32_t threshold)
+{
+ TRY(auto serialized_request, SetSchedulerThresholdSerializer::serialize_request(m_handle_id, threshold));
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use.");
+ TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_THRESHOLD, MemoryView(serialized_request)));
+ CHECK_SUCCESS(SetSchedulerThresholdSerializer::deserialize_reply(MemoryView(result)));
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status ConfiguredInferModelHrpcClient::set_scheduler_priority(uint8_t priority)
+{
+ TRY(auto serialized_request, SetSchedulerPrioritySerializer::serialize_request(m_handle_id, priority));
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use.");
+ TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_PRIORITY, MemoryView(serialized_request)));
+ CHECK_SUCCESS(SetSchedulerPrioritySerializer::deserialize_reply(MemoryView(result)));
+
+ return HAILO_SUCCESS;
+}
+
+Expected<LatencyMeasurementResult> ConfiguredInferModelHrpcClient::get_hw_latency_measurement()
+{
+ TRY(auto serialized_request, GetHwLatencyMeasurementSerializer::serialize_request(m_handle_id));
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use.");
+ TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__GET_HW_LATENCY_MEASUREMENT, MemoryView(serialized_request)));
+
+ TRY(auto tuple, GetHwLatencyMeasurementSerializer::deserialize_reply(MemoryView(result)));
+
+ auto status = std::get<0>(tuple);
+ if (HAILO_NOT_AVAILABLE == status) {
+ return make_unexpected(HAILO_NOT_AVAILABLE);
+ }
+ CHECK_SUCCESS(status);
+
+ auto avg_hw_latency = std::get<1>(tuple);
+ LatencyMeasurementResult latency_measurement_result {avg_hw_latency};
+
+ return latency_measurement_result;
+};
+
+hailo_status ConfiguredInferModelHrpcClient::activate()
+{
+ TRY(auto serialized_request, ActivateSerializer::serialize_request(m_handle_id));
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use.");
+ TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__ACTIVATE, MemoryView(serialized_request)));
+
+ CHECK_SUCCESS(ActivateSerializer::deserialize_reply(MemoryView(result)));
+
+ return HAILO_SUCCESS;
+};
+
+hailo_status ConfiguredInferModelHrpcClient::deactivate()
+{
+ TRY(auto serialized_request, DeactivateSerializer::serialize_request(m_handle_id));
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use.");
+ TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__DEACTIVATE, MemoryView(serialized_request)));
+
+ CHECK_SUCCESS(DeactivateSerializer::deserialize_reply(MemoryView(result)));
+
+ return HAILO_SUCCESS;
+};
+
+Expected<size_t> ConfiguredInferModelHrpcClient::get_async_queue_size()
+{
+ size_t queue_size = m_max_ongoing_transfers;
+ return queue_size;
+}
+
+hailo_status ConfiguredInferModelHrpcClient::validate_bindings(ConfiguredInferModel::Bindings bindings)
+{
+ for (const auto &input_vstream : m_input_vstream_infos) {
+ TRY(auto input, bindings.input(input_vstream.name));
+
+ auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(input);
+ switch (buffer_type) {
+ case BufferType::VIEW:
+ {
+ auto buffer = input.get_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ CHECK(buffer->size() == m_inputs_frame_sizes.at(input_vstream.name), HAILO_INVALID_OPERATION,
+ "Input buffer size {} is different than expected {} for input '{}'", buffer->size(), m_inputs_frame_sizes.at(input_vstream.name), input_vstream.name);
+ break;
+ }
+ case BufferType::PIX_BUFFER:
+ {
+ auto buffer = input.get_pix_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ size_t buffer_size = 0;
+ for (size_t i = 0 ; i < buffer->number_of_planes ; i++) {
+ buffer_size += buffer->planes[i].bytes_used;
+ }
+
+ CHECK(buffer_size == m_inputs_frame_sizes.at(input_vstream.name), HAILO_INVALID_OPERATION,
+ "Input buffer size {} is different than expected {} for input '{}'", buffer_size, m_inputs_frame_sizes.at(input_vstream.name), input_vstream.name);
+ break;
+ }
+ case BufferType::DMA_BUFFER:
+ {
+ auto buffer = input.get_dma_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ CHECK(buffer->size == m_inputs_frame_sizes.at(input_vstream.name), HAILO_INVALID_OPERATION,
+ "Input buffer size {} is different than expected {} for input '{}'", buffer->size, m_inputs_frame_sizes.at(input_vstream.name), input_vstream.name);
+ break;
+ }
+ default:
+ CHECK(false, HAILO_NOT_FOUND, "Couldnt find input buffer for '{}'", input_vstream.name);
+ }
+ }
+ for (const auto &output_vstream : m_output_vstream_infos) {
+ TRY(auto output, bindings.output(output_vstream.name));
+ auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(output);
+ switch (buffer_type) {
+ case BufferType::VIEW:
+ {
+ auto buffer = output.get_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ CHECK(buffer->size() == m_outputs_frame_sizes.at(output_vstream.name), HAILO_INVALID_OPERATION,
+ "Output buffer size {} is different than expected {} for output '{}'", buffer->size(), m_outputs_frame_sizes.at(output_vstream.name), output_vstream.name);
+ break;
+ }
+ case BufferType::PIX_BUFFER:
+ {
+ CHECK(false, HAILO_NOT_SUPPORTED, "pix_buffer isn't supported for outputs in '{}'", output_vstream.name);
+ break;
+ }
+ case BufferType::DMA_BUFFER:
+ {
+ auto buffer = output.get_dma_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ CHECK(buffer->size == m_outputs_frame_sizes.at(output_vstream.name), HAILO_INVALID_OPERATION,
+ "Output buffer size {} is different than expected {} for out '{}'", buffer->size, m_outputs_frame_sizes.at(output_vstream.name), output_vstream.name);
+ break;
+ }
+ default:
+ CHECK(false, HAILO_NOT_FOUND, "Couldnt find output buffer for '{}'", output_vstream.name);
+ }
+ }
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status ConfiguredInferModelHrpcClient::shutdown()
+{
+ TRY(auto serialized_request, ShutdownSerializer::serialize_request(m_handle_id));
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use.");
+ TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__SHUTDOWN, MemoryView(serialized_request)));
+
+ CHECK_SUCCESS(ShutdownSerializer::deserialize_reply(MemoryView(result)));
+
+ return HAILO_SUCCESS;
+}
+
+
+} // namespace hailort
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file configured_infer_model_hrpc_client.hpp
+ * @brief ConfiguredInferModel HRPC client, represents the user's handle to the ConfiguredInferModel object (held in the hailort server)
+ **/
+
+#ifndef _HAILO_CONFIGURED_INFER_MODEL_HRPC_CLIENT_HPP_
+#define _HAILO_CONFIGURED_INFER_MODEL_HRPC_CLIENT_HPP_
+
+#include "hailo/infer_model.hpp"
+#include "infer_model_internal.hpp"
+#include "hrpc/client.hpp"
+
+namespace hailort
+{
+
+using callback_id_t = uint32_t;
+
+class InferStreamOnStack final
+{
+public:
+ InferStreamOnStack(MemoryView buffer) : m_buffer(buffer) {}
+ Expected<MemoryView> get_buffer();
+
+private:
+ MemoryView m_buffer;
+};
+
+class OutputBindingsOnStack final
+{
+public:
+ static Expected<OutputBindingsOnStack> create(ConfiguredInferModel::Bindings bindings,
+ const std::vector<std::string> &outputs_names);
+ Expected<InferStreamOnStack> output();
+ Expected<InferStreamOnStack> output(const std::string &name);
+
+private:
+ OutputBindingsOnStack(std::unordered_map<std::string, InferStreamOnStack> &&output_streams) :
+ m_output_streams(std::move(output_streams)) {}
+
+ std::unordered_map<std::string, InferStreamOnStack> m_output_streams;
+};
+
+class AsyncInferJobHrpcClient : public AsyncInferJobBase
+{
+public:
+ AsyncInferJobHrpcClient(EventPtr event);
+
+ virtual hailo_status wait(std::chrono::milliseconds timeout) override;
+
+private:
+ EventPtr m_event;
+};
+
+class CallbacksQueue
+{
+public:
+ CallbacksQueue(std::shared_ptr<hrpc::Client> client, const std::vector<std::string> &outputs_names);
+ ~CallbacksQueue();
+
+ CallbacksQueue(const CallbacksQueue &other) = delete;
+ CallbacksQueue& operator=(const CallbacksQueue &other) = delete;
+ CallbacksQueue(CallbacksQueue &&other) = delete;
+ CallbacksQueue& operator=(CallbacksQueue &&other) = delete;
+
+ Expected<std::shared_ptr<AsyncInferJobHrpcClient>> register_callback(callback_id_t id,
+ ConfiguredInferModel::Bindings bindings,
+ std::function<void(const AsyncInferCompletionInfo&)> callback);
+
+private:
+ const std::vector<std::string> m_outputs_names;
+ std::mutex m_mutex;
+ std::condition_variable m_cv;
+ std::queue<callback_id_t> m_callbacks_queue;
+ std::unordered_map<callback_id_t, std::function<void(const AsyncInferCompletionInfo&)>> m_callbacks;
+ std::atomic_bool m_is_running;
+ std::thread m_callback_thread;
+ std::unordered_map<callback_id_t, OutputBindingsOnStack> m_bindings;
+ std::unordered_map<callback_id_t, hailo_status> m_callbacks_status;
+};
+
+class ConfiguredInferModelHrpcClient : public ConfiguredInferModelBase
+{
+public:
+ static Expected<std::shared_ptr<ConfiguredInferModelHrpcClient>> create(std::shared_ptr<hrpc::Client> client,
+ rpc_object_handle_t handle_id, std::vector<hailo_vstream_info_t> &&input_vstream_infos,
+ std::vector<hailo_vstream_info_t> &&output_vstream_infos, uint32_t max_ongoing_transfers,
+ std::unique_ptr<CallbacksQueue> &&callbacks_queue, rpc_object_handle_t infer_model_handle_id,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes,
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes);
+ ConfiguredInferModelHrpcClient(std::shared_ptr<hrpc::Client> client, rpc_object_handle_t handle_id,
+ std::vector<hailo_vstream_info_t> &&input_vstream_infos, std::vector<hailo_vstream_info_t> &&output_vstream_infos,
+ uint32_t max_ongoing_transfers, std::unique_ptr<CallbacksQueue> &&callbacks_queue, rpc_object_handle_t infer_model_handle_id,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes,
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes) :
+ ConfiguredInferModelBase(inputs_frame_sizes, outputs_frame_sizes),
+ m_client(client), m_handle_id(handle_id), m_input_vstream_infos(std::move(input_vstream_infos)),
+ m_output_vstream_infos(std::move(output_vstream_infos)), m_max_ongoing_transfers(max_ongoing_transfers),
+ m_ongoing_transfers(0), m_callbacks_queue(std::move(callbacks_queue)), m_infer_model_handle_id(infer_model_handle_id),
+ m_callbacks_counter(0) {}
+ virtual ~ConfiguredInferModelHrpcClient();
+
+ ConfiguredInferModelHrpcClient(const ConfiguredInferModelHrpcClient &) = delete;
+ ConfiguredInferModelHrpcClient &operator=(const ConfiguredInferModelHrpcClient &) = delete;
+ ConfiguredInferModelHrpcClient(ConfiguredInferModelHrpcClient &&) = delete;
+ ConfiguredInferModelHrpcClient &operator=(ConfiguredInferModelHrpcClient &&) = delete;
+
+ virtual Expected<ConfiguredInferModel::Bindings> create_bindings() override;
+ virtual hailo_status wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count) override;
+
+ virtual hailo_status activate() override;
+ virtual hailo_status deactivate() override;
+
+ virtual Expected<AsyncInferJob> run_async(ConfiguredInferModel::Bindings bindings,
+ std::function<void(const AsyncInferCompletionInfo &)> callback) override;
+
+ virtual Expected<LatencyMeasurementResult> get_hw_latency_measurement() override;
+
+ virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout) override;
+ virtual hailo_status set_scheduler_threshold(uint32_t threshold) override;
+ virtual hailo_status set_scheduler_priority(uint8_t priority) override;
+
+ virtual Expected<size_t> get_async_queue_size() override;
+
+ virtual hailo_status shutdown() override;
+
+private:
+ virtual hailo_status validate_bindings(ConfiguredInferModel::Bindings bindings);
+ Expected<AsyncInferJob> run_async_impl(ConfiguredInferModel::Bindings bindings,
+ std::function<void(const AsyncInferCompletionInfo &)> callback);
+
+ std::weak_ptr<hrpc::Client> m_client;
+ rpc_object_handle_t m_handle_id;
+ std::vector<hailo_vstream_info_t> m_input_vstream_infos;
+ std::vector<hailo_vstream_info_t> m_output_vstream_infos;
+ uint32_t m_max_ongoing_transfers;
+ std::mutex m_ongoing_transfers_mutex;
+ std::condition_variable m_cv;
+ std::atomic_uint32_t m_ongoing_transfers;
+ std::unique_ptr<CallbacksQueue> m_callbacks_queue;
+ rpc_object_handle_t m_infer_model_handle_id;
+ std::atomic_uint32_t m_callbacks_counter;
+ std::mutex m_infer_mutex;
+};
+
+} /* namespace hailort */
+
+#endif /* _HAILO_CONFIGURED_INFER_MODEL_HRPC_CLIENT_HPP_ */
hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr<std::atomic<hailo_status>> pipeline_status,
PipelineDirection pipeline_direction)
{
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
-
- auto got_flush_event = Event::create_shared(Event::State::not_signalled);
- CHECK_EXPECTED(got_flush_event);
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
+ TRY(auto got_flush_event, Event::create_shared(Event::State::not_signalled));
// On HwWriteElement the stream always owns the buffer, hence, we set the mode explicitly.
auto status = stream->set_buffer_mode(StreamBufferMode::OWNING);
CHECK_SUCCESS_AS_EXPECTED(status);
auto hw_write_elem_ptr = make_shared_nothrow<HwWriteElement>(stream, name,
- duration_collector.release(), std::move(pipeline_status), got_flush_event.release(), pipeline_direction);
+ std::move(duration_collector), std::move(pipeline_status), std::move(got_flush_event), pipeline_direction);
CHECK_AS_EXPECTED(nullptr != hw_write_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", hw_write_elem_ptr->description());
hailo_vstream_stats_flags_t vstream_stats_flags, std::shared_ptr<std::atomic<hailo_status>> pipeline_status, size_t queue_size,
size_t frame_size, EventPtr shutdown_event, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
auto is_empty = true; // LastAsync always holds user buffers, therefore its created empty
auto is_dma_able = false;
queue_size = queue_size * 2; // Multiplying by 2 to ensure dual-buffering when edge-element is the bottleneck
- auto buffer_pool = BufferPool::create(frame_size, queue_size, shutdown_event, elem_flags, vstream_stats_flags, is_empty, is_dma_able);
- CHECK_EXPECTED(buffer_pool);
+ TRY(auto buffer_pool,
+ BufferPool::create(frame_size, queue_size, shutdown_event, elem_flags, vstream_stats_flags, is_empty, is_dma_able));
auto last_async_elem_ptr = make_shared_nothrow<LastAsyncElement>(name,
- duration_collector.release(), std::move(pipeline_status), buffer_pool.release(), async_pipeline);
+ std::move(duration_collector), std::move(pipeline_status), std::move(buffer_pool), async_pipeline);
CHECK_NOT_NULL_AS_EXPECTED(last_async_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", last_async_elem_ptr->description());
return HAILO_SUCCESS;
}
-hailo_status LastAsyncElement::enqueue_execution_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done)
+hailo_status LastAsyncElement::enqueue_execution_buffer(PipelineBuffer &&pipeline_buffer)
{
- return m_pool->enqueue_buffer(mem_view, exec_done);
+ return m_pool->enqueue_buffer(std::move(pipeline_buffer));
}
-Expected<bool> LastAsyncElement::can_push_buffer_upstream()
+Expected<bool> LastAsyncElement::can_push_buffer_upstream(uint32_t frames_count)
{
- return !m_pool->is_full();
+ return (m_pool->num_of_buffers_in_pool() + frames_count) < m_pool->max_capacity();
}
SourceElement::SourceElement(const std::string &name, DurationCollector &&duration_collector, std::shared_ptr<std::atomic<hailo_status>> &&pipeline_status,
auto status = stream->set_buffer_mode(StreamBufferMode::OWNING);
CHECK_SUCCESS_AS_EXPECTED(status);
- auto duration_collector = DurationCollector::create(build_params.elem_stats_flags);
- CHECK_EXPECTED(duration_collector);
+ TRY(auto duration_collector, DurationCollector::create(build_params.elem_stats_flags));
auto pipeline_status = build_params.pipeline_status;
- auto shutdown_event = Event::create_shared(Event::State::not_signalled);
- CHECK_EXPECTED(shutdown_event);
+ TRY(auto shutdown_event, Event::create_shared(Event::State::not_signalled));
auto hw_read_elem_ptr = make_shared_nothrow<HwReadElement>(stream, name, build_params.timeout,
- duration_collector.release(), shutdown_event.release(), std::move(pipeline_status), pipeline_direction);
+ std::move(duration_collector), std::move(shutdown_event), std::move(pipeline_status), pipeline_direction);
CHECK_AS_EXPECTED(nullptr != hw_read_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", hw_read_elem_ptr->description());
auto pool = next_pad_downstream().element().get_buffer_pool();
assert(pool);
- auto buffer = pool->get_available_buffer(std::move(optional), m_timeout);
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == buffer.status()) {
- return make_unexpected(buffer.status());
- }
- CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status());
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto buffer,
+ pool->get_available_buffer(std::move(optional), m_timeout),
+ "{} (D2H) failed.", name());
while (true) {
if (!m_stream->is_scheduled()) {
}
}
- MemoryView buffer_view(buffer.value().as_view());
+ TRY(MemoryView buffer_view, buffer.as_view(BufferProtection::NONE));
+
m_duration_collector.start_measurement();
auto status = m_stream->read(buffer_view);
if (HAILO_INVALID_FRAME == status) {
CHECK_SUCCESS_AS_EXPECTED(status, "{} (D2H) failed with status={}", name(), status);
m_duration_collector.complete_measurement();
- return buffer.release();
+ return buffer;
}
}
virtual Expected<PipelineBuffer> run_pull(PipelineBuffer &&optional, const PipelinePad &source) override;
virtual hailo_status execute_activate() override;
- virtual hailo_status enqueue_execution_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done) override;
+ virtual hailo_status enqueue_execution_buffer(PipelineBuffer &&pipeline_buffer) override;
- virtual Expected<bool> can_push_buffer_upstream() override;
+ virtual Expected<bool> can_push_buffer_upstream(uint32_t frames_count) override;
virtual hailo_status execute_post_deactivate(bool /*should_clear_abort*/) override { return HAILO_SUCCESS; };
virtual hailo_status execute_deactivate() override { return HAILO_SUCCESS; };
hailo_status FilterElement::run_push(PipelineBuffer &&buffer, const PipelinePad &/*sink*/)
{
- auto output = action(std::move(buffer), PipelineBuffer());
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == output.status()) {
- return output.status();
- }
- CHECK_EXPECTED_AS_STATUS(output);
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto output,
+ action(std::move(buffer), PipelineBuffer()));
- hailo_status status = next_pad().run_push(output.release());
+ hailo_status status = next_pad().run_push(std::move(output));
if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) {
LOGGER__INFO("run_push of {} was shutdown!", name());
return status;
Expected<PipelineBuffer> FilterElement::run_pull(PipelineBuffer &&optional, const PipelinePad &/*source*/)
{
- auto buffer = next_pad().run_pull();
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == buffer.status()) {
- LOGGER__INFO("run_pull in FilterElement was shutdown!");
- return make_unexpected(buffer.status());
- }
- CHECK_EXPECTED(buffer);
- return action(buffer.release(), std::move(optional));
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto buffer,
+ next_pad().run_pull());
+ return action(std::move(buffer), std::move(optional));
}
PipelinePad &FilterElement::next_pad_downstream()
const std::string &name, std::chrono::milliseconds timeout, hailo_pipeline_elem_stats_flags_t elem_flags,
std::shared_ptr<std::atomic<hailo_status>> pipeline_status, PipelineDirection pipeline_direction, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto transform_context = InputTransformContext::create(src_image_shape, src_format, dst_image_shape, dst_format,
- dst_quant_infos);
- CHECK_EXPECTED(transform_context, "Failed Creating InputTransformContext");
-
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
+ TRY(auto transform_context,
+ InputTransformContext::create(src_image_shape, src_format, dst_image_shape, dst_format,
+ dst_quant_infos), "Failed Creating InputTransformContext");
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
- auto pre_infer_elem_ptr = make_shared_nothrow<PreInferElement>(transform_context.release(),
- name, timeout, duration_collector.release(), std::move(pipeline_status), pipeline_direction,
+ auto pre_infer_elem_ptr = make_shared_nothrow<PreInferElement>(std::move(transform_context),
+ name, timeout, std::move(duration_collector), std::move(pipeline_status), pipeline_direction,
async_pipeline);
CHECK_AS_EXPECTED(nullptr != pre_infer_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
}
CHECK_AS_EXPECTED(HAILO_TIMEOUT != transformed_buffer.status(), HAILO_TIMEOUT,
"{} (H2D) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_timeout.count());
- CHECK_EXPECTED(transformed_buffer);
+ CHECK_EXPECTED(transformed_buffer); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
+
+ TRY(auto dst, transformed_buffer->as_view(BufferProtection::WRITE));
+ TRY(auto src, input.as_view(BufferProtection::READ));
- auto dst = transformed_buffer->as_view();
m_duration_collector.start_measurement();
- const auto status = m_transform_context->transform(input.as_view(), dst);
+ const auto status = m_transform_context->transform(src, dst);
m_duration_collector.complete_measurement();
input.set_action_status(status);
CHECK_SUCCESS_AS_EXPECTED(status);
// Note: The latency to be measured starts as the input buffer is sent to the InputVStream (via write())
- transformed_buffer->set_metadata(std::move(metadata));
+ transformed_buffer->set_metadata_start_time(metadata.get_start_time());
return transformed_buffer.release();
}
std::shared_ptr<std::atomic<hailo_status>> pipeline_status, std::chrono::milliseconds timeout,
PipelineDirection pipeline_direction, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
auto convert_nms_to_detections_elem_ptr = make_shared_nothrow<ConvertNmsToDetectionsElement>(std::move(nms_info),
- name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
+ name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
CHECK_AS_EXPECTED(nullptr != convert_nms_to_detections_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", convert_nms_to_detections_elem_ptr->description());
if (!buffer) {
input.set_action_status(buffer.status());
}
- CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status());
+ CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
- buffer->set_metadata(input.get_metadata());
+ buffer->set_metadata_start_time(input.get_metadata().get_start_time());
+ buffer->set_additional_data(input.get_metadata().get_additional_data<IouPipelineData>());
m_duration_collector.start_measurement();
const std::string &name, hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr<std::atomic<hailo_status>> pipeline_status,
std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
auto fill_nms_format_element = make_shared_nothrow<FillNmsFormatElement>(std::move(nms_config),
- name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
+ name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
CHECK_AS_EXPECTED(nullptr != fill_nms_format_element, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", fill_nms_format_element->description());
if (!buffer_expected) {
input.set_action_status(buffer_expected.status());
}
- CHECK_EXPECTED(buffer_expected, "{} (D2H) failed with status={}", name(), buffer_expected.status());
+ CHECK_EXPECTED(buffer_expected,
+ "{} (D2H) failed with status={}", name(),buffer_expected.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
auto buffer = buffer_expected.release();
- buffer.set_metadata(input.get_metadata());
+ buffer.set_metadata_start_time(input.get_metadata().get_start_time());
+ buffer.set_additional_data(input.get_metadata().get_additional_data<IouPipelineData>());
m_duration_collector.start_measurement();
auto detections = input.get_metadata().get_additional_data<IouPipelineData>();
- auto dst = buffer.as_view();
+ TRY(auto dst, buffer.as_view(BufferProtection::WRITE));
net_flow::NmsPostProcessOp::fill_nms_format_buffer(dst, detections->m_detections, detections->m_detections_classes_count,
m_nms_config);
hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr<std::atomic<hailo_status>> pipeline_status,
std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto transform_context = OutputTransformContext::create(src_image_shape, src_format, dst_image_shape, dst_format,
- dst_quant_infos, nms_info);
- CHECK_EXPECTED(transform_context, "Failed Creating OutputTransformContext");
+ TRY(auto transform_context, OutputTransformContext::create(src_image_shape, src_format, dst_image_shape, dst_format,
+ dst_quant_infos, nms_info), "Failed creating OutputTransformContext");
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
-
- auto post_infer_elem_ptr = make_shared_nothrow<PostInferElement>(transform_context.release(), name,
- duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
+ auto post_infer_elem_ptr = make_shared_nothrow<PostInferElement>(std::move(transform_context), name,
+ std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
CHECK_AS_EXPECTED(nullptr != post_infer_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", post_infer_elem_ptr->description());
if (!buffer) {
input.set_action_status(buffer.status());
}
- CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status());
+ CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
// Note: The latency to be measured starts as the buffer is read from the HW (it's 'input' in this case)
- buffer->set_metadata(input.get_metadata());
+ buffer->set_metadata_start_time(input.get_metadata().get_start_time());
+
+ TRY(auto src, input.as_view(BufferProtection::READ));
+ TRY(auto dst, buffer->as_view(BufferProtection::WRITE));
- auto dst = buffer->as_view();
m_duration_collector.start_measurement();
- const auto status = m_transform_context->transform(input.as_view(), dst);
+ const auto status = m_transform_context->transform(src, dst);
m_duration_collector.complete_measurement();
input.set_action_status(status);
std::shared_ptr<std::atomic<hailo_status>> pipeline_status, std::chrono::milliseconds timeout,
PipelineDirection pipeline_direction, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
auto convert_nms_removed_overlapping_elem_ptr = make_shared_nothrow<RemoveOverlappingBboxesElement>(std::move(nms_config),
- name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
+ name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
CHECK_AS_EXPECTED(nullptr != convert_nms_removed_overlapping_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", convert_nms_removed_overlapping_elem_ptr->description());
if (!buffer) {
input.set_action_status(buffer.status());
}
- CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status());
+ CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
- buffer->set_metadata(input.get_metadata());
+ buffer->set_metadata_start_time(input.get_metadata().get_start_time());
+ buffer->set_additional_data(input.get_metadata().get_additional_data<IouPipelineData>());
m_duration_collector.start_measurement();
auto detections_pipeline_data = input.get_metadata().get_additional_data<IouPipelineData>();
const std::string &name, hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr<std::atomic<hailo_status>> pipeline_status,
std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
auto argmax_elem_ptr = make_shared_nothrow<ArgmaxPostProcessElement>(argmax_op,
- name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
+ name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
CHECK_AS_EXPECTED(nullptr != argmax_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", argmax_elem_ptr->description());
return argmax_elem_ptr;
if (!buffer) {
input.set_action_status(buffer.status());
}
- CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status());
+ CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
std::map<std::string, MemoryView> inputs;
std::map<std::string, MemoryView> outputs;
auto &input_name = m_argmax_op->inputs_metadata().begin()->first;
auto &output_name = m_argmax_op->outputs_metadata().begin()->first;
- inputs.insert({input_name, input.as_view()});
- outputs.insert({output_name, buffer->as_view()});
+
+ TRY(auto src, input.as_view(BufferProtection::READ));
+ TRY(auto dst, buffer->as_view(BufferProtection::WRITE));
+
+ inputs.insert({input_name, src});
+ outputs.insert({output_name, dst});
m_duration_collector.start_measurement();
auto post_process_result = m_argmax_op->execute(inputs, outputs);
m_duration_collector.complete_measurement();
std::shared_ptr<std::atomic<hailo_status>> pipeline_status, std::chrono::milliseconds timeout,
PipelineDirection pipeline_direction, std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto duration_collector = DurationCollector::create(elem_flags);
- CHECK_EXPECTED(duration_collector);
+ TRY(auto duration_collector, DurationCollector::create(elem_flags));
auto softmax_elem_ptr = make_shared_nothrow<SoftmaxPostProcessElement>(softmax_op,
- name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
+ name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline);
CHECK_AS_EXPECTED(nullptr != softmax_elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
LOGGER__INFO("Created {}", softmax_elem_ptr->description());
return softmax_elem_ptr;
if (!buffer) {
input.set_action_status(buffer.status());
}
- CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status());
+ CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here
std::map<std::string, MemoryView> inputs;
std::map<std::string, MemoryView> outputs;
auto &input_name = m_softmax_op->inputs_metadata().begin()->first;
auto &output_name = m_softmax_op->outputs_metadata().begin()->first;
- inputs.insert({input_name, input.as_view()});
- outputs.insert({output_name, buffer->as_view()});
+
+ TRY(auto src, input.as_view(BufferProtection::READ));
+ TRY(auto dst, buffer->as_view(BufferProtection::WRITE));
+
+ inputs.insert({input_name, src});
+ outputs.insert({output_name, dst});
m_duration_collector.start_measurement();
auto post_process_result = m_softmax_op->execute(inputs, outputs);
m_duration_collector.complete_measurement();
std::shared_ptr<std::atomic<hailo_status>> pipeline_status, std::chrono::milliseconds timeout, PipelineDirection pipeline_direction,
std::shared_ptr<AsyncPipeline> async_pipeline)
{
- auto duration_collector = DurationCollector::create(HAILO_PIPELINE_ELEM_STATS_NONE);
- CHECK_EXPECTED(duration_collector);
- auto elem_ptr = make_shared_nothrow<CopyBufferElement>(name, duration_collector.release(), std::move(pipeline_status),
+ TRY(auto duration_collector, DurationCollector::create(HAILO_PIPELINE_ELEM_STATS_NONE));
+ auto elem_ptr = make_shared_nothrow<CopyBufferElement>(name, std::move(duration_collector), std::move(pipeline_status),
timeout, pipeline_direction, async_pipeline);
CHECK_AS_EXPECTED(nullptr != elem_ptr, HAILO_OUT_OF_HOST_MEMORY);
* @brief Implemention of the async HL infer
**/
-#include <iostream>
-
#include "common/utils.hpp"
#include "hailo/hailort_common.hpp"
#include "hailo/vdevice.hpp"
#include "hailo/infer_model.hpp"
-#include "vdevice/vdevice_internal.hpp"
#include "hef/hef_internal.hpp"
#include "net_flow/pipeline/infer_model_internal.hpp"
#include "net_flow/pipeline/async_infer_runner.hpp"
namespace hailort
{
-std::string InferModel::InferStream::Impl::name() const
+std::string InferModelBase::InferStream::Impl::name() const
{
return m_vstream_info.name;
}
-hailo_3d_image_shape_t InferModel::InferStream::Impl::shape() const
+hailo_3d_image_shape_t InferModelBase::InferStream::Impl::shape() const
{
return m_vstream_info.shape;
}
-hailo_format_t InferModel::InferStream::Impl::format() const
+hailo_format_t InferModelBase::InferStream::Impl::format() const
{
return m_user_buffer_format;
}
-size_t InferModel::InferStream::Impl::get_frame_size() const
+size_t InferModelBase::InferStream::Impl::get_frame_size() const
{
return HailoRTCommon::get_frame_size(m_vstream_info, m_user_buffer_format);
}
-Expected<hailo_nms_shape_t> InferModel::InferStream::Impl::get_nms_shape() const
+Expected<hailo_nms_shape_t> InferModelBase::InferStream::Impl::get_nms_shape() const
{
CHECK_AS_EXPECTED(HailoRTCommon::is_nms(m_vstream_info.format.order), HAILO_INVALID_OPERATION,
"Output {} is not NMS", name());
return res;
}
-std::vector<hailo_quant_info_t> InferModel::InferStream::Impl::get_quant_infos() const
+std::vector<hailo_quant_info_t> InferModelBase::InferStream::Impl::get_quant_infos() const
{
// TODO: Support quant infos vector
return {m_vstream_info.quant_info};
}
-void InferModel::InferStream::Impl::set_format_type(hailo_format_type_t type)
+void InferModelBase::InferStream::Impl::set_format_type(hailo_format_type_t type)
{
m_user_buffer_format.type = type;
}
-void InferModel::InferStream::Impl::set_format_order(hailo_format_order_t order)
+void InferModelBase::InferStream::Impl::set_format_order(hailo_format_order_t order)
{
m_user_buffer_format.order = order;
}
-bool InferModel::InferStream::Impl::is_nms() const
+bool InferModelBase::InferStream::Impl::is_nms() const
{
return HailoRTCommon::is_nms(m_vstream_info.format.order);
}
-void InferModel::InferStream::Impl::set_nms_score_threshold(float32_t threshold)
+void InferModelBase::InferStream::Impl::set_nms_score_threshold(float32_t threshold)
{
m_nms_score_threshold = threshold;
}
-void InferModel::InferStream::Impl::set_nms_iou_threshold(float32_t threshold)
+void InferModelBase::InferStream::Impl::set_nms_iou_threshold(float32_t threshold)
{
m_nms_iou_threshold = threshold;
}
-void InferModel::InferStream::Impl::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class)
+void InferModelBase::InferStream::Impl::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class)
{
m_nms_max_proposals_per_class = max_proposals_per_class;
m_vstream_info.nms_shape.max_bboxes_per_class = max_proposals_per_class;
}
-void InferModel::InferStream::Impl::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size)
+void InferModelBase::InferStream::Impl::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size)
{
m_nms_max_accumulated_mask_size = max_accumulated_mask_size;
m_vstream_info.nms_shape.max_accumulated_mask_size = max_accumulated_mask_size;
}
-InferModel::InferStream::InferStream(std::shared_ptr<InferModel::InferStream::Impl> pimpl) : m_pimpl(pimpl)
+float32_t InferModelBase::InferStream::Impl::nms_score_threshold() const
{
+ return m_nms_score_threshold;
}
-const std::string InferModel::InferStream::name() const
+float32_t InferModelBase::InferStream::Impl::nms_iou_threshold() const
+{
+ return m_nms_iou_threshold;
+}
+
+uint32_t InferModelBase::InferStream::Impl::nms_max_proposals_per_class() const
+{
+ return m_nms_max_proposals_per_class;
+}
+
+uint32_t InferModelBase::InferStream::Impl::nms_max_accumulated_mask_size() const
+{
+ return m_nms_max_accumulated_mask_size;
+}
+
+InferModelBase::InferStream::InferStream(std::shared_ptr<InferModelBase::InferStream::Impl> pimpl) : m_pimpl(pimpl)
+{
+}
+
+const std::string InferModelBase::InferStream::name() const
{
return m_pimpl->name();
}
-hailo_3d_image_shape_t InferModel::InferStream::shape() const
+hailo_3d_image_shape_t InferModelBase::InferStream::shape() const
{
return m_pimpl->shape();
}
-hailo_format_t InferModel::InferStream::format() const
+hailo_format_t InferModelBase::InferStream::format() const
{
return m_pimpl->format();
}
-size_t InferModel::InferStream::get_frame_size() const
+size_t InferModelBase::InferStream::get_frame_size() const
{
return m_pimpl->get_frame_size();
}
-Expected<hailo_nms_shape_t> InferModel::InferStream::get_nms_shape() const
+Expected<hailo_nms_shape_t> InferModelBase::InferStream::get_nms_shape() const
{
return m_pimpl->get_nms_shape();
}
-std::vector<hailo_quant_info_t> InferModel::InferStream::get_quant_infos() const
+std::vector<hailo_quant_info_t> InferModelBase::InferStream::get_quant_infos() const
{
return m_pimpl->get_quant_infos();
}
-void InferModel::InferStream::set_format_type(hailo_format_type_t type)
+void InferModelBase::InferStream::set_format_type(hailo_format_type_t type)
{
m_pimpl->set_format_type(type);
}
-void InferModel::InferStream::set_format_order(hailo_format_order_t order)
+void InferModelBase::InferStream::set_format_order(hailo_format_order_t order)
{
m_pimpl->set_format_order(order);
}
-bool InferModel::InferStream::is_nms() const
+bool InferModelBase::InferStream::is_nms() const
{
return m_pimpl->is_nms();
}
-void InferModel::InferStream::set_nms_score_threshold(float32_t threshold)
+void InferModelBase::InferStream::set_nms_score_threshold(float32_t threshold)
{
m_pimpl->set_nms_score_threshold(threshold);
}
-void InferModel::InferStream::set_nms_iou_threshold(float32_t threshold)
+void InferModelBase::InferStream::set_nms_iou_threshold(float32_t threshold)
{
m_pimpl->set_nms_iou_threshold(threshold);
}
-void InferModel::InferStream::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class)
+void InferModelBase::InferStream::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class)
{
m_pimpl->set_nms_max_proposals_per_class(max_proposals_per_class);
}
-void InferModel::InferStream::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size)
+void InferModelBase::InferStream::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size)
{
m_pimpl->set_nms_max_accumulated_mask_size(max_accumulated_mask_size);
}
-InferModel::InferModel(VDevice &vdevice, Hef &&hef, std::unordered_map<std::string, InferModel::InferStream> &&inputs,
- std::unordered_map<std::string, InferModel::InferStream> &&outputs)
+float32_t InferModelBase::InferStream::nms_score_threshold() const
+{
+ return m_pimpl->nms_score_threshold();
+}
+
+float32_t InferModelBase::InferStream::nms_iou_threshold() const
+{
+ return m_pimpl->nms_iou_threshold();
+}
+
+uint32_t InferModelBase::InferStream::nms_max_proposals_per_class() const
+{
+ return m_pimpl->nms_max_proposals_per_class();
+}
+
+uint32_t InferModelBase::InferStream::nms_max_accumulated_mask_size() const
+{
+ return m_pimpl->nms_max_accumulated_mask_size();
+}
+
+Expected<std::shared_ptr<InferModelBase>> InferModelBase::create(VDevice &vdevice, const std::string &hef_path)
+{
+ TRY(auto hef, Hef::create(hef_path));
+ TRY(auto inputs, create_infer_stream_inputs(hef));
+ TRY(auto outputs, create_infer_stream_outputs(hef));
+
+ auto ptr = make_shared_nothrow<InferModelBase>(vdevice, std::move(hef), std::move(inputs), std::move(outputs));
+ CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return ptr;
+}
+
+Expected<std::shared_ptr<InferModelBase>> InferModelBase::create(VDevice &vdevice, const MemoryView hef_buffer)
+{
+ TRY(auto hef, Hef::create(hef_buffer));
+ TRY(auto inputs, create_infer_stream_inputs(hef));
+ TRY(auto outputs, create_infer_stream_outputs(hef));
+
+ auto ptr = make_shared_nothrow<InferModelBase>(vdevice, std::move(hef), std::move(inputs), std::move(outputs));
+ CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return ptr;
+}
+
+InferModelBase::InferModelBase(VDevice &vdevice, Hef &&hef, std::unordered_map<std::string, InferModelBase::InferStream> &&inputs,
+ std::unordered_map<std::string, InferModelBase::InferStream> &&outputs)
: m_vdevice(vdevice), m_hef(std::move(hef)), m_inputs(std::move(inputs)), m_outputs(std::move(outputs)),
m_config_params(HailoRTDefaults::get_configure_params())
{
}
}
-InferModel::InferModel(InferModel &&other) :
+InferModelBase::InferModelBase(InferModelBase &&other) :
m_vdevice(std::move(other.m_vdevice)),
m_hef(std::move(other.m_hef)),
m_inputs(std::move(other.m_inputs)),
{
}
-const Hef &InferModel::hef() const
+const Hef &InferModelBase::hef() const
{
return m_hef;
}
-void InferModel::set_batch_size(uint16_t batch_size)
+void InferModelBase::set_batch_size(uint16_t batch_size)
{
m_config_params.batch_size = batch_size;
}
-void InferModel::set_power_mode(hailo_power_mode_t power_mode)
+void InferModelBase::set_power_mode(hailo_power_mode_t power_mode)
{
m_config_params.power_mode = power_mode;
}
-void InferModel::set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency)
+void InferModelBase::set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency)
{
m_config_params.latency = latency;
}
-Expected<ConfiguredInferModel> InferModel::configure()
+Expected<ConfiguredInferModel> InferModelBase::configure()
{
auto configure_params = m_vdevice.get().create_configure_params(m_hef);
CHECK_EXPECTED(configure_params);
std::unordered_map<std::string, hailo_format_t> inputs_formats;
std::unordered_map<std::string, hailo_format_t> outputs_formats;
+ std::unordered_map<std::string, size_t> inputs_frame_sizes;
+ std::unordered_map<std::string, size_t> outputs_frame_sizes;
auto input_vstream_infos = network_groups.value()[0]->get_input_vstream_infos();
CHECK_EXPECTED(input_vstream_infos);
for (const auto &vstream_info : input_vstream_infos.value()) {
assert(contains(m_inputs, std::string(vstream_info.name)));
inputs_formats[vstream_info.name] = m_inputs.at(vstream_info.name).format();
+ inputs_frame_sizes[vstream_info.name] = m_inputs.at(vstream_info.name).get_frame_size();
}
auto output_vstream_infos = network_groups.value()[0]->get_output_vstream_infos();
for (const auto &vstream_info : output_vstream_infos.value()) {
assert(contains(m_outputs, std::string(vstream_info.name)));
outputs_formats[vstream_info.name] = m_outputs.at(vstream_info.name).format();
+ outputs_frame_sizes[vstream_info.name] = m_outputs.at(vstream_info.name).get_frame_size();
}
CHECK_AS_EXPECTED(std::all_of(m_inputs.begin(), m_inputs.end(), [](const auto &input_pair) {
}
auto configured_infer_model_pimpl = ConfiguredInferModelImpl::create(network_groups.value()[0], inputs_formats, outputs_formats,
- get_input_names(), get_output_names(), m_vdevice);
+ get_input_names(), get_output_names(), m_vdevice, inputs_frame_sizes, outputs_frame_sizes);
CHECK_EXPECTED(configured_infer_model_pimpl);
// The hef buffer is being used only when working with the service.
return ConfiguredInferModel(configured_infer_model_pimpl.release());
}
-Expected<ConfiguredInferModel> InferModel::configure_for_ut(std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
+Expected<ConfiguredInferModel> InferModel
Base::configure_for_ut(std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes, const std::unordered_map<std::string, size_t> outputs_frame_sizes,
std::shared_ptr<ConfiguredNetworkGroup> net_group)
{
if (nullptr == net_group) {
net_group = network_groups.value()[0];
}
- auto configured_infer_model_pimpl = ConfiguredInferModelImpl::create_for_ut(net_group, async_infer_runner, input_names, output_names);
+ auto configured_infer_model_pimpl = ConfiguredInferModelImpl::create_for_ut(net_group, async_infer_runner, input_names, output_names,
+ inputs_frame_sizes, outputs_frame_sizes);
CHECK_EXPECTED(configured_infer_model_pimpl);
return ConfiguredInferModel(configured_infer_model_pimpl.release());
}
-Expected<InferModel::InferStream> InferModel::input()
+Expected<InferModelBase::InferStream> InferModelBase::input()
{
CHECK_AS_EXPECTED(1 == m_inputs.size(), HAILO_INVALID_OPERATION, "Model has more than one input!");
auto copy = m_inputs.begin()->second;
return copy;
}
-Expected<InferModel::InferStream> InferModel::output()
+Expected<InferModelBase::InferStream> InferModelBase::output()
{
CHECK_AS_EXPECTED(1 == m_outputs.size(), HAILO_INVALID_OPERATION, "Model has more than one output!");
auto copy = m_outputs.begin()->second;
return copy;
}
-Expected<InferModel::InferStream> InferModel::input(const std::string &name)
+Expected<InferModelBase::InferStream> InferModelBase::input(const std::string &name)
{
CHECK_AS_EXPECTED(contains(m_inputs, name), HAILO_NOT_FOUND, "Input {} not found!", name);
auto copy = m_inputs.at(name);
return copy;
}
-Expected<InferModel::InferStream> InferModel::output(const std::string &name)
+Expected<InferModelBase::InferStream> InferModelBase::output(const std::string &name)
{
CHECK_AS_EXPECTED(contains(m_outputs, name), HAILO_NOT_FOUND, "Output {}, not found!", name);
auto copy = m_outputs.at(name);
return copy;
}
-const std::vector<InferModel::InferStream> &InferModel::inputs() const
+const std::vector<InferModelBase::InferStream> &InferModelBase::inputs() const
{
return m_inputs_vector;
}
-const std::vector<InferModel::InferStream> &InferModel::outputs() const
+const std::vector<InferModelBase::InferStream> &InferModelBase::outputs() const
{
return m_outputs_vector;
}
-const std::vector<std::string> &InferModel::get_input_names() const
+const std::vector<std::string> &InferModelBase::get_input_names() const
{
return m_input_names;
}
-const std::vector<std::string> &InferModel::get_output_names() const
+const std::vector<std::string> &InferModelBase::get_output_names() const
{
return m_output_names;
}
-ConfiguredInferModel::ConfiguredInferModel(std::shared_ptr<ConfiguredInferModelImpl> pimpl) : m_pimpl(pimpl)
+Expected<std::unordered_map<std::string, InferModel::InferStream>> InferModelBase::create_infer_stream_inputs(Hef &hef)
{
+ auto input_vstream_infos = hef.get_input_vstream_infos();
+ CHECK_EXPECTED(input_vstream_infos);
+
+ std::unordered_map<std::string, InferModel::InferStream> inputs;
+ for (const auto &vstream_info : input_vstream_infos.value()) {
+ auto pimpl = make_shared_nothrow<InferModel::InferStream::Impl>(vstream_info);
+ CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY);
+
+ InferModel::InferStream stream(pimpl);
+ inputs.emplace(vstream_info.name, std::move(stream));
+ }
+
+ return inputs;
+}
+
+Expected<std::unordered_map<std::string, InferModel::InferStream>> InferModelBase::create_infer_stream_outputs(Hef &hef)
+{
+ auto output_vstream_infos = hef.get_output_vstream_infos();
+ CHECK_EXPECTED(output_vstream_infos);
+
+ std::unordered_map<std::string, InferModel::InferStream> outputs;
+ for (const auto &vstream_info : output_vstream_infos.value()) {
+ auto pimpl = make_shared_nothrow<InferModel::InferStream::Impl>(vstream_info);
+ CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY);
+
+ InferModel::InferStream stream(pimpl);
+ outputs.emplace(vstream_info.name, std::move(stream));
+ }
+
+ return outputs;
+}
+
+ConfiguredInferModel::ConfiguredInferModel(std::shared_ptr<ConfiguredInferModelBase> pimpl) : m_pimpl(pimpl)
+{
+}
+
+ConfiguredInferModelBase::ConfiguredInferModelBase(const std::unordered_map<std::string, size_t> inputs_frame_sizes,
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes) :
+ m_inputs_frame_sizes(inputs_frame_sizes), m_outputs_frame_sizes(outputs_frame_sizes)
+{
+}
+
+ConfiguredInferModel ConfiguredInferModelBase::create(std::shared_ptr<ConfiguredInferModelBase> base)
+{
+ return ConfiguredInferModel(base);
}
Expected<ConfiguredInferModel::Bindings> ConfiguredInferModel::create_bindings()
return m_pimpl->create_bindings();
}
-hailo_status ConfiguredInferModel::wait_for_async_ready(std::chrono::milliseconds timeout)
+hailo_status ConfiguredInferModel::wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count)
{
- return m_pimpl->wait_for_async_ready(timeout);
+ return m_pimpl->wait_for_async_ready(timeout, frames_count);
}
hailo_status ConfiguredInferModel::activate()
return m_pimpl->activate();
}
-void ConfiguredInferModel::deactivate()
+hailo_status ConfiguredInferModel::deactivate()
{
- m_pimpl->deactivate();
+ return m_pimpl->deactivate();
}
hailo_status ConfiguredInferModel::run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout)
Expected<AsyncInferJob> ConfiguredInferModel::run_async(ConfiguredInferModel::Bindings bindings,
std::function<void(const AsyncInferCompletionInfo &)> callback)
{
- return m_pimpl->run_async(bindings, callback);
+ auto async_infer_job = m_pimpl->run_async(bindings, callback);
+ if (HAILO_SUCCESS != async_infer_job.status()) {
+ shutdown();
+ return make_unexpected(async_infer_job.status());
+ }
+
+ return async_infer_job.release();
}
Expected<LatencyMeasurementResult> ConfiguredInferModel::get_hw_latency_measurement()
return m_pimpl->get_async_queue_size();
}
-void ConfiguredInferModel::shutdown()
+hailo_status ConfiguredInferModel::shutdown()
+{
+ return m_pimpl->shutdown();
+}
+
+Expected<AsyncInferJob> ConfiguredInferModel::run_async(const std::vector<ConfiguredInferModel::Bindings> &bindings,
+ std::function<void(const AsyncInferCompletionInfo &)> callback)
+{
+ auto job_pimpl = make_shared_nothrow<AsyncInferJobImpl>(static_cast<uint32_t>(bindings.size()));
+ if (nullptr == job_pimpl) {
+ shutdown();
+ return make_unexpected(HAILO_OUT_OF_HOST_MEMORY);
+ }
+
+ auto transfer_done = [bindings, job_pimpl, callback](const AsyncInferCompletionInfo &completion_info) {
+ bool should_call_callback = ConfiguredInferModelBase::get_stream_done(completion_info.status, job_pimpl);
+ if (should_call_callback) {
+ AsyncInferCompletionInfo final_completion_info(ConfiguredInferModelBase::get_completion_status(job_pimpl));
+ callback(final_completion_info);
+ ConfiguredInferModelBase::mark_callback_done(job_pimpl);
+ }
+ };
+
+ for (const auto &binding : bindings) {
+ TRY(auto partial_job, run_async(binding, transfer_done));
+ partial_job.detach();
+ }
+
+ return AsyncInferJobImpl::create(job_pimpl);
+}
+
+Expected<ConfiguredInferModel::Bindings> ConfiguredInferModelBase::create_bindings(
+ std::unordered_map<std::string, ConfiguredInferModel::Bindings::InferStream> &&inputs,
+ std::unordered_map<std::string, ConfiguredInferModel::Bindings::InferStream> &&outputs)
+{
+ return ConfiguredInferModel::Bindings(std::move(inputs), std::move(outputs));
+}
+
+Expected<ConfiguredInferModel::Bindings::InferStream> ConfiguredInferModelBase::create_infer_stream(
+ const hailo_vstream_info_t &vstream_info)
+{
+ auto pimpl = make_shared_nothrow<ConfiguredInferModel::Bindings::InferStream::Impl>(vstream_info);
+ CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY);
+
+ ConfiguredInferModel::Bindings::InferStream stream(pimpl);
+ return stream;
+}
+
+BufferType ConfiguredInferModelBase::get_infer_stream_buffer_type(ConfiguredInferModel::Bindings::InferStream stream)
+{
+ return stream.m_pimpl->get_type();
+}
+
+bool ConfiguredInferModelBase::get_stream_done(hailo_status status, std::shared_ptr<AsyncInferJobImpl> job_pimpl)
+{
+ return job_pimpl->stream_done(status);
+}
+
+hailo_status ConfiguredInferModelBase::get_completion_status(std::shared_ptr<AsyncInferJobImpl> job_pimpl)
+{
+ return job_pimpl->completion_status();
+}
+
+void ConfiguredInferModelBase::mark_callback_done(std::shared_ptr<AsyncInferJobImpl> job_pimpl)
+{
+ job_pimpl->mark_callback_done();
+}
+
+hailo_status ConfiguredInferModelBase::run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout)
{
- m_pimpl->abort();
+ auto job = run_async(bindings, [] (const AsyncInferCompletionInfo &) {});
+ CHECK_EXPECTED_AS_STATUS(job);
+
+ auto status = job->wait(timeout);
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
}
Expected<std::shared_ptr<ConfiguredInferModelImpl>> ConfiguredInferModelImpl::create(std::shared_ptr<ConfiguredNetworkGroup> net_group,
const std::unordered_map<std::string, hailo_format_t> &inputs_formats,
const std::unordered_map<std::string, hailo_format_t> &outputs_formats,
- const std::vector<std::string> &input_names, const std::vector<std::string> &output_names, VDevice &vdevice, const uint32_t timeout)
+ const std::vector<std::string> &input_names, const std::vector<std::string> &output_names, VDevice &vdevice,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes, const std::unordered_map<std::string, size_t> outputs_frame_sizes,
+ const uint32_t timeout)
{
auto async_infer_runner = AsyncInferRunnerImpl::create(net_group, inputs_formats, outputs_formats, timeout);
CHECK_EXPECTED(async_infer_runner);
}
auto configured_infer_model_pimpl = make_shared_nothrow<ConfiguredInferModelImpl>(net_group, async_infer_runner.release(),
- input_names, output_names);
+ input_names, output_names, inputs_frame_sizes, outputs_frame_sizes);
CHECK_NOT_NULL_AS_EXPECTED(configured_infer_model_pimpl, HAILO_OUT_OF_HOST_MEMORY);
return configured_infer_model_pimpl;
}
Expected<std::shared_ptr<ConfiguredInferModelImpl>> ConfiguredInferModelImpl::create_for_ut(std::shared_ptr<ConfiguredNetworkGroup> net_group,
- std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner, const std::vector<std::string> &input_names, const std::vector<std::string> &output_names)
+ std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner, const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes, const std::unordered_map<std::string, size_t> outputs_frame_sizes)
{
auto configured_infer_model_pimpl = make_shared_nothrow<ConfiguredInferModelImpl>(net_group, async_infer_runner,
- input_names, output_names);
+ input_names, output_names, inputs_frame_sizes, outputs_frame_sizes);
CHECK_NOT_NULL_AS_EXPECTED(configured_infer_model_pimpl, HAILO_OUT_OF_HOST_MEMORY);
return configured_infer_model_pimpl;
}
ConfiguredInferModelImpl::ConfiguredInferModelImpl(std::shared_ptr<ConfiguredNetworkGroup> cng,
- std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
- const std::vector<std::string> &input_names,
- const std::vector<std::string> &output_names) : m_cng(cng), m_async_infer_runner(async_infer_runner),
- m_ongoing_parallel_transfers(0), m_input_names(input_names), m_output_names(output_names)
+ std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner, const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes, const std::unordered_map<std::string, size_t> outputs_frame_sizes) :
+ ConfiguredInferModelBase(inputs_frame_sizes, outputs_frame_sizes),
+ m_cng(cng), m_async_infer_runner(async_infer_runner), m_ongoing_parallel_transfers(0), m_input_names(input_names), m_output_names(output_names)
{
}
ConfiguredInferModelImpl::~ConfiguredInferModelImpl()
{
- abort();
+ shutdown();
}
Expected<ConfiguredInferModel::Bindings> ConfiguredInferModelImpl::create_bindings()
std::unordered_map<std::string, ConfiguredInferModel::Bindings::InferStream> inputs;
std::unordered_map<std::string, ConfiguredInferModel::Bindings::InferStream> outputs;
- auto input_vstream_infos = m_cng->get_input_vstream_infos();
+ auto cng = m_cng.lock();
+ CHECK_NOT_NULL_AS_EXPECTED(cng, HAILO_INTERNAL_FAILURE);
+
+ auto input_vstream_infos = cng->get_input_vstream_infos();
CHECK_EXPECTED(input_vstream_infos);
for (const auto &vstream_info : input_vstream_infos.value()) {
- auto pimpl = make_shared_nothrow<ConfiguredInferModel::Bindings::InferStream::Impl>(vstream_info);
- CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY);
-
- ConfiguredInferModel::Bindings::InferStream stream(pimpl);
+ TRY(auto stream, ConfiguredInferModelBase::create_infer_stream(vstream_info));
inputs.emplace(vstream_info.name, std::move(stream));
}
- auto output_vstream_infos = m_cng->get_output_vstream_infos();
+ auto output_vstream_infos = cng->get_output_vstream_infos();
CHECK_EXPECTED(output_vstream_infos);
for (const auto &vstream_info : output_vstream_infos.value()) {
- auto pimpl = make_shared_nothrow<ConfiguredInferModel::Bindings::InferStream::Impl>(vstream_info);
- CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY);
-
- ConfiguredInferModel::Bindings::InferStream stream(pimpl);
+ TRY(auto stream, ConfiguredInferModelBase::create_infer_stream(vstream_info));
outputs.emplace(vstream_info.name, std::move(stream));
}
- return ConfiguredInferModel::Bindings(std::move(inputs), std::move(outputs));
+ TRY(auto bindings, ConfiguredInferModelBase::create_bindings(std::move(inputs), std::move(outputs)));
+ return bindings;
}
-hailo_status ConfiguredInferModelImpl::wait_for_async_ready(std::chrono::milliseconds timeout)
+hailo_status ConfiguredInferModelImpl::wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count)
{
std::unique_lock<std::mutex> lock(m_mutex);
hailo_status status = HAILO_SUCCESS;
- bool was_successful = m_cv.wait_for(lock, timeout, [this, &status] () -> bool {
- auto pools_are_ready = m_async_infer_runner->can_push_buffers();
+ bool was_successful = m_cv.wait_for(lock, timeout, [this, frames_count, &status] () -> bool {
+ auto pools_are_ready = m_async_infer_runner->can_push_buffers(frames_count);
if (HAILO_SUCCESS != pools_are_ready.status()) {
status = pools_are_ready.status();
return true;
return HAILO_SUCCESS;
}
-void ConfiguredInferModelImpl::abort()
+hailo_status ConfiguredInferModelImpl::shutdown()
{
m_async_infer_runner->abort();
std::unique_lock<std::mutex> lock(m_mutex);
m_cv.wait_for(lock, WAIT_FOR_ASYNC_IN_DTOR_TIMEOUT, [this] () -> bool {
return m_ongoing_parallel_transfers == 0;
});
+
+ return deactivate();
}
hailo_status ConfiguredInferModelImpl::activate()
{
- auto activated_ng = m_cng->activate();
+ auto cng = m_cng.lock();
+ CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE);
+
+ auto activated_ng = cng->activate();
CHECK_EXPECTED_AS_STATUS(activated_ng);
m_ang = activated_ng.release();
return HAILO_SUCCESS;
}
-void ConfiguredInferModelImpl::deactivate()
+hailo_status ConfiguredInferModelImpl::deactivate()
{
m_ang = nullptr;
-}
-
-hailo_status ConfiguredInferModelImpl::run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout)
-{
- auto job = run_async(bindings, [] (const AsyncInferCompletionInfo &) {});
- CHECK_EXPECTED_AS_STATUS(job);
-
- auto status = job->wait(timeout);
- CHECK_SUCCESS(status);
return HAILO_SUCCESS;
}
hailo_status ConfiguredInferModelImpl::validate_bindings(ConfiguredInferModel::Bindings bindings)
{
for (const auto &input_name : m_input_names) {
- auto buffer_type = bindings.input(input_name)->m_pimpl->get_type();
+ TRY(auto input, bindings.input(input_name));
+ auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(input);
switch (buffer_type) {
case BufferType::VIEW:
{
- CHECK_EXPECTED_AS_STATUS(bindings.input(input_name)->get_buffer());
+ auto buffer = input.get_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ CHECK(buffer->size() == m_inputs_frame_sizes.at(input_name), HAILO_INVALID_OPERATION,
+ "Input buffer size {} is different than expected {} for input '{}'", buffer->size(), m_inputs_frame_sizes.at(input_name), input_name);
break;
}
case BufferType::PIX_BUFFER:
{
- CHECK_EXPECTED_AS_STATUS(bindings.input(input_name)->get_pix_buffer());
+ auto buffer = input.get_pix_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ size_t buffer_size = 0;
+ for (size_t i = 0 ; i < buffer->number_of_planes ; i++) {
+ buffer_size += buffer->planes[i].bytes_used;
+ }
+ CHECK(buffer_size == m_inputs_frame_sizes.at(input_name), HAILO_INVALID_OPERATION,
+ "Input buffer size {} is different than expected {} for input '{}'", buffer_size, m_inputs_frame_sizes.at(input_name), input_name);
break;
}
case BufferType::DMA_BUFFER:
{
- CHECK_EXPECTED_AS_STATUS(bindings.input(input_name)->get_dma_buffer());
+ auto buffer = input.get_dma_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ CHECK(buffer->size == m_inputs_frame_sizes.at(input_name), HAILO_INVALID_OPERATION,
+ "Input buffer size {} is different than expected {} for input '{}'", buffer->size, m_inputs_frame_sizes.at(input_name), input_name);
break;
}
default:
}
}
for (const auto &output_name : m_output_names) {
- auto buffer_type = bindings.output(output_name)->m_pimpl->get_type();
+ TRY(auto output, bindings.output(output_name));
+ auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(output);
switch (buffer_type) {
case BufferType::VIEW:
{
- CHECK_EXPECTED_AS_STATUS(bindings.output(output_name)->get_buffer());
+ auto buffer = output.get_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ CHECK(buffer->size() == m_outputs_frame_sizes.at(output_name), HAILO_INVALID_OPERATION,
+ "Output buffer size {} is different than expected {} for output '{}'", buffer->size(), m_outputs_frame_sizes.at(output_name), output_name);
break;
}
case BufferType::PIX_BUFFER:
}
case BufferType::DMA_BUFFER:
{
- CHECK_EXPECTED_AS_STATUS(bindings.output(output_name)->get_dma_buffer());
+ auto buffer = output.get_dma_buffer();
+ CHECK_EXPECTED_AS_STATUS(buffer);
+ CHECK(buffer->size == m_outputs_frame_sizes.at(output_name), HAILO_INVALID_OPERATION,
+ "Output buffer size {} is different than expected {} for out '{}'", buffer->size, m_outputs_frame_sizes.at(output_name), output_name);
break;
}
default:
{
CHECK_SUCCESS_AS_EXPECTED(validate_bindings(bindings));
- auto job_pimpl = make_shared_nothrow<AsyncInferJob::Impl>(static_cast<uint32_t>(m_input_names.size() + m_output_names.size()));
+ auto job_pimpl = make_shared_nothrow<AsyncInferJobImpl>(static_cast<uint32_t>(m_input_names.size() + m_output_names.size()));
CHECK_NOT_NULL_AS_EXPECTED(job_pimpl, HAILO_OUT_OF_HOST_MEMORY);
TransferDoneCallbackAsyncInfer transfer_done = [this, bindings, job_pimpl, callback](hailo_status status) {
- bool should_call_callback = job_pimpl->stream_done(status);
+ bool should_call_callback = ConfiguredInferModelBase::get_stream_done(status, job_pimpl);
if (should_call_callback) {
auto final_status = (m_async_infer_runner->get_pipeline_status() == HAILO_SUCCESS) ?
- job_pimpl->completion_status() : m_async_infer_runner->get_pipeline_status();
+ ConfiguredInferModelBase::get_completion_status(job_pimpl) : m_async_infer_runner->get_pipeline_status();
AsyncInferCompletionInfo completion_info(final_status);
callback(completion_info);
- job_pimpl->mark_callback_done();
-
+ ConfiguredInferModelBase::mark_callback_done(job_pimpl);
{
std::unique_lock<std::mutex> lock(m_mutex);
m_ongoing_parallel_transfers--;
}
m_cv.notify_all();
- AsyncInferJob job(job_pimpl);
- return job;
+ return AsyncInferJobImpl::create(job_pimpl);
}
Expected<LatencyMeasurementResult> ConfiguredInferModelImpl::get_hw_latency_measurement()
{
- return m_cng->get_latency_measurement();
+ auto cng = m_cng.lock();
+ CHECK_NOT_NULL_AS_EXPECTED(cng, HAILO_INTERNAL_FAILURE);
+
+ return cng->get_latency_measurement();
}
hailo_status ConfiguredInferModelImpl::set_scheduler_timeout(const std::chrono::milliseconds &timeout)
{
- return m_cng->set_scheduler_timeout(timeout);
+ auto cng = m_cng.lock();
+ CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE);
+
+ return cng->set_scheduler_timeout(timeout);
}
hailo_status ConfiguredInferModelImpl::set_scheduler_threshold(uint32_t threshold)
{
- return m_cng->set_scheduler_threshold(threshold);
+ auto cng = m_cng.lock();
+ CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE);
+
+ return cng->set_scheduler_threshold(threshold);
}
hailo_status ConfiguredInferModelImpl::set_scheduler_priority(uint8_t priority)
{
- return m_cng->set_scheduler_priority(priority);
+ auto cng = m_cng.lock();
+ CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE);
+
+ return cng->set_scheduler_priority(priority);
}
Expected<size_t> ConfiguredInferModelImpl::get_async_queue_size()
{
- return m_cng->get_min_buffer_pool_size();
+ auto cng = m_cng.lock();
+ CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE);
+
+ return cng->get_min_buffer_pool_size();
}
-AsyncInferJob::AsyncInferJob(std::shared_ptr<Impl> pimpl) : m_pimpl(pimpl), m_should_wait_in_dtor(true)
+AsyncInferJob::AsyncInferJob(std::shared_ptr<AsyncInferJobBase> pimpl) : m_pimpl(pimpl), m_should_wait_in_dtor(true)
{
}
m_should_wait_in_dtor = false;
}
-AsyncInferJob::Impl::Impl(uint32_t streams_count) : m_job_completion_status(HAILO_SUCCESS)
+AsyncInferJob AsyncInferJobBase::create(std::shared_ptr<AsyncInferJobBase> base)
+{
+ return AsyncInferJob(base);
+}
+
+AsyncInferJobImpl::AsyncInferJobImpl(uint32_t streams_count) : m_job_completion_status(HAILO_SUCCESS)
{
m_ongoing_transfers = streams_count;
m_callback_called = false;
}
-hailo_status AsyncInferJob::Impl::wait(std::chrono::milliseconds timeout)
+hailo_status AsyncInferJobImpl::wait(std::chrono::milliseconds timeout)
{
std::unique_lock<std::mutex> lock(m_mutex);
bool was_successful = m_cv.wait_for(lock, timeout, [this] () -> bool {
return HAILO_SUCCESS;
}
-bool AsyncInferJob::Impl::stream_done(const hailo_status &status)
+bool AsyncInferJobImpl::stream_done(const hailo_status &status)
{
bool should_call_callback = false;
{
return should_call_callback;
}
-hailo_status AsyncInferJob::Impl::completion_status()
+hailo_status AsyncInferJobImpl::completion_status()
{
return m_job_completion_status;
}
-void AsyncInferJob::Impl::mark_callback_done()
+void AsyncInferJobImpl::mark_callback_done()
{
{
std::unique_lock<std::mutex> lock(m_mutex);
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file infer_model_hrpc_client.cpp
+ * @brief InferModel HRPC client implementation
+ **/
+
+#include "infer_model_hrpc_client.hpp"
+#include "configured_infer_model_hrpc_client.hpp"
+
+namespace hailort
+{
+
+Expected<std::shared_ptr<InferModelHrpcClient>> InferModelHrpcClient::create(Hef &&hef,
+ std::shared_ptr<hrpc::Client> client, uint32_t infer_model_handle_id, uint32_t vdevice_handle, VDevice &vdevice)
+{
+ TRY(auto inputs, create_infer_stream_inputs(hef));
+ TRY(auto outputs, create_infer_stream_outputs(hef));
+
+ auto ptr = make_shared_nothrow<InferModelHrpcClient>(client, infer_model_handle_id,
+ vdevice_handle, vdevice, std::move(hef), std::move(inputs), std::move(outputs));
+ CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return ptr;
+}
+
+InferModelHrpcClient::InferModelHrpcClient(std::shared_ptr<hrpc::Client> client, uint32_t handle,
+ uint32_t vdevice_handle, VDevice &vdevice, Hef &&hef,
+ std::unordered_map<std::string, InferStream> &&inputs, std::unordered_map<std::string, InferStream> &&outputs) :
+ InferModelBase(vdevice, std::move(hef), std::move(inputs), std::move(outputs)),
+ m_client(client),
+ m_handle(handle),
+ m_vdevice_handle(vdevice_handle)
+{
+}
+
+InferModelHrpcClient::~InferModelHrpcClient()
+{
+ if (INVALID_HANDLE_ID == m_handle) {
+ return;
+ }
+
+ auto request = DestroyInferModelSerializer::serialize_request(m_handle);
+ if (!request) {
+ LOGGER__CRITICAL("Failed to serialize InferModel_release request");
+ return;
+ }
+
+ auto client = m_client.lock();
+ if (client) {
+ auto execute_request_result = client->execute_request(HailoRpcActionID::INFER_MODEL__DESTROY, MemoryView(*request));
+ if (!execute_request_result) {
+ LOGGER__CRITICAL("Failed to destroy infer model! status = {}", execute_request_result.status());
+ return;
+ }
+
+ auto deserialize_reply_result = DestroyInferModelSerializer::deserialize_reply(MemoryView(*execute_request_result));
+ if (HAILO_SUCCESS != deserialize_reply_result) {
+ LOGGER__CRITICAL("Failed to destroy infer model! status = {}", deserialize_reply_result);
+ return;
+ }
+ }
+}
+
+Expected<ConfiguredInferModel> InferModelHrpcClient::configure()
+{
+ rpc_create_configured_infer_model_request_params_t request_params;
+ for (const auto &input : m_inputs) {
+ rpc_stream_params_t current_stream_params;
+ current_stream_params.format_order = static_cast<uint32_t>(input.second.format().order);
+ current_stream_params.format_type = static_cast<uint32_t>(input.second.format().type);
+ current_stream_params.nms_iou_threshold = input.second.nms_iou_threshold();
+ current_stream_params.nms_score_threshold = input.second.nms_score_threshold();
+ current_stream_params.nms_max_proposals_per_class = input.second.nms_max_proposals_per_class();
+ current_stream_params.nms_max_accumulated_mask_size = input.second.nms_max_accumulated_mask_size();
+
+ request_params.input_streams_params[input.second.name()] = current_stream_params;
+ }
+
+ for (const auto &output : m_outputs) {
+ rpc_stream_params_t current_stream_params;
+ current_stream_params.format_order = static_cast<uint32_t>(output.second.format().order);
+ current_stream_params.format_type = static_cast<uint32_t>(output.second.format().type);
+ current_stream_params.nms_iou_threshold = output.second.nms_iou_threshold();
+ current_stream_params.nms_score_threshold = output.second.nms_score_threshold();
+ current_stream_params.nms_max_proposals_per_class = output.second.nms_max_proposals_per_class();
+ current_stream_params.nms_max_accumulated_mask_size = output.second.nms_max_accumulated_mask_size();
+
+ request_params.output_streams_params[output.second.name()] = current_stream_params;
+ }
+
+ request_params.batch_size = m_config_params.batch_size;
+ request_params.power_mode = m_config_params.power_mode;
+ request_params.latency_flag = m_config_params.latency;
+ request_params.infer_model_handle = m_handle;
+ request_params.vdevice_handle = m_vdevice_handle;
+
+ TRY(auto request, CreateConfiguredInferModelSerializer::serialize_request(request_params));
+ auto client = m_client.lock();
+ CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE,
+ "Lost comunication with the server. This may happen if VDevice is released while the InferModel is in use.");
+ TRY(auto result, client->execute_request(HailoRpcActionID::INFER_MODEL__CREATE_CONFIGURED_INFER_MODEL,
+ MemoryView(request)));
+ TRY(auto tuple, CreateConfiguredInferModelSerializer::deserialize_reply(MemoryView(result)));
+ CHECK_SUCCESS_AS_EXPECTED(std::get<0>(tuple));
+ auto configured_infer_handle = std::get<1>(tuple);
+ auto async_queue_size = std::get<2>(tuple);
+
+ std::unordered_map<std::string, size_t> inputs_frame_sizes;
+ std::unordered_map<std::string, size_t> outputs_frame_sizes;
+ for (const auto &input : m_inputs) {
+ inputs_frame_sizes.emplace(input.second.name(), input.second.get_frame_size());
+ }
+ for (const auto &output : m_outputs) {
+ outputs_frame_sizes.emplace(output.second.name(), output.second.get_frame_size());
+ }
+
+ auto callbacks_queue = make_unique_nothrow<CallbacksQueue>(client, m_output_names);
+ CHECK_NOT_NULL_AS_EXPECTED(callbacks_queue, HAILO_OUT_OF_HOST_MEMORY);
+
+ TRY(auto input_vstream_infos, m_hef.get_input_vstream_infos());
+ TRY(auto output_vstream_infos, m_hef.get_output_vstream_infos());
+ TRY(auto cim_client_ptr, ConfiguredInferModelHrpcClient::create(client,
+ configured_infer_handle,
+ std::move(input_vstream_infos), std::move(output_vstream_infos),
+ async_queue_size, std::move(callbacks_queue), m_handle,
+ inputs_frame_sizes, outputs_frame_sizes));
+
+ return ConfiguredInferModelBase::create(cim_client_ptr);
+}
+
+Expected<ConfiguredInferModel> InferModelHrpcClient::configure_for_ut(std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
+ const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes,
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes,
+ std::shared_ptr<ConfiguredNetworkGroup> net_group)
+{
+ (void)async_infer_runner;
+ (void)input_names;
+ (void)output_names;
+ (void)net_group;
+ (void)inputs_frame_sizes;
+ (void)outputs_frame_sizes;
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+} /* namespace hailort */
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file infer_model_hrpc_client.hpp
+ * @brief Infer model HRPC client, represents the user's handle to the InferModel object
+ **/
+
+#ifndef _HAILO_INFER_MODEL_HRPC_CLIENT_HPP_
+#define _HAILO_INFER_MODEL_HRPC_CLIENT_HPP_
+
+#include "hailo/hailort.h"
+#include "hailo/infer_model.hpp"
+#include "hrpc/client.hpp"
+#include "net_flow/pipeline/infer_model_internal.hpp"
+
+namespace hailort
+{
+
+class InferModelHrpcClient : public InferModelBase
+{
+public:
+ static Expected<std::shared_ptr<InferModelHrpcClient>> create(Hef &&hef,
+ std::shared_ptr<hrpc::Client> client, uint32_t infer_model_handle_id,
+ uint32_t vdevice_handle, VDevice &vdevice);
+
+ InferModelHrpcClient(std::shared_ptr<hrpc::Client> client, uint32_t id,
+ uint32_t vdevice_handle, VDevice &vdevice, Hef &&hef, std::unordered_map<std::string, InferStream> &&inputs,
+ std::unordered_map<std::string, InferStream> &&outputs);
+ virtual ~InferModelHrpcClient();
+
+ InferModelHrpcClient(const InferModelHrpcClient &) = delete;
+ InferModelHrpcClient &operator=(const InferModelHrpcClient &) = delete;
+ InferModelHrpcClient(InferModelHrpcClient &&) = delete;
+ InferModelHrpcClient &operator=(InferModelHrpcClient &&) = delete;
+
+ virtual Expected<ConfiguredInferModel> configure() override;
+
+ virtual Expected<ConfiguredInferModel> configure_for_ut(std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
+ const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes = {},
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes = {},
+ std::shared_ptr<ConfiguredNetworkGroup> net_group = nullptr) override;
+
+private:
+ std::weak_ptr<hrpc::Client> m_client;
+ uint32_t m_handle;
+ uint32_t m_vdevice_handle;
+};
+
+} /* namespace hailort */
+
+#endif /* _HAILO_INFER_MODEL_HRPC_CLIENT_HPP_ */
/**
* @file infer_model_internal.hpp
* @brief Implemention of the infer model
+ *
+ * InferModel (Interface)
+ * |-- InferModelBase (Base implementation)
+ * |-- InferModelHrpcClient (RPC handle communicating with the server)
**/
#ifndef _HAILO_INFER_MODEL_INTERNAL_HPP_
#define _HAILO_INFER_MODEL_INTERNAL_HPP_
#include "hailo/infer_model.hpp"
-#include "hailo/vstream.hpp"
#include "net_flow/pipeline/async_infer_runner.hpp"
#include "net_flow/ops/nms_post_process.hpp"
+#include "hrpc/client.hpp"
namespace hailort
{
TransferDoneCallbackAsyncInfer m_stream_callback;
};
+class InferModelBase : public InferModel
+{
+public:
+ static Expected<std::shared_ptr<InferModelBase>> create(VDevice &vdevice, const std::string &hef_path);
+ static Expected<std::shared_ptr<InferModelBase>> create(VDevice &vdevice, const MemoryView hef_buffer);
+
+ InferModelBase(VDevice &vdevice, Hef &&hef, std::unordered_map<std::string, InferStream> &&inputs,
+ std::unordered_map<std::string, InferStream> &&outputs);
+ virtual ~InferModelBase() = default;
+ InferModelBase(InferModelBase &&);
+
+ virtual const Hef &hef() const override;
+ virtual void set_batch_size(uint16_t batch_size) override;
+ virtual void set_power_mode(hailo_power_mode_t power_mode) override;
+ virtual void set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency) override;
+ virtual Expected<ConfiguredInferModel> configure() override;
+ virtual Expected<InferStream> input() override;
+ virtual Expected<InferStream> output() override;
+ virtual Expected<InferStream> input(const std::string &name) override;
+ virtual Expected<InferStream> output(const std::string &name) override;
+ virtual const std::vector<InferStream> &inputs() const override;
+ virtual const std::vector<InferStream> &outputs() const override;
+ virtual const std::vector<std::string> &get_input_names() const override;
+ virtual const std::vector<std::string> &get_output_names() const override;
+
+ virtual Expected<ConfiguredInferModel> configure_for_ut(std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
+ const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes = {},
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes = {},
+ std::shared_ptr<ConfiguredNetworkGroup> net_group = nullptr) override;
+
+protected:
+ static Expected<std::unordered_map<std::string, InferModel::InferStream>> create_infer_stream_inputs(Hef &hef);
+ static Expected<std::unordered_map<std::string, InferModel::InferStream>> create_infer_stream_outputs(Hef &hef);
+
+ std::reference_wrapper<VDevice> m_vdevice;
+ Hef m_hef;
+ std::unordered_map<std::string, InferStream> m_inputs;
+ std::unordered_map<std::string, InferStream> m_outputs;
+ std::vector<InferStream> m_inputs_vector;
+ std::vector<InferStream> m_outputs_vector;
+ std::vector<std::string> m_input_names;
+ std::vector<std::string> m_output_names;
+ ConfigureNetworkParams m_config_params;
+};
+
class InferModel::InferStream::Impl
{
public:
void set_nms_max_proposals_per_class(uint32_t max_proposals_per_class);
void set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size);
+ float32_t nms_score_threshold() const;
+ float32_t nms_iou_threshold() const;
+ uint32_t nms_max_proposals_per_class() const;
+ uint32_t nms_max_accumulated_mask_size() const;
+
private:
friend class InferModel;
+ friend class InferModelBase;
+ friend class VDevice;
hailo_vstream_info_t m_vstream_info;
hailo_format_t m_user_buffer_format;
uint32_t m_nms_max_accumulated_mask_size;
};
-class AsyncInferJob::Impl
+class AsyncInferJobBase
+{
+public:
+ static AsyncInferJob create(std::shared_ptr<AsyncInferJobBase> base);
+ virtual ~AsyncInferJobBase() = default;
+ virtual hailo_status wait(std::chrono::milliseconds timeout) = 0;
+};
+
+class AsyncInferJobImpl : public AsyncInferJobBase
{
public:
- Impl(uint32_t streams_count);
- hailo_status wait(std::chrono::milliseconds timeout);
+ AsyncInferJobImpl(uint32_t streams_count);
+ virtual hailo_status wait(std::chrono::milliseconds timeout) override;
+
+private:
+ friend class ConfiguredInferModelBase;
+
bool stream_done(const hailo_status &status);
hailo_status completion_status();
void mark_callback_done();
-private:
std::condition_variable m_cv;
std::mutex m_mutex;
std::atomic_uint32_t m_ongoing_transfers;
hailo_status m_job_completion_status;
};
-class ConfiguredInferModelImpl
+/*
+ * ConfiguredInferModel (interface wrapper - external API)
+ * |-- ConfiguredInferModelBase (interface)
+ * |--|-- ConfiguredInferModelImpl (non-RPC implementation)
+ * |--|-- ConfiguredInferModelHrpcClient (RPC handle communicating with the server)
+ */
+
+class ConfiguredInferModelBase
+{
+public:
+ static ConfiguredInferModel create(std::shared_ptr<ConfiguredInferModelBase> base);
+
+ ConfiguredInferModelBase(const std::unordered_map<std::string, size_t> inputs_frame_sizes,
+ const std::unordered_map<std::string, size_t> outputs_frame_sizes);
+ virtual ~ConfiguredInferModelBase() = default;
+ virtual Expected<ConfiguredInferModel::Bindings> create_bindings() = 0;
+ virtual hailo_status wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count = 1) = 0;
+ virtual hailo_status activate() = 0;
+ virtual hailo_status deactivate() = 0;
+ virtual hailo_status run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout);
+ virtual Expected<AsyncInferJob> run_async(ConfiguredInferModel::Bindings bindings,
+ std::function<void(const AsyncInferCompletionInfo &)> callback = ASYNC_INFER_EMPTY_CALLBACK) = 0;
+ virtual Expected<LatencyMeasurementResult> get_hw_latency_measurement() = 0;
+ virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout) = 0;
+ virtual hailo_status set_scheduler_threshold(uint32_t threshold) = 0;
+ virtual hailo_status set_scheduler_priority(uint8_t priority) = 0;
+ virtual Expected<size_t> get_async_queue_size() = 0;
+ virtual hailo_status shutdown() = 0;
+
+ static Expected<ConfiguredInferModel::Bindings> create_bindings(
+ std::unordered_map<std::string, ConfiguredInferModel::Bindings::InferStream> &&inputs,
+ std::unordered_map<std::string, ConfiguredInferModel::Bindings::InferStream> &&outputs);
+ static Expected<ConfiguredInferModel::Bindings::InferStream> create_infer_stream(const hailo_vstream_info_t &vstream_info);
+ static BufferType get_infer_stream_buffer_type(ConfiguredInferModel::Bindings::InferStream expected_stream);
+ static bool get_stream_done(hailo_status status, std::shared_ptr<AsyncInferJobImpl> job_pimpl);
+ static hailo_status get_completion_status(std::shared_ptr<AsyncInferJobImpl> job_pimpl);
+ static void mark_callback_done(std::shared_ptr<AsyncInferJobImpl> job_pimpl);
+
+private:
+ virtual hailo_status validate_bindings(ConfiguredInferModel::Bindings bindings) = 0;
+
+protected:
+ std::unordered_map<std::string, size_t> m_inputs_frame_sizes;
+ std::unordered_map<std::string, size_t> m_outputs_frame_sizes;
+
+};
+
+class ConfiguredInferModelImpl : public ConfiguredInferModelBase
{
public:
static Expected<std::shared_ptr<ConfiguredInferModelImpl>> create(std::shared_ptr<ConfiguredNetworkGroup> net_group,
const std::unordered_map<std::string, hailo_format_t> &inputs_formats, const std::unordered_map<std::string, hailo_format_t> &outputs_formats,
const std::vector<std::string> &input_names, const std::vector<std::string> &output_names, VDevice &vdevice,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes, const std::unordered_map<std::string, size_t> outputs_frame_sizes,
const uint32_t timeout = HAILO_DEFAULT_VSTREAM_TIMEOUT_MS);
- ConfiguredInferModelImpl(std::shared_ptr<ConfiguredNetworkGroup> cng,
- std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
- const std::vector<std::string> &input_names,
- const std::vector<std::string> &output_names);
+ ConfiguredInferModelImpl(std::shared_ptr<ConfiguredNetworkGroup> cng, std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner,
+ const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes, const std::unordered_map<std::string, size_t> outputs_frame_sizes);
~ConfiguredInferModelImpl();
- Expected<ConfiguredInferModel::Bindings> create_bindings();
- hailo_status wait_for_async_ready(std::chrono::milliseconds timeout);
- void abort();
- hailo_status activate();
- void deactivate();
- hailo_status run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout);
- Expected<AsyncInferJob> run_async(ConfiguredInferModel::Bindings bindings,
- std::function<void(const AsyncInferCompletionInfo &)> callback);
- Expected<LatencyMeasurementResult> get_hw_latency_measurement();
- hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout);
- hailo_status set_scheduler_threshold(uint32_t threshold);
- hailo_status set_scheduler_priority(uint8_t priority);
- Expected<size_t> get_async_queue_size();
+ virtual Expected<ConfiguredInferModel::Bindings> create_bindings() override;
+ virtual hailo_status wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count) override;
+ virtual hailo_status activate() override;
+ virtual hailo_status deactivate() override;
+ virtual Expected<AsyncInferJob> run_async(ConfiguredInferModel::Bindings bindings,
+ std::function<void(const AsyncInferCompletionInfo &)> callback) override;
+ virtual Expected<LatencyMeasurementResult> get_hw_latency_measurement() override;
+ virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout) override;
+ virtual hailo_status set_scheduler_threshold(uint32_t threshold) override;
+ virtual hailo_status set_scheduler_priority(uint8_t priority) override;
+ virtual Expected<size_t> get_async_queue_size() override;
+ virtual hailo_status shutdown() override;
static Expected<std::shared_ptr<ConfiguredInferModelImpl>> create_for_ut(std::shared_ptr<ConfiguredNetworkGroup> net_group,
- std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner, const std::vector<std::string> &input_names, const std::vector<std::string> &output_names);
+ std::shared_ptr<AsyncInferRunnerImpl> async_infer_runner, const std::vector<std::string> &input_names, const std::vector<std::string> &output_names,
+ const std::unordered_map<std::string, size_t> inputs_frame_sizes, const std::unordered_map<std::string, size_t> outputs_frame_sizes);
private:
- hailo_status validate_bindings(ConfiguredInferModel::Bindings bindings);
+ virtual hailo_status validate_bindings(ConfiguredInferModel::Bindings bindings);
- std::shared_ptr<ConfiguredNetworkGroup> m_cng;
+ std::weak_ptr<ConfiguredNetworkGroup> m_cng;
std::unique_ptr<ActivatedNetworkGroup> m_ang;
std::shared_ptr<AsyncInferRunnerImpl> m_async_infer_runner;
uint32_t m_ongoing_parallel_transfers;
m_barrier->arrive_and_wait();
if (HAILO_SUCCESS == m_pipeline_status->load()) {
- std::unique_lock<std::mutex> lock(m_mutex);
- m_input_buffers[sink.name()] = std::move(buffer);
- if (m_input_buffers.size() == m_sink_name_to_index.size()) { // Last sink to set its buffer
-
+ size_t input_buffers_size = 0;
+ {
+ std::unique_lock<std::mutex> lock(m_mutex);
+ m_input_buffers[sink.name()] = std::move(buffer);
+ input_buffers_size = m_input_buffers.size();
+ }
+ if (input_buffers_size == m_sink_name_to_index.size()) { // Last sink to set its buffer
for (auto &input_buffer : m_input_buffers) {
if (HAILO_SUCCESS != input_buffer.second.action_status()) {
handle_non_recoverable_async_error(input_buffer.second.action_status());
std::map<std::string, MemoryView> inputs;
std::map<std::string, MemoryView> outputs;
for (size_t i = 0; i < input_buffers.size(); ++i) {
- inputs.insert({m_sinks_names[i], input_buffers[i].as_view()});
+ TRY(auto src, input_buffers[i].as_view(BufferProtection::READ));
+ inputs.insert({m_sinks_names[i], src});
}
auto pool = next_pad_downstream().element().get_buffer_pool();
assert(pool);
}
}
CHECK_EXPECTED(acquired_buffer);
- outputs.insert({"", acquired_buffer->as_view()}); // TODO: fill with correct name
+ TRY(auto dst, acquired_buffer->as_view(BufferProtection::WRITE));
+ outputs.insert({"", dst}); // TODO: fill with correct name
m_duration_collector.start_measurement();
auto post_process_result = m_nms_op->execute(inputs, outputs);
input_views.reserve(inputs.size());
for (auto &input_buf : inputs) {
- input_views.push_back(input_buf.as_view());
+ TRY(auto src, input_buf.as_view(BufferProtection::READ));
+ input_views.push_back(src);
}
auto pool = next_pad_downstream().element().get_buffer_pool();
assert(pool);
CHECK_EXPECTED(acquired_buffer);
m_duration_collector.start_measurement();
- const auto status = fuse_buffers(input_views, m_nms_infos, acquired_buffer.value().as_view());
+ TRY(auto dst, acquired_buffer.value().as_view(BufferProtection::WRITE));
+ const auto status = fuse_buffers(input_views, m_nms_infos, dst);
m_duration_collector.complete_measurement();
for (auto &input : inputs) {
}
CHECK_EXPECTED(acquired_buffer, "Failed to acquire buffer");
outputs.emplace_back(acquired_buffer.release());
- raw_buffers.push_back(outputs.back().as_view());
+ TRY(auto dst, outputs.back().as_view(BufferProtection::WRITE));
+ raw_buffers.push_back(dst);
}
m_duration_collector.start_measurement();
- const auto status = m_demuxer->transform_demux(input.as_view(), raw_buffers);
+ TRY(auto src, input.as_view(BufferProtection::READ));
+ const auto status = m_demuxer->transform_demux(src, raw_buffers);
m_duration_collector.complete_measurement();
input.set_action_status(status);
CHECK_NOT_NULL_AS_EXPECTED(shared_input_buff, HAILO_OUT_OF_HOST_MEMORY);
for (uint32_t i = 0; i < input_pix_buffer.number_of_planes; i++) {
- outputs.emplace_back(MemoryView(input_pix_buffer.planes[i].user_ptr, input_pix_buffer.planes[i].bytes_used),
- [input_ptr = shared_input_buff](hailo_status status)
- {
+ if (HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR == input_pix_buffer.memory_type) {
+ outputs.emplace_back(MemoryView(input_pix_buffer.planes[i].user_ptr, input_pix_buffer.planes[i].bytes_used),
+ [input_ptr = shared_input_buff](hailo_status status)
+ {
+ if (HAILO_SUCCESS != status) {
+ input_ptr->set_action_status(status);
+ }
+ });
+ } else if (HAILO_PIX_BUFFER_MEMORY_TYPE_DMABUF == input_pix_buffer.memory_type) {
+ hailo_dma_buffer_t dma_buffer;
+ dma_buffer.fd = input_pix_buffer.planes[i].fd;
+ dma_buffer.size = input_pix_buffer.planes[i].plane_size;
+ TransferDoneCallbackAsyncInfer exec_done = [input_ptr = shared_input_buff](hailo_status status) {
if (HAILO_SUCCESS != status) {
input_ptr->set_action_status(status);
- }
- });
+ }};
+ hailo_status action_status = HAILO_SUCCESS;
+ BufferPoolPtr pool = m_sources[i].next()->element().get_buffer_pool();
+ outputs.emplace_back(PipelineBuffer(dma_buffer, exec_done, action_status, pool->is_holding_user_buffers(), pool));
+ } else {
+ return make_unexpected(HAILO_INVALID_ARGUMENT);
+ }
}
}
m_barrier->terminate();
return;
}
- named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_shared->as_view(),
+
+ BufferRepresentation buffer_representation{};
+ if (buffer_shared->get_buffer_type() == BufferType::VIEW) {
+ auto src = buffer_shared->as_view(BufferProtection::READ);
+ if (HAILO_SUCCESS != src.status()) {
+ handle_non_recoverable_async_error(src.status());
+ m_input_buffers.clear();
+ m_barrier->terminate();
+ return;
+ }
+ buffer_representation.buffer_type = BufferType::VIEW;
+ buffer_representation.view = src.value();
+ } else if (buffer_shared->get_buffer_type() == BufferType::DMA_BUFFER) {
+ auto dma_buffer = buffer_shared->get_metadata().get_additional_data<DmaBufferPipelineData>();
+ buffer_representation.buffer_type = BufferType::DMA_BUFFER;
+ buffer_representation.dma_buffer = dma_buffer->m_dma_buffer;
+ } else {
+ handle_non_recoverable_async_error(HAILO_INVALID_ARGUMENT);
+ m_input_buffers.clear();
+ m_barrier->terminate();
+ return;
+ }
+
+ named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_representation,
[buffer_shared](hailo_status status) { buffer_shared->set_action_status(status); }));
}
for (auto &output_buffer : source_name_to_output_buffer) {
const auto &stream_name = m_source_name_to_stream_name.at(output_buffer.first);
- named_buffers_callbacks.emplace(stream_name, std::make_pair(output_buffer.second->as_view(),
+
+ BufferRepresentation buffer_representation{};
+ if (output_buffer.second->get_buffer_type() == BufferType::VIEW) {
+ auto dst = output_buffer.second->as_view(BufferProtection::WRITE);
+ if (HAILO_SUCCESS != dst.status()) {
+ handle_non_recoverable_async_error(dst.status());
+ m_input_buffers.clear();
+ m_barrier->terminate();
+ return;
+ }
+ buffer_representation.buffer_type = BufferType::VIEW;
+ buffer_representation.view = dst.value();
+ } else if (output_buffer.second->get_buffer_type() == BufferType::DMA_BUFFER) {
+ auto dma_buffer = output_buffer.second->get_metadata().get_additional_data<DmaBufferPipelineData>();
+ buffer_representation.buffer_type = BufferType::DMA_BUFFER;
+ buffer_representation.dma_buffer = dma_buffer->m_dma_buffer;
+ } else {
+ handle_non_recoverable_async_error(HAILO_INVALID_ARGUMENT);
+ m_input_buffers.clear();
+ m_barrier->terminate();
+ return;
+ }
+
+ named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_representation,
[this, buffer = output_buffer.second, source_name = output_buffer.first](hailo_status status){
buffer->set_action_status(status);
if (HAILO_SUCCESS == m_pipeline_status->load()) {
}));
}
- auto status = m_net_group->wait_for_ongoing_callbacks_count_under(m_max_ongoing_transfers);
+ auto cng = m_net_group.lock();
+ if (!cng) {
+ // Error - cng (VDevice) was released mid inference
+ handle_non_recoverable_async_error(HAILO_INTERNAL_FAILURE);
+ m_input_buffers.clear();
+ m_barrier->terminate();
+ return;
+ }
+
+ auto status = cng->wait_for_ongoing_callbacks_count_under(m_max_ongoing_transfers);
if (HAILO_SUCCESS != status ) {
handle_non_recoverable_async_error(status);
m_input_buffers.clear();
return;
}
- status = m_net_group->infer_async(named_buffers_callbacks, [](hailo_status){});
+ status = cng->infer_async(named_buffers_callbacks, [](hailo_status){});
if (HAILO_SUCCESS != status ) {
handle_non_recoverable_async_error(status);
m_input_buffers.clear();
return ret_val;
}
-Expected<uint32_t> AsyncHwElement::get_sink_index_from_input_stream_name(const std::string &input_stream_name)
+Expected<uint8_t> AsyncHwElement::get_sink_index_from_input_stream_name(const std::string &input_stream_name)
{
for (const auto &name_pair : m_sink_name_to_stream_name) {
if (name_pair.second == input_stream_name) {
- return Expected<uint32_t>(m_sink_name_to_index.at(name_pair.first));
+ auto cpy = static_cast<uint8_t>(m_sink_name_to_index.at(name_pair.first));
+ return cpy;
}
}
return make_unexpected(HAILO_INVALID_ARGUMENT);
m_barrier->terminate();
+ // Best effort - if cng is already released - nothing to shut-down
+ auto shutdown_status = HAILO_SUCCESS;
+ auto cng = m_net_group.lock();
+ if (cng) {
+ shutdown_status = cng->shutdown();
+ }
+
// Checking success of shutdown is best effort (terminate should be called even if shutdown fails)
- auto shutdown_status = m_net_group->shutdown();
auto terminate_status = PipelineElement::execute_terminate(error_status);
CHECK_SUCCESS(shutdown_status);
CHECK_SUCCESS(terminate_status);
virtual Expected<PipelineBuffer> run_pull(PipelineBuffer &&optional, const PipelinePad &source) override;
Expected<uint32_t> get_source_index_from_output_stream_name(const std::string &output_stream_name);
- Expected<uint32_t> get_sink_index_from_input_stream_name(const std::string &input_stream_name);
+ Expected<uint8_t> get_sink_index_from_input_stream_name(const std::string &input_stream_name);
virtual Expected<uint32_t> get_source_index_from_source_name(const std::string &source_name) override;
std::vector<BufferPoolPtr> get_hw_interacted_buffer_pools_h2d();
void handle_error_in_hw_async_elem(hailo_status error_status);
std::chrono::milliseconds m_timeout;
- std::shared_ptr<ConfiguredNetworkGroup> m_net_group;
+ std::weak_ptr<ConfiguredNetworkGroup> m_net_group;
size_t m_max_ongoing_transfers;
std::unordered_map<std::string, std::string> m_sink_name_to_stream_name;
#include "common/utils.hpp"
#include "common/runtime_statistics_internal.hpp"
#include "common/os_utils.hpp"
+#include "utils/dma_buffer_utils.hpp"
#include "hailo/expected.hpp"
#include "hailo/hailort.h"
PipelineBuffer::PipelineBuffer(Type type) :
m_type(type),
- m_pool(nullptr),
m_view(),
+ m_exec_done([](hailo_status){}),
m_metadata(),
m_is_user_buffer(false),
m_should_call_exec_done(true),
- m_action_status(HAILO_SUCCESS)
+ m_action_status(HAILO_SUCCESS),
+ m_buffer_type(BufferType::UNINITIALIZED)
{
- m_exec_done = [buffer_pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer](hailo_status){
- release_buffer(buffer_pool, mem_view, is_user_buffer);
- };
}
PipelineBuffer::PipelineBuffer(hailo_status action_status, const TransferDoneCallbackAsyncInfer &exec_done) :
m_type(Type::DATA),
- m_pool(nullptr),
m_view(),
+ m_exec_done(exec_done),
m_metadata(),
m_is_user_buffer(false),
m_should_call_exec_done(true),
- m_action_status(action_status)
+ m_action_status(action_status),
+ m_buffer_type(BufferType::UNINITIALIZED)
{
- m_exec_done = [buffer_pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){
- exec_done(status);
- release_buffer(buffer_pool, mem_view, is_user_buffer);
- };
}
-PipelineBuffer::PipelineBuffer(MemoryView view, const TransferDoneCallbackAsyncInfer &exec_done, hailo_status action_status, bool is_user_buffer, BufferPoolPtr pool,
- bool should_measure) :
+PipelineBuffer::PipelineBuffer(MemoryView view, const TransferDoneCallbackAsyncInfer &exec_done, hailo_status action_status,
+ bool is_user_buffer, BufferPoolWeakPtr pool, bool should_measure) :
m_type(Type::DATA),
m_pool(pool),
m_view(view),
m_metadata(Metadata(add_timestamp(should_measure))),
m_is_user_buffer(is_user_buffer),
m_should_call_exec_done(true),
- m_action_status(action_status)
+ m_action_status(action_status),
+ m_buffer_type(BufferType::VIEW)
{
- m_exec_done = [buffer_pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){
+ m_exec_done = [pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){
exec_done(status);
- release_buffer(buffer_pool, mem_view, is_user_buffer);
+ if (auto buffer_pool = pool.lock()) {
+ return_buffer_to_pool(buffer_pool, mem_view, is_user_buffer);
+ }
};
}
PipelineBuffer::PipelineBuffer(hailo_pix_buffer_t buffer, const TransferDoneCallbackAsyncInfer &exec_done) :
m_type(Type::DATA),
- m_pool(nullptr),
m_view(),
+ m_exec_done(exec_done),
m_metadata(),
m_is_user_buffer(false),
m_should_call_exec_done(true),
- m_action_status(HAILO_SUCCESS)
+ m_action_status(HAILO_SUCCESS),
+ m_buffer_type(BufferType::PIX_BUFFER)
{
set_additional_data(std::make_shared<PixBufferPipelineData>(buffer));
- m_exec_done = [buffer_pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){
+}
+
+PipelineBuffer::PipelineBuffer(hailo_dma_buffer_t dma_buffer, const TransferDoneCallbackAsyncInfer &exec_done, hailo_status action_status,
+ bool is_user_buffer, BufferPoolWeakPtr pool, bool should_measure) :
+ m_type(Type::DATA),
+ m_pool(pool),
+ m_view(),
+ m_metadata(Metadata(add_timestamp(should_measure))),
+ m_is_user_buffer(is_user_buffer),
+ m_should_call_exec_done(true),
+ m_action_status(action_status),
+ m_buffer_type(BufferType::DMA_BUFFER)
+{
+ set_additional_data(std::make_shared<DmaBufferPipelineData>(dma_buffer));
+ m_exec_done = [pool = m_pool, dma_buffer = get_metadata().get_additional_data<DmaBufferPipelineData>(), is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){
exec_done(status);
- release_buffer(buffer_pool, mem_view, is_user_buffer);
+ if (auto buffer_pool = pool.lock()) {
+ return_buffer_to_pool(buffer_pool, dma_buffer->m_dma_buffer, is_user_buffer);
+ }
};
}
m_metadata(std::move(other.m_metadata)),
m_is_user_buffer(std::move(other.m_is_user_buffer)),
m_should_call_exec_done(std::exchange(other.m_should_call_exec_done, false)),
- m_action_status(std::move(other.m_action_status))
+ m_action_status(std::move(other.m_action_status)),
+ m_buffer_type(other.m_buffer_type)
{}
PipelineBuffer &PipelineBuffer::operator=(PipelineBuffer &&other)
m_is_user_buffer = std::move(other.m_is_user_buffer);
m_should_call_exec_done = std::exchange(other.m_should_call_exec_done, false);
m_action_status = std::move(other.m_action_status);
+ m_buffer_type = std::move(other.m_buffer_type);
return *this;
}
size_t PipelineBuffer::size() const
{
- return m_view.size();
+ if (BufferType::DMA_BUFFER == m_buffer_type) {
+ auto dma_buffer = get_metadata().get_additional_data<DmaBufferPipelineData>();
+ return dma_buffer->m_dma_buffer.size;
+ } else if (BufferType::PIX_BUFFER == m_buffer_type) {
+ auto pix_buffer = get_metadata().get_additional_data<PixBufferPipelineData>();
+ size_t size = 0;
+ for (uint32_t i = 0; i < pix_buffer->m_pix_buffer.number_of_planes; i++) {
+ size += pix_buffer->m_pix_buffer.planes[i].plane_size;
+ }
+ return size;
+ } else {
+ return m_view.size();
+ }
}
-MemoryView PipelineBuffer::as_view()
+Expected<MemoryView> PipelineBuffer::as_view(BufferProtection dma_buffer_protection)
{
- return m_view;
+ if (BufferType::DMA_BUFFER == m_buffer_type) {
+ assert(BufferProtection::NONE != dma_buffer_protection);
+ auto status = set_dma_buf_as_memview(dma_buffer_protection);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+ } else if (BufferType::PIX_BUFFER == m_buffer_type) {
+ LOGGER__ERROR("Can't call as_view for buffer of type pix_buffer.");
+ return make_unexpected(HAILO_INVALID_ARGUMENT);
+ }
+
+ auto mem_view = m_view;
+
+ return mem_view;
}
PipelineBuffer::Type PipelineBuffer::get_type() const
return m_metadata;
}
+BufferType PipelineBuffer::get_buffer_type() const
+{
+ return m_buffer_type;
+}
+
Expected<hailo_pix_buffer_t> PipelineBuffer::as_hailo_pix_buffer(hailo_format_order_t order)
{
auto pix_buffer = get_metadata().get_additional_data<PixBufferPipelineData>();
if (nullptr == pix_buffer) {
- auto mem_view = as_view();
+ TRY(auto mem_view, as_view(BufferProtection::READ));
return HailoRTCommon::as_hailo_pix_buffer(mem_view, order);
} else {
uint32_t expected_number_of_planes;
}
}
-void PipelineBuffer::set_metadata(Metadata &&val)
+void PipelineBuffer::set_metadata_start_time(PipelineTimePoint val)
{
- m_metadata = std::move(val);
+ m_metadata.set_start_time(val);
}
PipelineTimePoint PipelineBuffer::add_timestamp(bool should_measure)
return should_measure ? std::chrono::steady_clock::now() : PipelineTimePoint{};
}
-void PipelineBuffer::release_buffer(BufferPoolPtr buffer_pool_ptr, MemoryView mem_view, bool is_user_buffer)
+void PipelineBuffer::return_buffer_to_pool(BufferPoolWeakPtr buffer_pool_weak_ptr, MemoryView mem_view, bool is_user_buffer)
{
- if ((nullptr != buffer_pool_ptr) && (!is_user_buffer)) {
- hailo_status status = buffer_pool_ptr->release_buffer(mem_view);
+ if (is_user_buffer) {
+ return;
+ }
+
+ if (auto buffer_pool_ptr = buffer_pool_weak_ptr.lock() ) {
+ auto pipeline_buffer = PipelineBuffer(mem_view, [](hailo_status){}, HAILO_SUCCESS, false, buffer_pool_ptr, buffer_pool_ptr->should_measure_vstream_latency());
+ hailo_status status = buffer_pool_ptr->return_buffer_to_pool(std::move(pipeline_buffer));
+ if (HAILO_SUCCESS != status) {
+ LOGGER__CRITICAL("Releasing buffer in buffer pool failed! status = {}", status);
+ }
+ }
+}
+
+void PipelineBuffer::return_buffer_to_pool(BufferPoolWeakPtr buffer_pool_weak_ptr, hailo_dma_buffer_t dma_buffer, bool is_user_buffer)
+{
+ if (is_user_buffer) {
+ return;
+ }
+
+ if (auto buffer_pool_ptr = buffer_pool_weak_ptr.lock() ) {
+ auto pipeline_buffer = PipelineBuffer(dma_buffer, [](hailo_status){}, HAILO_SUCCESS, false, buffer_pool_ptr, buffer_pool_ptr->should_measure_vstream_latency());
+ hailo_status status = buffer_pool_ptr->return_buffer_to_pool(std::move(pipeline_buffer));
if (HAILO_SUCCESS != status) {
LOGGER__CRITICAL("Releasing buffer in buffer pool failed! status = {}", status);
}
m_action_status = status;
}
+hailo_status PipelineBuffer::set_dma_buf_as_memview(BufferProtection dma_buffer_protection)
+{
+ auto dma_buffer = get_metadata().get_additional_data<DmaBufferPipelineData>();
+
+ TRY(m_view, DmaBufferUtils::mmap_dma_buffer(dma_buffer->m_dma_buffer, dma_buffer_protection));
+
+ m_exec_done = [mem_view=m_view, exec_done=m_exec_done, dma_buffer, dma_buffer_protection](hailo_status status) {
+ auto mumap_status = DmaBufferUtils::munmap_dma_buffer(dma_buffer->m_dma_buffer, mem_view, dma_buffer_protection);
+ if (HAILO_SUCCESS != mumap_status) {
+ LOGGER__ERROR("Failed to unmap dma buffer");
+ status = HAILO_FILE_OPERATION_FAILURE;
+ }
+ exec_done(status);
+ };
+
+ m_buffer_type = BufferType::VIEW;
+ return HAILO_SUCCESS;
+}
+
void PipelineBuffer::call_exec_done()
{
if (m_should_call_exec_done) {
}
const bool measure_vstream_latency = (vstream_flags & HAILO_VSTREAM_STATS_MEASURE_LATENCY) != 0;
- auto free_mem_views = SpscQueue<MemoryView>::create(buffer_count, shutdown_event, BUFFER_POOL_DEFAULT_QUEUE_TIMEOUT);
- CHECK_EXPECTED(free_mem_views);
-
- auto done_cbs = SpscQueue<TransferDoneCallbackAsyncInfer>::create(buffer_count, shutdown_event, BUFFER_POOL_DEFAULT_QUEUE_TIMEOUT);
- CHECK_EXPECTED(done_cbs);
+ TRY(auto pipeline_buffers_queue, SpscQueue<PipelineBuffer>::create(buffer_count, shutdown_event, BUFFER_POOL_DEFAULT_QUEUE_TIMEOUT));
std::vector<Buffer> buffers;
+ buffers.reserve(buffer_count);
+
+ auto buffer_pool_ptr = make_shared_nothrow<BufferPool>(buffer_size, is_empty, measure_vstream_latency, std::move(buffers),
+ std::move(pipeline_buffers_queue), std::move(queue_size_accumulator), buffer_count);
+ CHECK_AS_EXPECTED(nullptr != buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY);
+
if (!is_empty) {
- buffers.reserve(buffer_count);
for (size_t i = 0; i < buffer_count; i++) {
BufferStorageParams buffer_storage_params;
if (is_dma_able) {
auto buffer = Buffer::create(buffer_size, buffer_storage_params);
CHECK_EXPECTED(buffer);
- auto status = free_mem_views->enqueue(MemoryView(buffer.value()));
+ auto pipeline_buffer = PipelineBuffer(MemoryView(buffer.value()), [](hailo_status){}, HAILO_SUCCESS, false, buffer_pool_ptr,
+ buffer_pool_ptr->m_measure_vstream_latency);
+
+ auto status = buffer_pool_ptr->enqueue_buffer(std::move(pipeline_buffer));
CHECK_SUCCESS_AS_EXPECTED(status);
- buffers.emplace_back(buffer.release());
+ buffer_pool_ptr->m_buffers.emplace_back(buffer.release());
}
}
- auto buffer_pool_ptr = make_shared_nothrow<BufferPool>(buffer_size, is_empty, measure_vstream_latency, std::move(buffers),
- free_mem_views.release(), done_cbs.release(), std::move(queue_size_accumulator), buffer_count);
- CHECK_AS_EXPECTED(nullptr != buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY);
-
return buffer_pool_ptr;
}
BufferPool::BufferPool(size_t buffer_size, bool is_holding_user_buffers, bool measure_vstream_latency, std::vector<Buffer> &&buffers,
- SpscQueue<MemoryView> &&free_mem_views, SpscQueue<TransferDoneCallbackAsyncInfer> &&done_cbs, AccumulatorPtr &&queue_size_accumulator,
+ SpscQueue<PipelineBuffer> &&pipeline_buffers_queue, AccumulatorPtr &&queue_size_accumulator,
size_t max_buffer_count) :
m_buffer_size(buffer_size),
m_is_holding_user_buffers(is_holding_user_buffers),
m_max_buffer_count(max_buffer_count),
m_measure_vstream_latency(measure_vstream_latency),
m_buffers(std::move(buffers)),
- m_free_mem_views(std::move(free_mem_views)),
- m_done_cbs(std::move(done_cbs)),
+ m_pipeline_buffers_queue(std::move(pipeline_buffers_queue)),
m_queue_size_accumulator(std::move(queue_size_accumulator)),
m_is_already_running(false)
{
return m_buffer_size.load();
}
-hailo_status BufferPool::enqueue_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done)
+hailo_status BufferPool::enqueue_buffer(PipelineBuffer &&pipeline_buffer)
{
+ // TODO: add support for pix_buffer here (currently enqueue_buffer is called only to add the output_buffers which are never pix_buffers)
m_is_already_running = true;
auto pool_buffer_size = buffer_size();
- CHECK(mem_view.size() == pool_buffer_size, HAILO_INTERNAL_FAILURE,
- "Buffer size is not the same as expected for pool! ({} != {})", mem_view.size(), pool_buffer_size);
+ CHECK(pipeline_buffer.size() == pool_buffer_size, HAILO_INTERNAL_FAILURE,
+ "Buffer size is not the same as expected for pool! ({} != {})", pipeline_buffer.size(), pool_buffer_size);
std::unique_lock<std::mutex> lock(m_enqueue_mutex);
- auto status = m_free_mem_views.enqueue(mem_view);
+ auto status = m_pipeline_buffers_queue.enqueue(std::move(pipeline_buffer));
if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) {
return HAILO_SHUTDOWN_EVENT_SIGNALED;
}
CHECK_SUCCESS(status);
- // TODO: Stop using 2 queues, hold a queue of pipeline_buffer instead.
- status = m_done_cbs.enqueue(exec_done, true); // we get here only if acquire_free_mem_view succeeded, so we want to push cb to keep sync between the queues
- CHECK_SUCCESS(status);
-
return HAILO_SUCCESS;
}
+bool BufferPool::should_measure_vstream_latency()
+{
+ return m_measure_vstream_latency;
+}
bool BufferPool::is_full()
{
return (m_max_buffer_count - num_of_buffers_in_pool() == 0);
}
+size_t BufferPool::max_capacity()
+{
+ return m_max_buffer_count;
+}
+
size_t BufferPool::num_of_buffers_in_pool()
{
- return m_free_mem_views.size_approx();
+ return m_pipeline_buffers_queue.size_approx();
}
bool BufferPool::is_holding_user_buffers()
bool ignore_shutdown_event)
{
m_is_already_running = true;
-
std::unique_lock<std::mutex> lock(m_dequeue_mutex);
- auto mem_view = acquire_free_mem_view(timeout, ignore_shutdown_event);
- if ((HAILO_SUCCESS != mem_view.status()) && (m_is_holding_user_buffers)) {
- auto done_cb = acquire_on_done_cb(timeout, ignore_shutdown_event);
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == done_cb.status()) {
- return make_unexpected(HAILO_SHUTDOWN_EVENT_SIGNALED);
- }
- CHECK_EXPECTED(done_cb);
- done_cb.value()(mem_view.status());
- }
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == mem_view.status()) {
- return make_unexpected(HAILO_SHUTDOWN_EVENT_SIGNALED);
- }
- CHECK_EXPECTED(mem_view);
-
- if (m_is_holding_user_buffers) {
- auto done_cb = acquire_on_done_cb(timeout, true); // we get here only if acquire_free_mem_view succeeded, so we want to pop cb to keep sync between the queues
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == done_cb.status()) {
- return make_unexpected(HAILO_SHUTDOWN_EVENT_SIGNALED);
- }
- CHECK_EXPECTED(done_cb);
-
- return PipelineBuffer(mem_view.release(), done_cb.release(), HAILO_SUCCESS, m_is_holding_user_buffers, shared_from_this(), m_measure_vstream_latency);
- }
-
- return PipelineBuffer(mem_view.release(), [](hailo_status){}, HAILO_SUCCESS, m_is_holding_user_buffers, shared_from_this(), m_measure_vstream_latency);
-}
-
-Expected<MemoryView> BufferPool::acquire_free_mem_view(std::chrono::milliseconds timeout,
- bool ignore_shutdown_event)
-{
if (nullptr != m_queue_size_accumulator) {
- m_queue_size_accumulator->add_data_point(static_cast<double>(m_free_mem_views.size_approx()));
+ m_queue_size_accumulator->add_data_point(static_cast<double>(m_pipeline_buffers_queue.size_approx()));
}
- auto mem_view = m_free_mem_views.dequeue(timeout, ignore_shutdown_event);
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == mem_view.status()) {
- return make_unexpected(mem_view.status());
- }
- else if (HAILO_TIMEOUT == mem_view.status()) {
- LOGGER__WARNING(
- "Failed to acquire buffer because the buffer pool is empty. This could be caused by uneven reading and writing speeds, with a short user-defined timeout. (timeout={}ms)",
- timeout.count());
- return make_unexpected(mem_view.status());
- }
- CHECK_EXPECTED(mem_view);
-
- return mem_view.release();
-}
-
-Expected<TransferDoneCallbackAsyncInfer> BufferPool::acquire_on_done_cb(std::chrono::milliseconds timeout,
- bool ignore_shutdown_event)
-{
- auto done_cb = m_done_cbs.dequeue(timeout, ignore_shutdown_event);
- if (HAILO_SHUTDOWN_EVENT_SIGNALED == done_cb.status()) {
- return make_unexpected(done_cb.status());
+ auto pipeline_buffer = m_pipeline_buffers_queue.dequeue(timeout, ignore_shutdown_event);
+ if (HAILO_SHUTDOWN_EVENT_SIGNALED == pipeline_buffer.status()) {
+ return make_unexpected(pipeline_buffer.status());
}
- else if (HAILO_TIMEOUT == done_cb.status()) {
+ else if (HAILO_TIMEOUT == pipeline_buffer.status()) {
LOGGER__WARNING(
"Failed to acquire buffer because the buffer pool is empty. This could be caused by uneven reading and writing speeds, with a short user-defined timeout. (timeout={}ms)",
timeout.count());
- return make_unexpected(done_cb.status());
+ return make_unexpected(pipeline_buffer.status());
}
- CHECK_EXPECTED(done_cb);
+ CHECK_EXPECTED(pipeline_buffer);
- return done_cb.release();
+ return pipeline_buffer.release();
}
AccumulatorPtr BufferPool::get_queue_size_accumulator()
return acquired_buffer.release();
}
-hailo_status BufferPool::release_buffer(MemoryView mem_view)
+hailo_status BufferPool::return_buffer_to_pool(PipelineBuffer &&pipeline_buffer)
{
std::unique_lock<std::mutex> lock(m_enqueue_mutex);
// This can be called after the shutdown event was signaled so we ignore it here
- return m_free_mem_views.enqueue(std::move(mem_view), true);
+ return m_pipeline_buffers_queue.enqueue(std::move(pipeline_buffer), true);
}
hailo_status BufferPool::map_to_vdevice(VDevice &vdevice, hailo_dma_buffer_direction_t direction)
}
}
-hailo_status PipelineElement::enqueue_execution_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done)
+hailo_status PipelineElement::enqueue_execution_buffer(PipelineBuffer &&pipeline_buffer)
{
- (void)mem_view;
- (void)exec_done;
+ (void)pipeline_buffer;
LOGGER__ERROR("enqueue_execution_buffer is not implemented for {}!", name());
return HAILO_NOT_IMPLEMENTED;
};
return HAILO_SUCCESS;
}
-Expected<bool> PipelineElement::can_push_buffer_upstream()
+Expected<bool> PipelineElement::can_push_buffer_upstream(uint32_t /*frames_count*/)
{
return make_unexpected(HAILO_NOT_IMPLEMENTED);
}
-Expected<bool> PipelineElement::can_push_buffer_downstream()
+Expected<bool> PipelineElement::can_push_buffer_downstream(uint32_t /*frames_count*/)
{
return make_unexpected(HAILO_NOT_IMPLEMENTED);
}
#include "hailo/runtime_statistics.hpp"
#include "hailo/dma_mapped_buffer.hpp"
#include "net_flow/ops/nms_post_process.hpp"
-
+#include "hailo/network_group.hpp"
#include "utils/thread_safe_queue.hpp"
#include <memory>
PUSH,
};
-enum class BufferType
+enum class BufferProtection
{
- UNINITIALIZED,
- VIEW,
- PIX_BUFFER,
- DMA_BUFFER,
+ NONE,
+ READ,
+ WRITE,
+ READ_WRITE,
};
+
using TransferDoneCallbackAsyncInfer = std::function<void(hailo_status)>;
using PipelineTimePoint = std::chrono::steady_clock::time_point;
hailo_pix_buffer_t m_pix_buffer;
};
+struct DmaBufferPipelineData : AdditionalData
+{
+ DmaBufferPipelineData(const hailo_dma_buffer_t &buffer) : m_dma_buffer(buffer) {};
+ hailo_dma_buffer_t m_dma_buffer;
+};
+
class BufferPool;
using BufferPoolPtr = std::shared_ptr<BufferPool>;
+using BufferPoolWeakPtr = std::weak_ptr<BufferPool>;
class PipelineBuffer final
{
PipelineBuffer(Type type);
// TODO HRT-12185: remove the option to pass a lambda as a parameter and save it as a member since it increases the memory consumption Significantly
PipelineBuffer(hailo_status action_status = HAILO_SUCCESS, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){});
- PipelineBuffer(MemoryView view, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){},
- hailo_status action_status = HAILO_SUCCESS, bool is_user_buffer = true, BufferPoolPtr pool = nullptr, bool should_measure = false);
+ PipelineBuffer(MemoryView view, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){}, hailo_status action_status = HAILO_SUCCESS,
+ bool is_user_buffer = true, BufferPoolWeakPtr pool = BufferPoolWeakPtr(), bool should_measure = false);
PipelineBuffer(hailo_pix_buffer_t buffer, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){});
+ PipelineBuffer(hailo_dma_buffer_t dma_buffer, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){},
+ hailo_status action_status = HAILO_SUCCESS, bool is_user_buffer = true, BufferPoolWeakPtr pool = BufferPoolWeakPtr(), bool should_measure = false);
~PipelineBuffer();
uint8_t* data();
size_t size() const;
- MemoryView as_view();
+ Expected<MemoryView> as_view(BufferProtection dma_buffer_protection);
Expected<hailo_pix_buffer_t> as_hailo_pix_buffer(hailo_format_order_t order = HAILO_FORMAT_ORDER_AUTO);
Type get_type() const;
Metadata get_metadata() const;
- void set_metadata(Metadata &&val);
+ void set_metadata_start_time(PipelineTimePoint val);
void set_additional_data(std::shared_ptr<AdditionalData> data) { m_metadata.set_additional_data(data);}
hailo_status action_status();
void set_action_status(hailo_status status);
void call_exec_done();
+ BufferType get_buffer_type() const;
private:
Type m_type;
- BufferPoolPtr m_pool;
+ BufferPoolWeakPtr m_pool;
MemoryView m_view;
TransferDoneCallbackAsyncInfer m_exec_done;
Metadata m_metadata;
bool m_is_user_buffer;
bool m_should_call_exec_done;
hailo_status m_action_status;
+ BufferType m_buffer_type;
static PipelineTimePoint add_timestamp(bool should_measure);
- static void release_buffer(BufferPoolPtr buffer_pool_ptr, MemoryView mem_view, bool is_user_buffer);
+ static void return_buffer_to_pool(BufferPoolWeakPtr buffer_pool_weak_ptr, MemoryView mem_view, bool is_user_buffer);
+ static void return_buffer_to_pool(BufferPoolWeakPtr buffer_pool_weak_ptr, hailo_dma_buffer_t dma_buffer, bool is_user_buffer);
+ hailo_status set_dma_buf_as_memview(BufferProtection dma_buffer_protection);
};
-// The buffer pool has to be created as a shared pointer (via the create function) because we use shared_from_this(),
-// which is only allowed if there is already a shared pointer pointing to "this"!
-class BufferPool : public std::enable_shared_from_this<BufferPool>
+class BufferPool
{
public:
static Expected<BufferPoolPtr> create(size_t buffer_size, size_t buffer_count, EventPtr shutdown_event,
hailo_pipeline_elem_stats_flags_t elem_flags, hailo_vstream_stats_flags_t vstream_flags, bool is_empty = false,
bool dma_able = false);
- BufferPool(size_t buffer_size, bool is_holding_user_buffers, bool measure_vstream_latency, std::vector<Buffer> &&buffers, SpscQueue<MemoryView> &&free_mem_views,
- SpscQueue<TransferDoneCallbackAsyncInfer> &&done_cbs, AccumulatorPtr &&queue_size_accumulator, size_t max_buffer_count);
+ BufferPool(size_t buffer_size, bool is_holding_user_buffers, bool measure_vstream_latency, std::vector<Buffer> &&buffers,
+ SpscQueue<PipelineBuffer> &&pipeline_buffers_queue, AccumulatorPtr &&queue_size_accumulator, size_t max_buffer_count);
virtual ~BufferPool() = default;
size_t buffer_size();
- hailo_status enqueue_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){});
+ hailo_status enqueue_buffer(PipelineBuffer &&pipeline_buffer);
Expected<PipelineBuffer> acquire_buffer(std::chrono::milliseconds timeout, bool ignore_shutdown_event = false);
AccumulatorPtr get_queue_size_accumulator();
Expected<PipelineBuffer> get_available_buffer(PipelineBuffer &&optional, std::chrono::milliseconds timeout);
+ bool should_measure_vstream_latency();
bool is_full();
+ size_t max_capacity();
size_t num_of_buffers_in_pool();
bool is_holding_user_buffers();
hailo_status map_to_vdevice(VDevice &vdevice, hailo_dma_buffer_direction_t direction);
hailo_status set_buffer_size(uint32_t buffer_size);
private:
- Expected<MemoryView> acquire_free_mem_view(std::chrono::milliseconds timeout, bool ignore_shutdown_event = false);
- Expected<TransferDoneCallbackAsyncInfer> acquire_on_done_cb(std::chrono::milliseconds timeout, bool ignore_shutdown_event = false);
- hailo_status release_buffer(MemoryView mem_view);
+ hailo_status return_buffer_to_pool(PipelineBuffer &&pipeline_buffer);
std::atomic<size_t> m_buffer_size;
bool m_is_holding_user_buffers;
// to the mapping objects.
std::vector<hailort::DmaMappedBuffer> m_dma_mapped_buffers;
- SpscQueue<MemoryView> m_free_mem_views;
- SpscQueue<TransferDoneCallbackAsyncInfer> m_done_cbs;
+ SpscQueue<PipelineBuffer> m_pipeline_buffers_queue;
AccumulatorPtr m_queue_size_accumulator;
// we have enqueue and dequeue mutex to allow mpmc
std::mutex m_enqueue_mutex;
std::string links_description() const;
void print_deep_description(std::vector<std::string> &visited_elements);
- virtual hailo_status enqueue_execution_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done);
+ virtual hailo_status enqueue_execution_buffer(PipelineBuffer &&pipeline_buffer);
hailo_status empty_buffer_pool(BufferPoolPtr pool, hailo_status error_status, std::chrono::milliseconds timeout);
- virtual Expected<bool> can_push_buffer_upstream();
- virtual Expected<bool> can_push_buffer_downstream();
+ virtual Expected<bool> can_push_buffer_upstream(uint32_t frames_count);
+ virtual Expected<bool> can_push_buffer_downstream(uint32_t frames_count);
virtual Expected<uint32_t> get_source_index_from_source_name(const std::string &/*source_name*/) {
// This function is overriden in multi-srcs elements
return HAILO_SUCCESS;
}
-Expected<bool> AsyncPushQueueElement::can_push_buffer_downstream()
+Expected<bool> AsyncPushQueueElement::can_push_buffer_downstream(uint32_t frames_count)
{
- return !m_queue.is_queue_full();
+ return m_queue.size_approx() + frames_count < m_queue.max_capacity();
}
Expected<std::shared_ptr<PullQueueElement>> PullQueueElement::create(const std::string &name, std::chrono::milliseconds timeout,
{
CHECK_AS_EXPECTED(optional, HAILO_INVALID_ARGUMENT, "Optional buffer must be valid in {}!", name());
- hailo_status status = m_pool->enqueue_buffer(optional.as_view());
+ hailo_status status = m_pool->enqueue_buffer(std::move(optional));
if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) {
LOGGER__INFO("Shutdown event was signaled in enqueue of queue element {}!", name());
return make_unexpected(HAILO_SHUTDOWN_EVENT_SIGNALED);
virtual hailo_status run_push(PipelineBuffer &&buffer, const PipelinePad &sink) override;
virtual void run_push_async(PipelineBuffer &&buffer, const PipelinePad &sink) override;
virtual hailo_status execute_dequeue_user_buffers(hailo_status error_status) override;
- virtual Expected<bool> can_push_buffer_downstream() override;
+ virtual Expected<bool> can_push_buffer_downstream(uint32_t frames_count) override;
protected:
virtual hailo_status run_in_thread() override;
}
assert(1 == m_entry_element->sinks().size());
- auto status = m_entry_element->sinks()[0].run_push(PipelineBuffer(buffer, [](hailo_status){}, HAILO_SUCCESS, false, nullptr, m_measure_pipeline_latency));
+ auto status = m_entry_element->sinks()[0].run_push(PipelineBuffer(buffer, [](hailo_status){}, HAILO_SUCCESS, false, BufferPoolWeakPtr(), m_measure_pipeline_latency));
if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) {
LOGGER__INFO("Sending to VStream was shutdown!");
status = m_pipeline_status->load();
}
assert(1 == m_entry_element->sources().size());
- auto recv_buffer = m_entry_element->sources()[0].run_pull(PipelineBuffer(buffer, [](hailo_status){}, HAILO_SUCCESS, false, nullptr, m_measure_pipeline_latency));
+ auto recv_buffer = m_entry_element->sources()[0].run_pull(PipelineBuffer(buffer, [](hailo_status){}, HAILO_SUCCESS, false, BufferPoolWeakPtr(), m_measure_pipeline_latency));
auto status = recv_buffer.status();
if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) {
LOGGER__INFO("Receiving to VStream was shutdown!");
#include "net_flow/ops/ssd_post_process.hpp"
#include "net_flow/ops/yolox_post_process.hpp"
#include "net_flow/ops/yolov8_post_process.hpp"
+#include "net_flow/ops/yolov8_bbox_only_post_process.hpp"
#include "net_flow/ops/yolov5_post_process.hpp"
#include "net_flow/ops/yolov5_bbox_only_post_process.hpp"
#include "net_flow/ops/argmax_post_process.hpp"
{
auto metadata = std::dynamic_pointer_cast<net_flow::Yolov8OpMetadata>(op_metadata);
assert(nullptr != metadata);
- auto op_expected = net_flow::YOLOV8PostProcessOp::create(metadata);
- CHECK_EXPECTED(op_expected);
- op = op_expected.release();
- break;
+ if (metadata->nms_config().bbox_only) {
+ auto bbox_only_metadata = std::dynamic_pointer_cast<net_flow::Yolov8BboxOnlyOpMetadata>(op_metadata);
+ assert(nullptr != bbox_only_metadata);
+ TRY(op, net_flow::YOLOv8BboxOnlyPostProcessOp::create(bbox_only_metadata));
+ break;
+ } else {
+ TRY(op, net_flow::YOLOV8PostProcessOp::create(metadata));
+ break;
+ }
}
case (net_flow::OperationType::YOLOV5):
{
m_ongoing_transfers++;
}
+Expected<std::shared_ptr<ConfiguredNetworkGroupBase>> ConfiguredNetworkGroupBase::create(const ConfigureNetworkParams &config_params,
+ std::vector<std::shared_ptr<CoreOp>> &&core_ops, NetworkGroupMetadata &&metadata)
+{
+ auto net_group_ptr = std::shared_ptr<ConfiguredNetworkGroupBase>(new (std::nothrow)
+ ConfiguredNetworkGroupBase(config_params, std::move(core_ops), std::move(metadata)));
+ CHECK_NOT_NULL_AS_EXPECTED(net_group_ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return net_group_ptr;
+}
+
Expected<std::unique_ptr<ActivatedNetworkGroup>> ConfiguredNetworkGroupBase::activate(
const hailo_activate_network_group_params_t &network_group_params)
{
InferRequest infer_request{};
for (auto &named_buffer_callback : named_buffers_callbacks) {
const auto &name = named_buffer_callback.first;
- const auto &buffer = named_buffer_callback.second.first;
const auto &callback = named_buffer_callback.second.second;
- infer_request.transfers.emplace(name, TransferRequest{buffer, callback});
+ if (BufferType::VIEW == named_buffer_callback.second.first.buffer_type) {
+ const auto &buffer = named_buffer_callback.second.first.view;
+ infer_request.transfers.emplace(name, TransferRequest{buffer, callback});
+ } else if (BufferType::DMA_BUFFER == named_buffer_callback.second.first.buffer_type) {
+ const auto &dma_buffer = named_buffer_callback.second.first.dma_buffer;
+ infer_request.transfers.emplace(name, TransferRequest{dma_buffer, callback});
+ } else {
+ LOGGER__ERROR("infer_async does not support buffers with type {}", named_buffer_callback.second.first.buffer_type);
+ return HAILO_INVALID_ARGUMENT;
+ }
}
infer_request.callback = [this, infer_request_done_cb](hailo_status status){
if (status == HAILO_STREAM_ABORT) {
return HAILO_SUCCESS;
}
+Expected<uint32_t> ConfiguredNetworkGroupBase::get_cache_read_size() const
+{
+ CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION,
+ "get_cache_read_size() is not supported for multi core-op network groups");
+
+ return m_core_ops[0]->get_cache_read_size();
+
+}
+
+Expected<uint32_t> ConfiguredNetworkGroupBase::get_cache_write_size() const
+{
+ CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION,
+ "get_cache_write_size() is not supported for multi core-op network groups");
+
+ return m_core_ops[0]->get_cache_write_size();
+}
+
+
+hailo_status ConfiguredNetworkGroupBase::init_cache(uint32_t read_offset, int32_t write_offset_delta)
+{
+ CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION,
+ "init_cache() is not supported for multi core-op network groups");
+
+ return m_core_ops[0]->init_cache(read_offset, write_offset_delta);
+}
+
+Expected<hailo_cache_info_t> ConfiguredNetworkGroupBase::get_cache_info() const
+{
+ CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION,
+ "get_cache_info() is not supported for multi core-op network groups");
+
+ return m_core_ops[0]->get_cache_info();
+}
+
+hailo_status ConfiguredNetworkGroupBase::update_cache_offset(int32_t offset_delta_bytes)
+{
+ CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION,
+ "update_cache_offset() is not supported for multi core-op network groups");
+
+ return m_core_ops[0]->update_cache_offset(offset_delta_bytes);
+}
+
} /* namespace hailort */
using stream_name_t = std::string;
using op_name_t = std::string;
+#define HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR "HAILORT_AUTO_UPDATE_CACHE_OFFSET"
+#define HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DEFAULT "default"
+#define HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DISABLED "disabled"
class ConfiguredNetworkGroupBase : public ConfiguredNetworkGroup
{
public:
static Expected<std::shared_ptr<ConfiguredNetworkGroupBase>> create(const ConfigureNetworkParams &config_params,
- std::vector<std::shared_ptr<CoreOp>> &&core_ops, NetworkGroupMetadata &&metadata)
- {
- auto net_group_ptr = std::shared_ptr<ConfiguredNetworkGroupBase>(new (std::nothrow)
- ConfiguredNetworkGroupBase(config_params, std::move(core_ops), std::move(metadata)));
- CHECK_NOT_NULL_AS_EXPECTED(net_group_ptr, HAILO_OUT_OF_HOST_MEMORY);
-
- return net_group_ptr;
- }
+ std::vector<std::shared_ptr<CoreOp>> &&core_ops, NetworkGroupMetadata &&metadata);
virtual ~ConfiguredNetworkGroupBase() = default;
ConfiguredNetworkGroupBase(const ConfiguredNetworkGroupBase &other) = delete;
virtual hailo_status set_nms_max_accumulated_mask_size(const std::string &edge_name, uint32_t max_accumulated_mask_size) override;
Expected<std::shared_ptr<net_flow::NmsOpMetadata>> get_nms_meta_data(const std::string &edge_name);
+
+ virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override;
+ virtual Expected<hailo_cache_info_t> get_cache_info() const override;
+ virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override;
+
private:
ConfiguredNetworkGroupBase(const ConfigureNetworkParams &config_params,
std::vector<std::shared_ptr<CoreOp>> &&core_ops, NetworkGroupMetadata &&metadata);
hailo_status activate_low_level_streams();
hailo_status deactivate_low_level_streams();
+ Expected<uint32_t> get_cache_read_size() const;
+ Expected<uint32_t> get_cache_write_size() const;
const ConfigureNetworkParams m_config_params;
std::vector<std::shared_ptr<CoreOp>> m_core_ops;
virtual hailo_status set_nms_max_bboxes_per_class(const std::string &edge_name, uint32_t max_bboxes_per_class) override;
virtual hailo_status set_nms_max_accumulated_mask_size(const std::string &edge_name, uint32_t max_accumulated_mask_size) override;
+ virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override;
+ virtual Expected<hailo_cache_info_t> get_cache_info() const override;
+ virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override;
+
private:
ConfiguredNetworkGroupClient(NetworkGroupIdentifier &&identifier, const std::string &network_group_name);
hailo_status create_client();
namespace hailort
{
-Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer_write(hailo_dma_buffer_t dma_buffer)
+Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer(hailo_dma_buffer_t dma_buffer, BufferProtection dma_buffer_protection)
{
- void* dma_buf_ptr = mmap(NULL, dma_buffer.size, PROT_WRITE, MAP_SHARED, dma_buffer.fd, 0);
- CHECK_AS_EXPECTED(MAP_FAILED != dma_buf_ptr, HAILO_INTERNAL_FAILURE, "Failed to run mmap on DMA buffer for writing");
+ int prot = 0;
+ uint64_t dma_buf_sync_flags = 0;
+ if (BufferProtection::READ == dma_buffer_protection) {
+ prot = PROT_READ;
+ dma_buf_sync_flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ;
+ } else if (BufferProtection::WRITE == dma_buffer_protection) {
+ prot = PROT_WRITE;
+ dma_buf_sync_flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE;
+ } else {
+ return make_unexpected(HAILO_INVALID_ARGUMENT);
+ }
+
+ void* dma_buf_ptr = mmap(NULL, dma_buffer.size, prot, MAP_SHARED, dma_buffer.fd, 0);
+ CHECK_AS_EXPECTED(MAP_FAILED != dma_buf_ptr, HAILO_INTERNAL_FAILURE, "Failed to run mmap on DMA buffer");
struct dma_buf_sync sync = {
- .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE,
+ .flags = dma_buf_sync_flags,
};
auto err = ioctl(dma_buffer.fd, DMA_BUF_IOCTL_SYNC, &sync);
- CHECK_AS_EXPECTED(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC ioctl, errno {}", err);
+ CHECK_AS_EXPECTED(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC on FD, size: {}, fd: {}, address: {}, errno {}", dma_buffer.size,
+ dma_buffer.fd, static_cast<void*>(dma_buf_ptr), err);
return MemoryView(dma_buf_ptr, dma_buffer.size);
}
-hailo_status DmaBufferUtils::munmap_dma_buffer_write(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview)
+hailo_status DmaBufferUtils::munmap_dma_buffer(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview, BufferProtection dma_buffer_protection)
{
- struct dma_buf_sync sync = {
- .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE,
- };
-
- auto err = ioctl(dma_buffer.fd, DMA_BUF_IOCTL_SYNC, &sync);
- CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC ioctl, errno {}", err);
-
- err = munmap(dma_buffer_memview.data(), dma_buffer.size);
- CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to munmap dma buffer, errno {}", err);
-
- return HAILO_SUCCESS;
-}
-
-Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer_read(hailo_dma_buffer_t dma_buffer)
-{
- void* dma_buf_ptr = mmap(NULL, dma_buffer.size, PROT_READ, MAP_SHARED, dma_buffer.fd, 0);
- CHECK_AS_EXPECTED(MAP_FAILED != dma_buf_ptr, HAILO_INTERNAL_FAILURE, "Failed to run mmap on DMA buffer for reading");
+ uint64_t dma_buf_sync_flags = 0;
+ if (BufferProtection::READ == dma_buffer_protection) {
+ dma_buf_sync_flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ;
+ } else if (BufferProtection::WRITE == dma_buffer_protection) {
+ dma_buf_sync_flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE;
+ } else {
+ return make_unexpected(HAILO_INVALID_ARGUMENT);
+ }
struct dma_buf_sync sync = {
- .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ,
+ .flags = dma_buf_sync_flags,
};
- auto err = ioctl(dma_buffer.fd, DMA_BUF_IOCTL_SYNC, &sync);
- CHECK_AS_EXPECTED(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC ioctl, errno {}", err);
- return MemoryView(dma_buf_ptr, dma_buffer.size);
-}
-
-hailo_status DmaBufferUtils::munmap_dma_buffer_read(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview)
-{
- struct dma_buf_sync sync = {
- .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ,
- };
auto err = ioctl(dma_buffer.fd, DMA_BUF_IOCTL_SYNC, &sync);
CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC ioctl, errno {}", err);
- err = munmap(dma_buffer_memview.data(), dma_buffer.size);
- CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to unmap dma buffer, errno {}", err);
+ err = munmap(static_cast<void*>(dma_buffer_memview.data()), dma_buffer.size);
+ CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to munmap dma buffer, size: {}, fd: {}, address: {}, errno {}", dma_buffer.size, dma_buffer.fd,
+ static_cast<void*>(dma_buffer_memview.data()), err);
return HAILO_SUCCESS;
}
namespace hailort
{
-Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer_write(hailo_dma_buffer_t /*dma_buffer*/)
+Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer(hailo_dma_buffer_t /*dma_buffer*/, BufferProtection /*dma_buffer_protection*/)
{
return make_unexpected(HAILO_NOT_IMPLEMENTED);
}
-hailo_status DmaBufferUtils::munmap_dma_buffer_write(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/)
-{
- return HAILO_NOT_IMPLEMENTED;
-}
-
-Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer_read(hailo_dma_buffer_t /*dma_buffer*/)
-{
- return make_unexpected(HAILO_NOT_IMPLEMENTED);
-}
-
-hailo_status DmaBufferUtils::munmap_dma_buffer_read(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/)
+hailo_status DmaBufferUtils::munmap_dma_buffer(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/,
+ BufferProtection /*dma_buffer_protection*/)
{
return HAILO_NOT_IMPLEMENTED;
}
namespace hailort
{
-Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer_write(hailo_dma_buffer_t /*dma_buffer*/)
+Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer(hailo_dma_buffer_t /*dma_buffer*/, BufferProtection /*dma_buffer_protection*/)
{
return make_unexpected(HAILO_NOT_IMPLEMENTED);
}
-hailo_status DmaBufferUtils::munmap_dma_buffer_write(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/)
-{
- return HAILO_NOT_IMPLEMENTED;
-}
-
-Expected<MemoryView> DmaBufferUtils::mmap_dma_buffer_read(hailo_dma_buffer_t /*dma_buffer*/)
-{
- return make_unexpected(HAILO_NOT_IMPLEMENTED);
-}
-
-hailo_status DmaBufferUtils::munmap_dma_buffer_read(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/)
+hailo_status DmaBufferUtils::munmap_dma_buffer(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/,
+ BufferProtection /*dma_buffer_protection*/)
{
return HAILO_NOT_IMPLEMENTED;
}
#include "hef/hef_internal.hpp"
#include "hailort_rpc_client.hpp"
#include "net_flow/ops_metadata/yolov8_op_metadata.hpp"
+#include "net_flow/ops_metadata/yolov8_bbox_only_op_metadata.hpp"
#include "net_flow/ops_metadata/yolox_op_metadata.hpp"
#include "net_flow/ops_metadata/ssd_op_metadata.hpp"
#include "net_flow/ops_metadata/softmax_op_metadata.hpp"
#include "net_flow/ops_metadata/nms_op_metadata.hpp"
#include "net_flow/ops_metadata/yolov5_op_metadata.hpp"
#include "net_flow/ops_metadata/yolov5_seg_op_metadata.hpp"
+#include "net_flow/ops_metadata/yolov5_bbox_only_op_metadata.hpp"
#include <grpcpp/health_check_service_interface.h>
VDevice_convert_identifier_to_proto(identifier, proto_identifier);
VDevice_get_callback_id_Reply reply;
- ClientContextWithTimeout context;
+ grpc::ClientContext context;
grpc::Status status = m_stub->VDevice_get_callback_id(&context, request, &reply);
CHECK_GRPC_STATUS_AS_EXPECTED(status);
assert(reply.status() < HAILO_STATUS_COUNT);
nms_config_proto.number_of_classes(),
nms_config_proto.background_removal(),
nms_config_proto.background_removal_index(),
- nms_config_proto.cross_classes()};
+ nms_config_proto.cross_classes(),
+ nms_config_proto.bbox_only()};
}
switch (static_cast<net_flow::OperationType>(op_metadata_proto.type())) {
case net_flow::OperationType::YOLOV8:
{
- auto expected_yolov8_post_process_config = create_yolov8_post_process_config(op_metadata_proto);
- CHECK_EXPECTED(expected_yolov8_post_process_config);
- auto expteted_yolov8_metadata = hailort::net_flow::Yolov8OpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config,
- expected_yolov8_post_process_config.value(),
- op_metadata_proto.network_name());
- CHECK_EXPECTED(expteted_yolov8_metadata);
- ops_metadata_ptr.push_back(expteted_yolov8_metadata.value());
+ TRY(auto yolov8_post_process_config, create_yolov8_post_process_config(op_metadata_proto));
+ if (nms_post_process_config.bbox_only) {
+ TRY(auto yolov8_bbox_only_metadata, hailort::net_flow::Yolov8BboxOnlyOpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config,
+ yolov8_post_process_config, op_metadata_proto.network_name()));
+ ops_metadata_ptr.push_back(yolov8_bbox_only_metadata);
+ } else {
+ TRY(auto yolov8_metadata, hailort::net_flow::Yolov8OpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config,
+ yolov8_post_process_config, op_metadata_proto.network_name()));
+ ops_metadata_ptr.push_back(yolov8_metadata);
+ }
break;
}
case net_flow::OperationType::YOLOV5:
{
- auto exected_yolov5_post_process_config = create_yolov5_post_process_config(op_metadata_proto);
- CHECK_EXPECTED(exected_yolov5_post_process_config);
- auto expteted_yolov5_metadata = hailort::net_flow::Yolov5OpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config,
- exected_yolov5_post_process_config.value(),
- op_metadata_proto.network_name());
- CHECK_EXPECTED(expteted_yolov5_metadata);
- ops_metadata_ptr.push_back(expteted_yolov5_metadata.value());
+ TRY(auto yolov5_post_process_config, create_yolov5_post_process_config(op_metadata_proto));
+ if (nms_post_process_config.bbox_only) {
+ TRY(auto yolov5_bbox_only_metadata, hailort::net_flow::Yolov5BboxOnlyOpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config,
+ yolov5_post_process_config, op_metadata_proto.network_name()));
+ ops_metadata_ptr.push_back(yolov5_bbox_only_metadata);
+ } else {
+ TRY(auto yolov5_metadata, hailort::net_flow::Yolov5OpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config,
+ yolov5_post_process_config, op_metadata_proto.network_name()));
+ ops_metadata_ptr.push_back(yolov5_metadata);
+ }
break;
}
return m_client->ConfiguredNetworkGroup_set_nms_max_accumulated_mask_size(m_identifier, edge_name, max_accumulated_mask_size);
}
+// TODO: support kv-cache over service (HRT-13968)
+hailo_status ConfiguredNetworkGroupClient::init_cache(uint32_t /* read_offset */, int32_t /* write_offset_delta */)
+{
+ return HAILO_NOT_IMPLEMENTED;
+}
+
+Expected<hailo_cache_info_t> ConfiguredNetworkGroupClient::get_cache_info() const
+{
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+hailo_status ConfiguredNetworkGroupClient::update_cache_offset(int32_t /* offset_delta_bytes */)
+{
+ return HAILO_NOT_IMPLEMENTED;
+}
+
hailo_status ConfiguredNetworkGroupClient::execute_callback(const ProtoCallbackIdentifier &cb_id)
{
if (cb_id.cb_type() == CALLBACK_TYPE_TRANSFER) {
std::unique_lock<std::mutex> lock(m_mutex);
for (const auto &name_buffer_cb : named_buffers_callbacks) {
auto cb_idx = get_unique_callback_idx();
- auto name_buffer_cb_tuple = std::make_tuple(name_buffer_cb.first, name_buffer_cb.second.first, name_buffer_cb.second.second);
+ CHECK(BufferType::VIEW == name_buffer_cb.second.first.buffer_type, HAILO_INVALID_OPERATION,
+ "Using dmabuf is not supported when working with hailort_service");
+
+ auto name_buffer_cb_tuple = std::make_tuple(name_buffer_cb.first, name_buffer_cb.second.first.view, name_buffer_cb.second.second);
auto tuple_ptr = make_shared_nothrow<NamedBufferCallbackTuple>(name_buffer_cb_tuple);
CHECK_NOT_NULL(tuple_ptr, HAILO_OUT_OF_HOST_MEMORY);
m_idx_to_callbacks.emplace(cb_idx, tuple_ptr);
- cb_idx_to_stream_buffer.emplace_back(std::make_tuple(cb_idx, name_buffer_cb.first, name_buffer_cb.second.first));
+ cb_idx_to_stream_buffer.emplace_back(std::make_tuple(cb_idx, name_buffer_cb.first, name_buffer_cb.second.first.view));
}
}
{
CHECK(1 == transfer_request.transfer_buffers.size(), HAILO_INVALID_OPERATION,
"NMS Reader stream supports only 1 transfer buffer");
+ CHECK(TransferBufferType::MEMORYVIEW == transfer_request.transfer_buffers[0].type(), HAILO_INVALID_OPERATION,
+ "NMS stream doesn't support DMABUF buffer type");
// Currently leave as transfer request - because nms reader uses transfer request queue
// TODO HRT-12239: Chagge when support async read with any aligned void ptr
return m_reader_thread.launch_transfer(std::move(transfer_request));
assert(1 == transfer_request.transfer_buffers.size());
assert(0 == transfer_request.transfer_buffers[0].offset());
+ // TODO HRT-13827: Add support for nms stream read with dmabuf
auto buffer = transfer_request.transfer_buffers[0].base_buffer();
- auto status = NMSStreamReader::read_nms(*m_base_stream, buffer.data(), 0, buffer.size(), m_stream_interface);
+ assert(buffer.has_value());
+ auto status = NMSStreamReader::read_nms(*m_base_stream, buffer.value().data(), 0, buffer.value().size(), m_stream_interface);
if ((HAILO_STREAM_NOT_ACTIVATED == status) || (HAILO_STREAM_ABORT == status)) {
// On both deactivation/abort, we want to send HAILO_STREAM_ABORT since it is part of the callback
{
CHECK(buffer_info.offset() == 0, HAILO_INTERNAL_FAILURE, "Cant use offset on queued buffer pool");
CHECK(buffer_info.size() == m_storage[0]->size(), HAILO_INTERNAL_FAILURE, "Invalid enqueue buffer size");
- CHECK(buffer_info.base_buffer().data() == m_storage[m_next_enqueue_buffer_index]->data(), HAILO_INTERNAL_FAILURE,
+ TRY(auto base_buffer, buffer_info.base_buffer());
+ CHECK(base_buffer.data() == m_storage[m_next_enqueue_buffer_index]->data(), HAILO_INTERNAL_FAILURE,
"Out of order enqueue for queued stream buffer pool");
- m_queue.push(buffer_info.base_buffer());
+ m_queue.push(base_buffer);
m_next_enqueue_buffer_index = (m_next_enqueue_buffer_index + 1) % (m_storage.size());
return HAILO_SUCCESS;
}
constexpr size_t MIN_QUEUE_SIZE = 2;
constexpr size_t DEFAULT_QUEUE_SIZE = 4;
+constexpr size_t RemoteProcessBufferPool::BACKING_ARRAY_LENGTH;
Expected<std::unique_ptr<RemoteProcessBufferPool>> RemoteProcessBufferPool::create(
hailo_stream_direction_t stream_direction, size_t frame_size, size_t queue_size)
{
- // queue_size must be some (power-of-2 minus 1) in order to fit CircularArray.
- queue_size = get_nearest_powerof_2(static_cast<uint32_t>(queue_size + 1), MIN_QUEUE_SIZE) - 1;
- queue_size = std::min(queue_size, ONGOING_TRANSFERS_SIZE);
+ CHECK((queue_size >= MIN_QUEUE_SIZE) && (queue_size < BACKING_ARRAY_LENGTH), HAILO_INVALID_ARGUMENT,
+ "Queue size must be in the range [{}, {}) (received {})", MIN_QUEUE_SIZE, BACKING_ARRAY_LENGTH, queue_size);
hailo_status status = HAILO_UNINITIALIZED;
auto buffer_pool = make_unique_nothrow<RemoteProcessBufferPool>(stream_direction, frame_size, queue_size,
// Guards memory allocation.
std::vector<BufferPtr> m_buffers_guard;
- using BufferQueue = CircularArray<SharedBuffer, std::array<SharedBuffer, ONGOING_TRANSFERS_SIZE>>;
+ // Note: We use a fixed size array to avoid dynamic memory allocation, needed for shared memory.
+ // CircularArrays support working with sizes less than the backing array length.
+ static constexpr size_t BACKING_ARRAY_LENGTH = 1024;
+ using BufferQueue = CircularArray<SharedBuffer, IsNotPow2Tag, std::array<SharedBuffer, BACKING_ARRAY_LENGTH>>;
// On input streams - buffers with user data, ready to be sent to the hw.
// On output streams - buffers that are ready, the stream can receive into them.
buffer.size(), get_frame_size());
auto wrapped_callback = [buffer, user_callback](hailo_status status) {
- user_callback(CompletionInfo{status, buffer.data(), buffer.size()});
+ user_callback(CompletionInfo(status, buffer.data(), buffer.size()));
};
return write_async(TransferRequest(buffer, wrapped_callback));
}
return write_async(MemoryView::create_const(buffer, size), user_callback);
}
+hailo_status InputStreamBase::write_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback)
+{
+ CHECK(0 != size, HAILO_INVALID_ARGUMENT, "Invalid size was passed to write_async");
+
+ auto wrapped_callback = [dmabuf_fd, size, user_callback](hailo_status status) {
+ user_callback(CompletionInfo(status, dmabuf_fd, size));
+ };
+ return write_async(TransferRequest(hailo_dma_buffer_t{dmabuf_fd, size}, wrapped_callback));
+}
+
hailo_status InputStreamBase::write_async(TransferRequest &&)
{
LOGGER__ERROR("write_async not implemented for sync API");
buffer.size(), get_frame_size());
auto wrapped_callback = [buffer, user_callback](hailo_status status) {
- user_callback(CompletionInfo{status, const_cast<uint8_t*>(buffer.data()), buffer.size()});
+ user_callback(CompletionInfo(status, const_cast<uint8_t*>(buffer.data()), buffer.size()));
};
return read_async(TransferRequest(buffer, wrapped_callback));
}
return read_async(MemoryView(buffer, size), user_callback);
}
+hailo_status OutputStreamBase::read_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback)
+{
+ CHECK(0 != size, HAILO_INVALID_ARGUMENT, "Invalid size was passed to read_async");
+
+ auto wrapped_callback = [dmabuf_fd, size, user_callback](hailo_status status) {
+ user_callback(CompletionInfo(status, dmabuf_fd, size));
+ };
+ return read_async(TransferRequest(hailo_dma_buffer_t{dmabuf_fd, size}, wrapped_callback));
+}
+
hailo_status OutputStreamBase::read_unaligned_address_async(const MemoryView &, const TransferDoneCallback &)
{
LOGGER__ERROR("read_unaligned_address_async not implemented OutputStreamBase");
virtual hailo_status write_async(const MemoryView &buffer, const TransferDoneCallback &user_callback) override final;
virtual hailo_status write_async(const void *buffer, size_t size, const TransferDoneCallback &user_callback) override final;
+ virtual hailo_status write_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) override final;
virtual hailo_status write_async(TransferRequest &&transfer_request);
virtual hailo_status read_async(MemoryView buffer, const TransferDoneCallback &user_callback) override final;
virtual hailo_status read_async(void *buffer, size_t size, const TransferDoneCallback &user_callback) override final;
+ virtual hailo_status read_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) override final;
virtual hailo_status read_async(TransferRequest &&transfer_request);
virtual hailo_status read_unaligned_address_async(const MemoryView &buffer, const TransferDoneCallback &user_callback);
TransferBuffer::TransferBuffer() :
m_base_buffer(MemoryView{}),
m_size(0),
- m_offset(0)
+ m_offset(0),
+ m_type(TransferBufferType::MEMORYVIEW)
+{}
+
+TransferBuffer::TransferBuffer(hailo_dma_buffer_t dmabuf) :
+ m_dmabuf(dmabuf),
+ m_size(dmabuf.size),
+ m_offset(0),
+ m_type(TransferBufferType::DMABUF)
{}
TransferBuffer::TransferBuffer(MemoryView base_buffer, size_t size, size_t offset) :
m_base_buffer(base_buffer),
m_size(size),
- m_offset(offset)
+ m_offset(offset),
+ m_type(TransferBufferType::MEMORYVIEW)
{
assert(m_size <= base_buffer.size());
assert(m_offset < base_buffer.size());
: TransferBuffer(base_buffer, base_buffer.size(), 0)
{}
+Expected<MemoryView> TransferBuffer::base_buffer()
+{
+ CHECK(TransferBufferType::DMABUF != m_type, HAILO_INTERNAL_FAILURE,
+ "base_buffer is not supported for DMABUF type TransferBuffer");
+
+ return Expected<MemoryView>(m_base_buffer);
+}
+
Expected<vdma::MappedBufferPtr> TransferBuffer::map_buffer(HailoRTDriver &driver, HailoRTDriver::DmaDirection direction)
{
CHECK_AS_EXPECTED(!m_mappings, HAILO_INTERNAL_FAILURE, "Buffer is already mapped");
+ if (TransferBufferType::DMABUF == m_type) {
+ auto mapped_buffer = vdma::MappedBuffer::create_shared_from_dmabuf(m_dmabuf.fd, m_dmabuf.size, driver, direction);
+ CHECK_EXPECTED(mapped_buffer);
- vdma::DmaAbleBufferPtr dma_able_buffer;
- const auto storage_key = std::make_pair(m_base_buffer.data(), m_base_buffer.size());
- if (auto storage = BufferStorageResourceManager::get_resource(storage_key)) {
- auto dma_able_buffer_exp = storage->get()->get_dma_able_buffer();
- CHECK_EXPECTED(dma_able_buffer_exp);
- dma_able_buffer = dma_able_buffer_exp.release();
+ m_mappings = mapped_buffer.value();
+ return mapped_buffer;
} else {
- auto dma_able_buffer_exp = vdma::DmaAbleBuffer::create_from_user_address(m_base_buffer.data(), m_base_buffer.size());
- CHECK_EXPECTED(dma_able_buffer_exp);
- dma_able_buffer = dma_able_buffer_exp.release();
- }
- auto mapped_buffer = vdma::MappedBuffer::create_shared(std::move(dma_able_buffer), driver, direction);
- CHECK_EXPECTED(mapped_buffer);
-
- m_mappings = mapped_buffer.value();
- return mapped_buffer;
+ vdma::DmaAbleBufferPtr dma_able_buffer;
+ const auto storage_key = std::make_pair(m_base_buffer.data(), m_base_buffer.size());
+ if (auto storage = BufferStorageResourceManager::get_resource(storage_key)) {
+ auto dma_able_buffer_exp = storage->get()->get_dma_able_buffer();
+ CHECK_EXPECTED(dma_able_buffer_exp);
+ dma_able_buffer = dma_able_buffer_exp.release();
+ } else {
+ auto dma_able_buffer_exp = vdma::DmaAbleBuffer::create_from_user_address(m_base_buffer.data(), m_base_buffer.size());
+ CHECK_EXPECTED(dma_able_buffer_exp);
+ dma_able_buffer = dma_able_buffer_exp.release();
+ }
+
+ auto mapped_buffer = vdma::MappedBuffer::create_shared(std::move(dma_able_buffer), driver, direction);
+ CHECK_EXPECTED(mapped_buffer);
+
+ m_mappings = mapped_buffer.value();
+ return mapped_buffer;
+ }
}
hailo_status TransferBuffer::copy_to(MemoryView buffer)
{
CHECK(buffer.size() == m_size, HAILO_INTERNAL_FAILURE, "buffer size {} must be {}", buffer.size(), m_size);
+ CHECK(TransferBufferType::MEMORYVIEW == m_type, HAILO_INTERNAL_FAILURE,
+ "copy_to function is only supported in MEMORYVIEW type TransferBuffer");
auto continuous_parts = get_continuous_parts();
memcpy(buffer.data(), continuous_parts.first.data(), continuous_parts.first.size());
hailo_status TransferBuffer::copy_from(const MemoryView buffer)
{
CHECK(buffer.size() == m_size, HAILO_INTERNAL_FAILURE, "buffer size {} must be {}", buffer.size(), m_size);
+ CHECK(TransferBufferType::MEMORYVIEW == m_type, HAILO_INTERNAL_FAILURE,
+ "copy_from function is only supported in MEMORYVIEW type TransferBuffer");
auto continuous_parts = get_continuous_parts();
memcpy(continuous_parts.first.data(), buffer.data(), continuous_parts.first.size());
#include "vdma/driver/hailort_driver.hpp"
#include "vdma/memory/mapped_buffer.hpp"
+#include "common/os_utils.hpp"
namespace hailort
{
+enum class TransferBufferType {
+ MEMORYVIEW = 0,
+ DMABUF
+};
+
// Contains buffer that can be transferred. The buffer can be circular -
// It relies at [m_offset, m_base_buffer.size()) and [0, m_base_buffer.size() - m_size).
class TransferBuffer final {
public:
TransferBuffer();
-
+ TransferBuffer(hailo_dma_buffer_t dmabuf);
TransferBuffer(MemoryView base_buffer);
TransferBuffer(MemoryView base_buffer, size_t size, size_t offset);
- MemoryView base_buffer() { return m_base_buffer; }
+ Expected<MemoryView> base_buffer();
size_t offset() const { return m_offset; }
size_t size() const { return m_size; }
hailo_status copy_to(MemoryView buffer);
hailo_status copy_from(const MemoryView buffer);
+ TransferBufferType type () const {return m_type;}
+
private:
bool is_wrap_around() const;
// 2. If the buffer is not circular, the first part will contain the buffer, the second will point to nullptr.
std::pair<MemoryView, MemoryView> get_continuous_parts();
- MemoryView m_base_buffer;
+ union {
+ MemoryView m_base_buffer;
+ hailo_dma_buffer_t m_dmabuf;
+ };
+
size_t m_size;
size_t m_offset;
+ TransferBufferType m_type;
// Once map_buffer is called, a MappedBuffer object is stored here to make sure the buffer is mapped.
vdma::MappedBufferPtr m_mappings;
}
return total_transfer_size;
}
+
+ Expected<bool> is_request_aligned() {
+ CHECK(!transfer_buffers.empty(), HAILO_INVALID_ARGUMENT, "TransferRequest is empty");
+ CHECK(TransferBufferType::MEMORYVIEW == transfer_buffers[0].type(), HAILO_INVALID_ARGUMENT,
+ "get_aligned_request is only supported in MEMORYVIEW type TransferBuffer");
+
+ const auto dma_able_alignment = OsUtils::get_dma_able_alignment();
+ TRY(auto base_buffer, transfer_buffers[0].base_buffer());
+ return (0 == reinterpret_cast<size_t>(base_buffer.data()) % dma_able_alignment);
+ }
};
struct InferRequest {
bool TransformContextUtils::should_reorder(const hailo_3d_image_shape_t &src_image_shape, const hailo_format_t &src_format,
const hailo_3d_image_shape_t &dst_image_shape, const hailo_format_t &dst_format)
{
- /* If shapes and format are different - need to use transform_context */
+ /* If the orders are NHCW <-> NHWC, but C == 1, no reordering is needed */
+ if ((src_image_shape.features == dst_image_shape.features) &&
+ (src_image_shape.features == 1) &&
+ (src_image_shape.height == dst_image_shape.height) &&
+ (src_image_shape.width == dst_image_shape.width) &&
+ (((src_format.order == HAILO_FORMAT_ORDER_NHCW) && (dst_format.order == HAILO_FORMAT_ORDER_NHWC)) ||
+ ((src_format.order == HAILO_FORMAT_ORDER_NHWC) && (dst_format.order == HAILO_FORMAT_ORDER_NHCW)))) {
+ return false;
+ }
+
+
+ /* If not all shapes and formats are the same - reordering is needed */
if (!((src_image_shape.features == dst_image_shape.features) &&
- (src_image_shape.height == dst_image_shape.height) &&
+ (src_image_shape.height == dst_image_shape.height) &&
(src_image_shape.width == dst_image_shape.width) &&
(src_format.order == dst_format.order))) {
return true;
const auto dst_frame_size = HailoRTCommon::get_periph_frame_size(dst_image_shape, dst_format);
Buffer quant_buffer;
- auto should_quantize = TransformContextUtils::should_quantize(HAILO_H2D_STREAM, src_format, dst_format);
+ auto should_quantize = TransformContextUtils::should_quantize(HAILO_H2D_STREAM, internal_src_format, dst_format);
CHECK_EXPECTED(should_quantize);
if (should_quantize.value()) {
auto expected_quant_buffer = Buffer::create(src_frame_size, 0);
}
Buffer transpose_buffer;
- bool should_transpose = TransformContextUtils::should_transpose(src_format.flags, dst_format.flags);
+ bool should_transpose = TransformContextUtils::should_transpose(internal_src_format.flags, dst_format.flags);
if (should_transpose) {
auto expected_transpose_buffer = Buffer::create(get_transpose_buffer_size(src_image_shape,
dst_format.type));
transpose_buffer = expected_transpose_buffer.release();
}
- auto should_reorder = TransformContextUtils::should_reorder(src_image_shape, src_format, dst_image_shape, dst_format);
+ auto should_reorder = TransformContextUtils::should_reorder(src_image_shape, internal_src_format, dst_image_shape, dst_format);
auto should_pad_periph = TransformContextUtils::should_pad_periph(dst_image_shape, dst_format);
std::unique_ptr<InputTransformContext> transform_context(new (std::nothrow) InputTransformContext(src_frame_size, src_image_shape,
const auto src_frame_size = HailoRTCommon::get_periph_frame_size(src_image_shape, src_format);
const auto dst_frame_size = HailoRTCommon::get_frame_size(dst_image_shape, internal_dst_format);
- auto should_quantize = TransformContextUtils::should_quantize(HAILO_D2H_STREAM, src_format, dst_format);
+ auto should_quantize = TransformContextUtils::should_quantize(HAILO_D2H_STREAM, src_format, internal_dst_format);
CHECK_EXPECTED(should_quantize);
Buffer transpose_buffer;
- auto should_transpose = TransformContextUtils::should_transpose(src_format.flags, dst_format.flags);
+ auto should_transpose = TransformContextUtils::should_transpose(src_format.flags, internal_dst_format.flags);
if (should_transpose) {
auto expected_transpose_buffer = Buffer::create(get_transpose_buffer_size(dst_image_shape, src_format.type));
CHECK_EXPECTED(expected_transpose_buffer);
transpose_buffer = expected_transpose_buffer.release();
}
- auto should_reorder = TransformContextUtils::should_reorder(src_image_shape, src_format, dst_image_shape, dst_format);
- auto should_pad_periph = TransformContextUtils::should_pad_periph(dst_image_shape, dst_format);
+ auto should_reorder = TransformContextUtils::should_reorder(src_image_shape, src_format, dst_image_shape, internal_dst_format);
+ auto should_pad_periph = TransformContextUtils::should_pad_periph(dst_image_shape, internal_dst_format);
std::unique_ptr<OutputTransformContext> frame_transform_context = std::make_unique<FrameOutputTransformContext>(src_frame_size,
src_image_shape, src_format, dst_frame_size, dst_image_shape, internal_dst_format, dst_quant_infos, std::move(transpose_buffer),
auto dst_frame_size = HailoRTCommon::get_nms_host_frame_size(nms_info, internal_dst_format);
Buffer quant_buffer;
- auto should_quantize = TransformContextUtils::should_quantize(HAILO_D2H_STREAM, src_format, dst_format);
+ auto should_quantize = TransformContextUtils::should_quantize(HAILO_D2H_STREAM, src_format, internal_dst_format);
CHECK_EXPECTED(should_quantize);
if (*should_quantize) {
dst_frame_size = HailoRTCommon::get_nms_host_frame_size(nms_info, internal_dst_format);
quant_buffer = expected_nms_quant_buffer.release();
}
- auto should_transpose = TransformContextUtils::should_transpose(src_format.flags, dst_format.flags);
+ auto should_transpose = TransformContextUtils::should_transpose(src_format.flags, internal_dst_format.flags);
std::unique_ptr<OutputTransformContext> nms_transform_context = std::make_unique<NMSOutputTransformContext>(src_frame_size,
src_format, dst_frame_size, internal_dst_format, dst_quant_infos, nms_info, std::move(quant_buffer),
Expected<Buffer> Buffer::create(size_t size, const BufferStorageParams ¶ms)
{
+ if (0 == size) {
+ return Buffer();
+ }
auto storage = BufferStorage::create(size, params);
CHECK_EXPECTED(storage);
auto result = DmaStorage::create(size);\r
CHECK_EXPECTED(result);\r
return std::static_pointer_cast<BufferStorage>(result.release());\r
+ } else if (0 != (params.flags & HAILO_BUFFER_FLAGS_CONTINUOUS)) {\r
+ auto result = ContinuousStorage::create(size);\r
+ CHECK_EXPECTED(result);\r
+ return std::static_pointer_cast<BufferStorage>(result.release());\r
}\r
\r
// TODO: HRT-10903\r
return make_unexpected(HAILO_NOT_IMPLEMENTED);\r
}\r
\r
+Expected<uint64_t> BufferStorage::dma_address()\r
+{\r
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);\r
+}\r
+\r
Expected<HeapStoragePtr> HeapStorage::create(size_t size)\r
{\r
std::unique_ptr<uint8_t[]> data(new (std::nothrow) uint8_t[size]);\r
\r
Expected<void *> DmaStorage::release() noexcept\r
{\r
- return make_unexpected(HAILO_NOT_IMPLEMENTED);\r
+ return make_unexpected(HAILO_INVALID_OPERATION);\r
}\r
\r
Expected<vdma::DmaAbleBufferPtr> DmaStorage::get_dma_able_buffer()\r
return vdma::DmaAbleBufferPtr{m_dma_able_buffer};\r
}\r
\r
+Expected<ContinuousStoragePtr> ContinuousStorage::create(size_t size)\r
+{\r
+ TRY(auto driver, HailoRTDriver::create_integrated_nnc());\r
+ TRY(auto continuous_buffer, vdma::ContinuousBuffer::create(size, *driver.get()));\r
+\r
+ auto result = make_shared_nothrow<ContinuousStorage>(std::move(driver), std::move(continuous_buffer));\r
+ CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY);\r
+\r
+ return result;\r
+}\r
+\r
+ContinuousStorage::ContinuousStorage(std::unique_ptr<HailoRTDriver> driver, vdma::ContinuousBuffer &&continuous_buffer) :\r
+ m_driver(std::move(driver)),\r
+ m_continuous_buffer(std::move(continuous_buffer))\r
+{}\r
+\r
+ContinuousStorage::ContinuousStorage(ContinuousStorage&& other) noexcept :\r
+ BufferStorage(std::move(other)),\r
+ m_driver(std::move(other.m_driver)),\r
+ m_continuous_buffer(std::move(other.m_continuous_buffer))\r
+{}\r
+\r
+size_t ContinuousStorage::size() const\r
+{\r
+ return m_continuous_buffer.size();\r
+}\r
+\r
+void *ContinuousStorage::user_address()\r
+{\r
+ return m_continuous_buffer.user_address();\r
+}\r
+\r
+Expected<uint64_t> ContinuousStorage::dma_address()\r
+{\r
+ return m_continuous_buffer.dma_address();\r
+}\r
+\r
+Expected<void *> ContinuousStorage::release() noexcept\r
+{\r
+ return make_unexpected(HAILO_INVALID_OPERATION);\r
+}\r
+\r
} /* namespace hailort */\r
#include "hailo/buffer.hpp"\r
\r
#include "utils/exported_resource_manager.hpp"\r
+#include "vdma/memory/continuous_buffer.hpp"\r
\r
#include <memory>\r
#include <cstdint>\r
class BufferStorage;\r
class HeapStorage;\r
class DmaStorage;\r
+class ContinuousStorage;\r
class HailoRTDriver;\r
class Buffer;\r
\r
\r
// Internal functions\r
virtual Expected<vdma::DmaAbleBufferPtr> get_dma_able_buffer();\r
+ virtual Expected<uint64_t> dma_address();\r
\r
BufferStorage() = default;\r
};\r
};\r
\r
\r
+using ContinuousStoragePtr = std::shared_ptr<ContinuousStorage>;\r
+\r
+/**\r
+ * Storage class for buffer that is continuous\r
+ */\r
+class ContinuousStorage : public BufferStorage\r
+{\r
+public:\r
+ static Expected<ContinuousStoragePtr> create(size_t size);\r
+ ContinuousStorage(std::unique_ptr<HailoRTDriver> driver, vdma::ContinuousBuffer &&continuous_buffer);\r
+ ContinuousStorage(ContinuousStorage&& other) noexcept;\r
+ ContinuousStorage(const ContinuousStorage &) = delete;\r
+ ContinuousStorage &operator=(ContinuousStorage &&) = delete;\r
+ ContinuousStorage &operator=(const ContinuousStorage &) = delete;\r
+ virtual ~ContinuousStorage() = default;\r
+\r
+ virtual size_t size() const override;\r
+ virtual void *user_address() override;\r
+ virtual Expected<uint64_t> dma_address() override;\r
+ virtual Expected<void *> release() noexcept override;\r
+\r
+private:\r
+ std::unique_ptr<HailoRTDriver> m_driver;\r
+ vdma::ContinuousBuffer m_continuous_buffer;\r
+};\r
+\r
} /* namespace hailort */\r
\r
#endif /* _HAILO_BUFFER_STORAGE_HPP_ */\r
#include "hailo/hailort.h"
#include "hailo/expected.hpp"
#include "utils/buffer_storage.hpp"
+#include "net_flow/pipeline/pipeline.hpp"
/** hailort namespace */
namespace hailort
class HAILORTAPI DmaBufferUtils
{
public:
-
- static Expected<MemoryView> mmap_dma_buffer_write(hailo_dma_buffer_t dma_buffer);
-
- static hailo_status munmap_dma_buffer_write(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview);
-
- static Expected<MemoryView> mmap_dma_buffer_read(hailo_dma_buffer_t dma_buffer);
-
- static hailo_status munmap_dma_buffer_read(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview);
-
+ static Expected<MemoryView> mmap_dma_buffer(hailo_dma_buffer_t dma_buffer, BufferProtection dma_buffer_protection);
+ static hailo_status munmap_dma_buffer(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview, BufferProtection dma_buffer_protection);
};
} /* namespace hailort */
}
}
-Expected<hailo_pix_buffer_t> HailoRTCommon::as_hailo_pix_buffer(MemoryView &memory_view, hailo_format_order_t order)
+Expected<hailo_pix_buffer_t> HailoRTCommon::as_hailo_pix_buffer(MemoryView memory_view, hailo_format_order_t order)
{
switch(order){
case HAILO_FORMAT_ORDER_NV12:
}
}
+bool HailoRTCommon::is_power_measurement_supported(const hailo_device_architecture_t &hw_arch)
+{
+ switch(hw_arch) {
+ case HAILO_ARCH_HAILO8:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool HailoRTCommon::is_current_measurement_supported(const hailo_device_architecture_t &hw_arch)
+{
+ return is_power_measurement_supported(hw_arch);
+}
+
+bool HailoRTCommon::is_temp_measurement_supported(const hailo_device_architecture_t &hw_arch)
+{
+ switch(hw_arch) {
+ case HAILO_ARCH_HAILO8:
+ case HAILO_ARCH_HAILO8L:
+ return true;
+ default:
+ return false;
+ }
+}
+
} /* namespace hailort */
#define HAILORT_LOGGER_FLUSH_EVERY_PRINT_ENV_VAR ("HAILORT_LOGGER_FLUSH_EVERY_PRINT")
#define PERIODIC_FLUSH_INTERVAL_IN_SECONDS (5)
-#ifdef _WIN32
-#define PATH_SEPARATOR "\\"
-#else
-#define PATH_SEPARATOR "/"
-#endif
std::string HailoRTLogger::parse_log_path(const char *log_path)
{
return make_shared_nothrow<spdlog::sinks::null_sink_st>();
}
if (!is_dir.value()) {
- std::cerr << "HailoRT warning: Cannot create log file " << filename << "! Path " << dir_path << " is not a directory." << std::endl;
- return make_shared_nothrow<spdlog::sinks::null_sink_st>();
+ auto status = Filesystem::create_directory(dir_path);
+ if (status != HAILO_SUCCESS) {
+ std::cerr << "HailoRT warning: Cannot create log file " << filename << "! Path " << dir_path << " is not valid." << std::endl;
+ return make_shared_nothrow<spdlog::sinks::null_sink_st>();
+ }
}
if (!Filesystem::is_path_accesible(dir_path)) {
spdlog::flush_every(std::chrono::seconds(PERIODIC_FLUSH_INTERVAL_IN_SECONDS));
}
+Expected<spdlog::level::level_enum> HailoRTLogger::get_console_logger_level_from_string(const std::string &user_console_logger_level)
+{
+ static const std::unordered_map<std::string, spdlog::level::level_enum> log_level_map = {
+ {"info", spdlog::level::info},
+ {"warning", spdlog::level::warn},
+ {"error", spdlog::level::err},
+ {"critical", spdlog::level::critical}
+ };
+ if(log_level_map.find(user_console_logger_level) != log_level_map.end()) {
+ return Expected<spdlog::level::level_enum>(log_level_map.at(user_console_logger_level));
+ }
+ return make_unexpected(HAILO_INVALID_ARGUMENT);
+}
} /* namespace hailort */
namespace hailort
{
+#define HAILORT_CONSOLE_LOGGER_LEVEL ("HAILORT_CONSOLE_LOGGER_LEVEL")
+
+#ifdef _WIN32
+#define PATH_SEPARATOR "\\"
+#else
+#define PATH_SEPARATOR "/"
+#endif
class HailoRTLogger {
public:
#endif
{
static std::unique_ptr<HailoRTLogger> instance = nullptr;
+ auto user_console_logger_level = std::getenv(HAILORT_CONSOLE_LOGGER_LEVEL);
+ if ((nullptr != user_console_logger_level) && (std::strlen(user_console_logger_level) > 0)){
+ auto expected_console_level = get_console_logger_level_from_string(user_console_logger_level);
+ if (expected_console_level) {
+ console_level = expected_console_level.release();
+ } else {
+ LOGGER__WARNING("Failed to parse console logger level from environment variable: {}, status: {}",
+ user_console_logger_level, expected_console_level.status());
+ }
+ }
if (nullptr == instance) {
instance = make_unique_nothrow<HailoRTLogger>(console_level, file_level, flush_level);
}
private:
static std::string parse_log_path(const char *log_path);
void set_levels(spdlog::level::level_enum console_level, spdlog::level::level_enum file_level, spdlog::level::level_enum flush_level);
+ static Expected<spdlog::level::level_enum> get_console_logger_level_from_string(const std::string &user_console_logger_level);
std::shared_ptr<spdlog::sinks::sink> m_console_sink;
struct ActivateCoreOpTrace : Trace
{
- ActivateCoreOpTrace(const device_id_t &device_id, vdevice_core_op_handle_t handle, double duration)
- : Trace("activate_core_op"), device_id(device_id), core_op_handle(handle), duration(duration)
+ ActivateCoreOpTrace(const device_id_t &device_id, vdevice_core_op_handle_t handle, double duration,
+ int dynamic_batch_size)
+ : Trace("activate_core_op"), device_id(device_id), core_op_handle(handle), duration(duration), dynamic_batch_size(dynamic_batch_size)
{}
device_id_t device_id;
vdevice_core_op_handle_t core_op_handle;
double duration;
+ int dynamic_batch_size;
};
// Currently, activate and switch are the same trace to make scheduler and fast-switch flow similar (although in the
#include <fstream>
#include <iomanip>
-#define PROFILER_DEFAULT_FILE_NAME ("hailo.tracer")
+
#define SCHEDULER_PROFILER_NAME ("SchedulerProfiler")
-#define PROFILER_FILE_ENV_VAR ("HAILO_TRACE_FILE")
+#define PROFILER_FILE_ENV_VAR ("HAILO_TRACE_PATH")
#define SCHEDULER_PROFILER_LOGGER_FILENAME ("scheduler_profiler.json")
#define SCHEDULER_PROFILER_LOGGER_PATTERN ("%v")
#define SCHEDULER_PROFILER_LOGGER_PATH ("SCHEDULER_PROFILER_LOGGER_PATH")
+static const std::string PROFILER_DEFAULT_FILE_NAME_PREFIX("hailort");
+static const std::string PROFILER_DEFAULT_FILE_NAME_SUFFIX(".hrtt");
+
namespace hailort
{
+std::string get_current_datetime() {
+ auto now = std::chrono::system_clock::now();
+ auto tp = std::chrono::system_clock::to_time_t(now);
+ std::stringstream ss;
+ ss << std::put_time(std::localtime(&tp), "%Y-%m-%d_%H-%M-%S");
+ return ss.str();
+}
+
SchedulerProfilerHandler::SchedulerProfilerHandler(int64_t &start_time)
#ifndef __ANDROID__
: m_file_sink(HailoRTLogger::create_file_sink(HailoRTLogger::get_log_path(SCHEDULER_PROFILER_LOGGER_PATH), SCHEDULER_PROFILER_LOGGER_FILENAME, false)),
void SchedulerProfilerHandler::serialize_and_dump_proto()
{
auto file_env_var = std::getenv(PROFILER_FILE_ENV_VAR);
- std::string file_name = PROFILER_DEFAULT_FILE_NAME;
+ std::string file_name = PROFILER_DEFAULT_FILE_NAME_PREFIX + "_" + get_current_datetime() + PROFILER_DEFAULT_FILE_NAME_SUFFIX;
if (nullptr != file_env_var) {
- file_name = std::string(file_env_var);
+ file_name = std::string(file_env_var) + PATH_SEPARATOR + file_name;
}
std::ofstream output_file(std::string(file_name), std::ios::out |std::ios::binary);
added_trace->mutable_activate_core_op()->set_new_core_op_handle(trace.core_op_handle);
added_trace->mutable_activate_core_op()->set_time_stamp(trace.timestamp);
added_trace->mutable_activate_core_op()->set_duration(trace.duration);
+ added_trace->mutable_activate_core_op()->set_dynamic_batch_size(trace.dynamic_batch_size);
}
void SchedulerProfilerHandler::handle_trace(const DeactivateCoreOpTrace &trace)
return m_inner.size_approx();
}
+ size_t max_capacity()
+ {
+ return m_inner.max_capacity();
+ }
+
bool is_queue_full()
{
return (m_inner.size_approx() == m_inner.max_capacity());
set(SRC_FILES
${CMAKE_CURRENT_SOURCE_DIR}/vdevice.cpp
${CMAKE_CURRENT_SOURCE_DIR}/vdevice_core_op.cpp
+ # ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_hrpc_client.cpp
${CMAKE_CURRENT_SOURCE_DIR}/vdevice_native_stream.cpp
${CMAKE_CURRENT_SOURCE_DIR}/callback_reorder_queue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/scheduler/infer_request_accumulator.cpp
)
+set(SRC_FILES ${SRC_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_hrpc_client.cpp)
+
set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE)
return m_core_op;
}
-std::shared_ptr<VdmaConfigCoreOp> ScheduledCoreOp::get_vdma_core_op(const device_id_t &device_id)
+Expected<std::shared_ptr<VdmaConfigCoreOp>> ScheduledCoreOp::get_vdma_core_op(const device_id_t &device_id)
{
- auto vdma_core_op = m_core_op->get_core_op_by_device_id(device_id);
- assert(vdma_core_op);
- return vdma_core_op.release();
+ return m_core_op->get_core_op_by_device_id(device_id);
}
std::chrono::time_point<std::chrono::steady_clock> ScheduledCoreOp::get_last_run_timestamp()
std::shared_ptr<CoreOp> get_core_op();
- std::shared_ptr<VdmaConfigCoreOp> get_vdma_core_op(const device_id_t &device_id);
+ Expected<std::shared_ptr<VdmaConfigCoreOp>> get_vdma_core_op(const device_id_t &device_id);
uint32_t get_max_ongoing_frames_per_device() const;
curr_device_info->current_batch_size = hw_batch_size;
if ((core_op_handle != curr_device_info->current_core_op_handle) || (!has_same_hw_batch_size_as_previous)) {
- auto next_core_op = get_vdma_core_op(core_op_handle, device_id);
+ TRY(auto next_core_op, get_vdma_core_op(core_op_handle, device_id));
std::shared_ptr<VdmaConfigCoreOp> current_core_op = nullptr;
if (curr_device_info->current_core_op_handle != INVALID_CORE_OP_HANDLE) {
- current_core_op = get_vdma_core_op(curr_device_info->current_core_op_handle, device_id);
+ TRY(current_core_op, get_vdma_core_op(curr_device_info->current_core_op_handle, device_id));
}
auto status = VdmaConfigManager::set_core_op(device_id, current_core_op, next_core_op, hw_batch_size);
return HAILO_SUCCESS;
}
- auto vdma_core_op = get_vdma_core_op(core_op_handle, device_id);
+ TRY (auto vdma_core_op, get_vdma_core_op(core_op_handle, device_id));
auto status = VdmaConfigManager::deactivate_core_op(vdma_core_op);
CHECK_SUCCESS(status, "Scheduler failed deactivate core op on {}", device_id);
auto current_device_info = m_devices[device_id];
assert(core_op_handle == current_device_info->current_core_op_handle);
auto scheduled_core_op = m_scheduled_core_ops.at(core_op_handle);
- auto vdma_core_op = get_vdma_core_op(core_op_handle, device_id);
+ TRY (auto vdma_core_op, get_vdma_core_op(core_op_handle, device_id));
auto infer_request = dequeue_infer_request(core_op_handle);
CHECK_EXPECTED_AS_STATUS(infer_request);
return static_cast<uint16_t>(std::min(requested_frames, max_ongoing_frames - ongoing_frames));
}
-std::shared_ptr<VdmaConfigCoreOp> CoreOpsScheduler::get_vdma_core_op(scheduler_core_op_handle_t core_op_handle,
+Expected<std::shared_ptr<VdmaConfigCoreOp>> CoreOpsScheduler::get_vdma_core_op(scheduler_core_op_handle_t core_op_handle,
const device_id_t &device_id)
{
return m_scheduled_core_ops.at(core_op_handle)->get_vdma_core_op(device_id);
Expected<InferRequest> dequeue_infer_request(scheduler_core_op_handle_t core_op_handle);
uint16_t get_frames_ready_to_transfer(scheduler_core_op_handle_t core_op_handle, const device_id_t &device_id) const;
- std::shared_ptr<VdmaConfigCoreOp> get_vdma_core_op(scheduler_core_op_handle_t core_op_handle,
+ Expected<std::shared_ptr<VdmaConfigCoreOp>> get_vdma_core_op(scheduler_core_op_handle_t core_op_handle,
const device_id_t &device_id);
void shutdown_core_op(scheduler_core_op_handle_t core_op_handle);
#include "vdevice/vdevice_internal.hpp"
#include "vdevice/vdevice_core_op.hpp"
+#include "vdevice/vdevice_hrpc_client.hpp"
#include "vdma/pcie/pcie_device.hpp"
#include "vdma/integrated/integrated_device.hpp"
#include "rpc/rpc_definitions.hpp"
#endif // HAILO_SUPPORT_MULTI_PROCESS
+#define HAILO_FORCE_HRPC_CLIENT_ENV_VAR "HAILO_FORCE_HRPC"
+#define HAILO_FORCE_HRPC_CLIENT_ON "1"
+
namespace hailort
{
return vdevice.value()->dma_unmap(address, size, direction);
}
+hailo_status VDeviceHandle::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ auto &manager = SharedResourceManager<std::string, VDeviceBase>::get_instance();
+ auto vdevice = manager.resource_lookup(m_handle);
+ CHECK_EXPECTED_AS_STATUS(vdevice);
+
+ return vdevice.value()->dma_map_dmabuf(dmabuf_fd, size, direction);
+}
+
+hailo_status VDeviceHandle::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ auto &manager = SharedResourceManager<std::string, VDeviceBase>::get_instance();
+ auto vdevice = manager.resource_lookup(m_handle);
+ CHECK_EXPECTED_AS_STATUS(vdevice);
+
+ return vdevice.value()->dma_unmap_dmabuf(dmabuf_fd, size, direction);
+}
+
bool VDevice::service_over_ip_mode()
{
#ifdef HAILO_SUPPORT_MULTI_PROCESS
return false; // no service -> no service over ip
}
+bool VDevice::force_hrpc_client()
+{
+ // The env var HAILO_FORCE_HRPC_CLIENT_ENV_VAR is supported for debug purposes
+ char *pcie_service_var = std::getenv(HAILO_FORCE_HRPC_CLIENT_ENV_VAR); // TODO: Remove duplication
+ return (nullptr != pcie_service_var) && (HAILO_FORCE_HRPC_CLIENT_ON == std::string(pcie_service_var));
+}
+
#ifdef HAILO_SUPPORT_MULTI_PROCESS
VDeviceClient::VDeviceClient(std::unique_ptr<HailoRtRpcClient> client, VDeviceIdentifier &&identifier,
(void) size;
(void) direction;
// It is ok to do nothing on service, because the buffer is copied anyway to the service.
- LOGGER__TRACE("VDevice `dma_map()` is doing nothing on service");
+ LOGGER__TRACE("VDevice `dma_map()` does nothing in service");
return HAILO_SUCCESS;
}
(void) size;
(void) direction;
// It is ok to do nothing on service, because the buffer is copied anyway to the service.
- LOGGER__TRACE("VDevice `dma_map()` is doing nothing on service");
+ LOGGER__TRACE("VDevice `dma_map()` does nothing in service");
+ return HAILO_SUCCESS;
+}
+
+hailo_status VDeviceClient::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ (void) dmabuf_fd;
+ (void) size;
+ (void) direction;
+ // It is ok to do nothing on service, because the buffer is copied anyway to the service.
+ LOGGER__TRACE("VDevice `dma_map_dmabuf()` does nothing in service");
+ return HAILO_SUCCESS;
+}
+
+hailo_status VDeviceClient::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ (void) dmabuf_fd;
+ (void) size;
+ (void) direction;
+ // It is ok to do nothing on service, because the buffer is copied anyway to the service.
+ LOGGER__TRACE("VDevice `dma_unmap_dmabuf()` does nothing in service");
return HAILO_SUCCESS;
}
auto status = VDeviceBase::validate_params(params);
CHECK_SUCCESS_AS_EXPECTED(status);
- std::unique_ptr<VDevice> vdevice;
+ std::unique_ptr<VDevice> vdevice = nullptr;
if (params.multi_process_service) {
#ifdef HAILO_SUPPORT_MULTI_PROCESS
return make_unexpected(HAILO_INVALID_OPERATION);
#endif // HAILO_SUPPORT_MULTI_PROCESS
} else {
- auto expected_vdevice = VDeviceHandle::create(params);
- CHECK_EXPECTED(expected_vdevice);
- vdevice = expected_vdevice.release();
+ auto acc_type = HailoRTDriver::AcceleratorType::ACC_TYPE_MAX_VALUE;
+ if (nullptr != params.device_ids) {
+ TRY(auto device_ids_contains_eth, VDeviceBase::device_ids_contains_eth(params));
+ if (!device_ids_contains_eth) {
+ TRY(acc_type, VDeviceBase::get_accelerator_type(params.device_ids, params.device_count));
+ }
+ } else {
+ TRY(acc_type, VDeviceBase::get_accelerator_type(params.device_ids, params.device_count));
+ }
+ if ((acc_type == HailoRTDriver::AcceleratorType::SOC_ACCELERATOR) || force_hrpc_client()) {
+ // Creating VDeviceClient
+ TRY(vdevice, VDeviceHrpcClient::create(params));
+ } else {
+ // Creating VDeviceHandle
+ TRY(vdevice, VDeviceHandle::create(params));
+ }
}
// Upcasting to VDevice unique_ptr
auto vdevice_ptr = std::unique_ptr<VDevice>(vdevice.release());
return create(params);
}
+Expected<HailoRTDriver::AcceleratorType> VDeviceBase::get_accelerator_type(hailo_device_id_t *device_ids, size_t device_count)
+{
+ auto acc_type = HailoRTDriver::AcceleratorType::ACC_TYPE_MAX_VALUE;
+ TRY(auto device_infos, HailoRTDriver::scan_devices());
+ if (nullptr != device_ids) {
+ // device_ids are provided - check that all ids are of the same type + that the id exists in the scan from device_infos
+ for (uint32_t i = 0; i < device_count; i++) {
+ const auto &id = device_ids[i].id;
+ auto device_info = std::find_if(device_infos.begin(), device_infos.end(), [&](const auto &device_info) {
+ return Device::device_ids_equal(device_info.device_id, id);
+ });
+ CHECK(device_info != device_infos.end(), HAILO_INVALID_ARGUMENT,
+ "VDevice creation failed. device_id {} not found", id);
+ CHECK(acc_type == HailoRTDriver::AcceleratorType::ACC_TYPE_MAX_VALUE || acc_type == device_info->accelerator_type, HAILO_INVALID_ARGUMENT,
+ "VDevice creation failed. device_ids of devices with different types are provided (e.g. Hailo8 and Hailo10). Please provide device_ids of the same device types");
+ acc_type = device_info->accelerator_type;
+ }
+ } else {
+ // No device_id is provided - check that all devices are of the same type
+ for (const auto &device_info : device_infos) {
+ CHECK(acc_type == HailoRTDriver::AcceleratorType::ACC_TYPE_MAX_VALUE || acc_type == device_info.accelerator_type, HAILO_INVALID_ARGUMENT,
+ "VDevice creation failed. Devices of different types are found and no device_id is provided. Please provide device_ids");
+ acc_type = device_info.accelerator_type;
+ }
+ }
+ return acc_type;
+}
+
hailo_status VDeviceBase::validate_params(const hailo_vdevice_params_t ¶ms)
{
CHECK(0 != params.device_count, HAILO_INVALID_ARGUMENT,
"VDevice creation failed. invalid device_count ({}).", params.device_count);
- if (params.device_ids != nullptr) {
- for (uint32_t i = 0; i < params.device_count; i++) {
- auto dev_type = Device::get_device_type(params.device_ids[i].id);
- CHECK_EXPECTED_AS_STATUS(dev_type);
- CHECK((Device::Type::ETH != dev_type.value() || (1 == params.device_count)), HAILO_INVALID_ARGUMENT,
- "VDevice over ETH is supported for 1 device. Passed device_count: {}", params.device_count);
- CHECK((Device::Type::ETH != dev_type.value() || (HAILO_SCHEDULING_ALGORITHM_NONE == params.scheduling_algorithm)), HAILO_INVALID_ARGUMENT,
- "VDevice over ETH is not supported when scheduler is enabled.");
- }
- }
+ TRY(auto device_ids_contains_eth, device_ids_contains_eth(params));
+ CHECK(!(device_ids_contains_eth && (1 != params.device_count)), HAILO_INVALID_ARGUMENT,
+ "VDevice over ETH is supported for 1 device. Passed device_count: {}", params.device_count);
+ CHECK(!(device_ids_contains_eth && (HAILO_SCHEDULING_ALGORITHM_NONE != params.scheduling_algorithm)), HAILO_INVALID_ARGUMENT,
+ "VDevice over ETH is not supported when scheduler is enabled.");
+
return HAILO_SUCCESS;
}
Expected<std::shared_ptr<InferModel>> VDevice::create_infer_model(const std::string &hef_path, const std::string &network_name)
{
CHECK_AS_EXPECTED(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Passing network name is not supported yet!");
+ TRY(auto infer_model_base, InferModelBase::create(*this, hef_path));
+ return std::shared_ptr<InferModel>(std::move(infer_model_base));
+}
- auto hef_expected = Hef::create(hef_path);
- CHECK_EXPECTED(hef_expected);
- auto hef = hef_expected.release();
-
- std::unordered_map<std::string, InferModel::InferStream> inputs;
- std::unordered_map<std::string, InferModel::InferStream> outputs;
-
- auto input_vstream_infos = hef.get_input_vstream_infos();
- CHECK_EXPECTED(input_vstream_infos);
-
- for (const auto &vstream_info : input_vstream_infos.value()) {
- auto pimpl = make_shared_nothrow<InferModel::InferStream::Impl>(vstream_info);
- CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY);
-
- InferModel::InferStream stream(pimpl);
- inputs.emplace(vstream_info.name, std::move(stream));
- }
-
- auto output_vstream_infos = hef.get_output_vstream_infos();
- CHECK_EXPECTED(output_vstream_infos);
-
- for (const auto &vstream_info : output_vstream_infos.value()) {
- auto pimpl = make_shared_nothrow<InferModel::InferStream::Impl>(vstream_info);
- CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY);
-
- InferModel::InferStream stream(pimpl);
- outputs.emplace(vstream_info.name, std::move(stream));
- }
-
- auto res = make_shared_nothrow<InferModel>(InferModel(*this, std::move(hef), std::move(inputs), std::move(outputs)));
- CHECK_NOT_NULL_AS_EXPECTED(res, HAILO_OUT_OF_HOST_MEMORY);
-
- return res;
+Expected<std::shared_ptr<InferModel>> VDevice::create_infer_model(const MemoryView hef_buffer, const std::string &network_name)
+{
+ CHECK_AS_EXPECTED(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Passing network name is not supported yet!");
+ TRY(auto infer_model_base, InferModelBase::create(*this, hef_buffer));
+ return std::shared_ptr<InferModel>(std::move(infer_model_base));
}
Expected<hailo_stream_interface_t> VDeviceBase::get_default_streams_interface() const
"VDevice with multiple devices is not supported on HAILO_ARCH_HAILO8L. device {} is HAILO_ARCH_HAILO8L", device_id);
CHECK_AS_EXPECTED(HAILO_ARCH_HAILO15M != device_arch.value(), HAILO_INVALID_OPERATION,
"VDevice with multiple devices is not supported on HAILO_ARCH_HAILO15M. device {} is HAILO_ARCH_HAILO15M", device_id);
+ CHECK_AS_EXPECTED(HAILO_ARCH_HAILO10H != device_arch.value(), HAILO_INVALID_OPERATION,
+ "VDevice with multiple devices is not supported on HAILO_ARCH_HAILO10H. device {} is HAILO_ARCH_HAILO10H", device_id);
}
auto dev_type = Device::get_device_type(device_id);
return (!disabled_by_flag && m_core_ops_scheduler);
}
+Expected<bool> VDeviceBase::device_ids_contains_eth(const hailo_vdevice_params_t ¶ms)
+{
+ if (params.device_ids != nullptr) {
+ for (uint32_t i = 0; i < params.device_count; i++) {
+ TRY(auto dev_type, Device::get_device_type(params.device_ids[i].id));
+ if (Device::Type::ETH == dev_type) {
+ return true;
+ }
+ }
+ }
+ return false; // in case no device_ids were provided, we assume there's no ETH device
+}
+
} /* namespace hailort */
CoreOpsSchedulerWeakPtr core_ops_scheduler, vdevice_core_op_handle_t core_op_handle,
const std::string &hef_hash, size_t max_queue_size,
hailo_status &status) :
- CoreOp(configure_params, core_ops.begin()->second->m_metadata, active_core_op_holder, status),
+ CoreOp(configure_params, core_ops.begin()->second->m_metadata, active_core_op_holder, status, !core_ops_scheduler.expired()),
m_vdevice(vdevice),
m_core_ops(std::move(core_ops)),
m_core_ops_scheduler(core_ops_scheduler),
m_infer_requests_accumulator = infer_request_accumulator;
}
+
+ if (has_caches() && is_scheduled()) {
+ // TODO: caches only work with a batch size of one currently (HRT-13628)
+ if (is_default_batch_size()) {
+ // Default batch size with the sched leads to dynamic batches with the max size supported by the streams
+ LOGGER__ERROR("Caches are not supported with the scheduler when using the default batch size");
+ status = HAILO_INVALID_OPERATION;
+ return;
+ }
+
+ const auto configured_batch_size = get_smallest_configured_batch_size(configure_params);
+ if (configured_batch_size > 1) {
+ LOGGER__ERROR("Caches are not supported with the scheduler when using a batch size greater than 1 (received {})",
+ configured_batch_size);
+ status = HAILO_INVALID_OPERATION;
+ return;
+ }
+ }
+
+ status = HAILO_SUCCESS;
}
Expected<hailo_stream_interface_t> VDeviceCoreOp::get_default_streams_interface()
return m_core_ops.begin()->second->get_intermediate_buffer(key);
}
+Expected<Buffer> VDeviceCoreOp::get_cache_buffer(uint32_t cache_id)
+{
+ CHECK_AS_EXPECTED(1 == m_core_ops.size(), HAILO_INVALID_OPERATION,
+ "get_cache_buffer function is not supported on more than 1 physical device.");
+ return m_core_ops.begin()->second->get_cache_buffer(cache_id);
+}
+
+Expected<std::map<uint32_t, Buffer>> VDeviceCoreOp::get_cache_buffers()
+{
+ CHECK_AS_EXPECTED(1 == m_core_ops.size(), HAILO_INVALID_OPERATION,
+ "get_cache_buffers function is not supported on more than 1 physical device.");
+ return m_core_ops.begin()->second->get_cache_buffers();
+}
+
+Expected<uint32_t> VDeviceCoreOp::get_cache_read_size() const
+{
+ CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION,
+ "get_cache_read_size function is not supported on more than 1 physical device.");
+ return m_core_ops.begin()->second->get_cache_read_size();
+}
+
+Expected<uint32_t> VDeviceCoreOp::get_cache_write_size() const
+{
+ CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION,
+ "get_cache_write_size function is not supported on more than 1 physical device.");
+ return m_core_ops.begin()->second->get_cache_write_size();
+}
+
+bool VDeviceCoreOp::has_caches() const
+{
+ for (const auto &core_op : m_core_ops) {
+ if (core_op.second->has_caches()) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+hailo_status VDeviceCoreOp::init_cache(uint32_t read_offset, int32_t write_offset_delta)
+{
+ CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION,
+ "init_cache function is not supported on more than 1 physical device.");
+ return m_core_ops.begin()->second->init_cache(read_offset, write_offset_delta);
+}
+
+Expected<hailo_cache_info_t> VDeviceCoreOp::get_cache_info() const
+{
+ CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION,
+ "get_cache_info function is not supported on more than 1 physical device.");
+ return m_core_ops.begin()->second->get_cache_info();
+}
+
+hailo_status VDeviceCoreOp::update_cache_offset(int32_t offset_delta_bytes)
+{
+ CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION,
+ "update_cache_offset function is not supported on more than 1 physical device.");
+ return m_core_ops.begin()->second->update_cache_offset(offset_delta_bytes);
+}
+
hailo_status VDeviceCoreOp::add_to_trace()
{
const auto batch_size = get_stream_batch_size(m_config_params.stream_params_by_name.begin()->first);
virtual Expected<HwInferResults> run_hw_infer_estimator() override;
virtual Expected<Buffer> get_intermediate_buffer(const IntermediateBufferKey &) override;
+ virtual Expected<Buffer> get_cache_buffer(uint32_t cache_id) override;
+ virtual Expected<std::map<uint32_t, Buffer>> get_cache_buffers() override;
+ virtual bool has_caches() const override;
+ virtual Expected<uint32_t> get_cache_read_size() const override;
+ virtual Expected<uint32_t> get_cache_write_size() const override;
+ virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override;
+ virtual Expected<hailo_cache_info_t> get_cache_info() const;
+ virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override;
VDeviceCoreOp(VDevice &vdevice,
ActiveCoreOpHolder &active_core_op_holder,
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file vdevice_hrpc_client.cpp
+ * @brief VDevice HRPC client implementation
+ **/
+
+#include "vdevice_hrpc_client.hpp"
+#include "hailo/hailort.h"
+#include "hrpc_protocol/serializer.hpp"
+#include "net_flow/pipeline/infer_model_hrpc_client.hpp"
+
+namespace hailort
+{
+
+Expected<std::unique_ptr<VDevice>> VDeviceHrpcClient::create(const hailo_vdevice_params_t ¶ms)
+{
+ CHECK_AS_EXPECTED(params.device_count == 1, HAILO_OUT_OF_PHYSICAL_DEVICES, "Only single device is supported!");
+
+ auto client = make_shared_nothrow<hrpc::Client>();
+ CHECK_NOT_NULL(client, HAILO_INTERNAL_FAILURE);
+
+ auto status = client->connect();
+ CHECK_SUCCESS_AS_EXPECTED(status, "Failed to connect to server");
+
+ TRY(auto request, CreateVDeviceSerializer::serialize_request(params));
+ TRY(auto result, client->execute_request(HailoRpcActionID::VDEVICE__CREATE, MemoryView(request)));
+ TRY(auto tuple, CreateVDeviceSerializer::deserialize_reply(MemoryView(result)));
+ status = std::get<0>(tuple);
+ CHECK_SUCCESS_AS_EXPECTED(status);
+
+ auto vdevice_handle = std::get<1>(tuple);
+ auto vdevice_client = make_unique_nothrow<VDeviceHrpcClient>(std::move(client), vdevice_handle);
+ CHECK_NOT_NULL(vdevice_client, HAILO_OUT_OF_HOST_MEMORY);
+
+ return std::unique_ptr<VDevice>(std::move(vdevice_client));
+}
+
+VDeviceHrpcClient::~VDeviceHrpcClient()
+{
+ if (INVALID_HANDLE_ID == m_handle) {
+ return;
+ }
+
+ auto request = DestroyVDeviceSerializer::serialize_request(m_handle);
+ if (!request) {
+ LOGGER__CRITICAL("Failed to serialize VDevice_release request");
+ return;
+ }
+
+ auto result = m_client->execute_request(HailoRpcActionID::VDEVICE__DESTROY, MemoryView(*request));
+ if (!result) {
+ LOGGER__CRITICAL("Failed to destroy VDevice! status = {}", result.status());
+ }
+
+ if (HAILO_SUCCESS != DestroyVDeviceSerializer::deserialize_reply(MemoryView(*result))) {
+ LOGGER__CRITICAL("Failed to destroy VDevice! status = {}", result.status());
+ }
+}
+
+Expected<std::shared_ptr<InferModel>> VDeviceHrpcClient::create_infer_model(const std::string &hef_path, const std::string &network_name)
+{
+ CHECK_AS_EXPECTED(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Passing network name is not supported yet!");
+
+ FileReader hef_reader(hef_path);
+ auto status = hef_reader.open();
+ CHECK_SUCCESS(status);
+
+ TRY(auto hef_size, hef_reader.get_size());
+ TRY(auto hef_buffer, Buffer::create(hef_size));
+ status = hef_reader.read(hef_buffer.data(), hef_size);
+ CHECK_SUCCESS(status);
+
+ status = hef_reader.close();
+ CHECK_SUCCESS(status);
+
+ TRY(auto request, CreateInferModelSerializer::serialize_request(m_handle, hef_size));
+ TRY(auto result, m_client->execute_request(HailoRpcActionID::VDEVICE__CREATE_INFER_MODEL,
+ MemoryView(request), [&hef_buffer] (hrpc::RpcConnection connection) -> hailo_status {
+ // TODO: change write to accept uint64_t, or accept file stream instead or write in chunks
+ auto status = connection.write_buffer(MemoryView(hef_buffer));
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+ }));
+ TRY(auto tuple, CreateInferModelSerializer::deserialize_reply(MemoryView(result)));
+
+ CHECK_SUCCESS_AS_EXPECTED(std::get<0>(tuple));
+ auto infer_model_handle = std::get<1>(tuple);
+
+ TRY(auto hef, Hef::create(MemoryView(hef_buffer)));
+ TRY(auto infer_model, InferModelHrpcClient::create(std::move(hef), m_client, infer_model_handle, m_handle, *this));
+
+ return std::shared_ptr<InferModel>(std::move(infer_model));
+}
+
+Expected<ConfiguredNetworkGroupVector> VDeviceHrpcClient::configure(Hef &hef, const NetworkGroupsParamsMap &configure_params)
+{
+ (void)m_handle;
+ (void)hef;
+ (void)configure_params;
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+Expected<std::vector<std::reference_wrapper<Device>>> VDeviceHrpcClient::get_physical_devices() const
+{
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+Expected<std::vector<std::string>> VDeviceHrpcClient::get_physical_devices_ids() const
+{
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+// Currently only homogeneous vDevice is allow (= all devices are from the same type)
+Expected<hailo_stream_interface_t> VDeviceHrpcClient::get_default_streams_interface() const
+{
+ return make_unexpected(HAILO_NOT_IMPLEMENTED);
+}
+
+hailo_status VDeviceHrpcClient::dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ (void)address;
+ (void)size;
+ (void)direction;
+ return HAILO_SUCCESS; // TODO: implement this (HRT-13689)
+}
+
+hailo_status VDeviceHrpcClient::dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ (void)address;
+ (void)size;
+ (void)direction;
+ return HAILO_SUCCESS; // TODO: implement this (HRT-13689)
+}
+
+hailo_status VDeviceHrpcClient::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ (void)dmabuf_fd;
+ (void)size;
+ (void)direction;
+ return HAILO_SUCCESS; // TODO: implement this (HRT-13689)
+}
+
+hailo_status VDeviceHrpcClient::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction)
+{
+ (void)dmabuf_fd;
+ (void)size;
+ (void)direction;
+ return HAILO_SUCCESS; // TODO: implement this (HRT-13689)
+}
+
+} /* namespace hailort */
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file vdevice_hrpc_client.hpp
+ * @brief VDevice HRPC client, represents the user's handle to the VDevice object (held in the hailort server)
+ **/
+
+#ifndef _HAILO_VDEVICE_HRPC_CLIENT_HPP_
+#define _HAILO_VDEVICE_HRPC_CLIENT_HPP_
+
+#include "hailo/hailort.h"
+#include "hrpc/client.hpp"
+#include "vdevice/vdevice_internal.hpp"
+
+namespace hailort
+{
+
+class VDeviceHrpcClient : public VDevice
+{
+public:
+ static Expected<std::unique_ptr<VDevice>> create(const hailo_vdevice_params_t ¶ms);
+
+ VDeviceHrpcClient(std::shared_ptr<hrpc::Client> client, uint32_t handle)
+ : m_client(client), m_handle(handle) {}
+
+ VDeviceHrpcClient(VDeviceHrpcClient &&) = delete;
+ VDeviceHrpcClient(const VDeviceHrpcClient &) = delete;
+ VDeviceHrpcClient &operator=(VDeviceHrpcClient &&) = delete;
+ VDeviceHrpcClient &operator=(const VDeviceHrpcClient &) = delete;
+ virtual ~VDeviceHrpcClient();
+
+ virtual Expected<std::shared_ptr<InferModel>> create_infer_model(const std::string &hef_path,
+ const std::string &network_name = "") override;
+ virtual Expected<ConfiguredNetworkGroupVector> configure(Hef &hef, const NetworkGroupsParamsMap &configure_params={}) override;
+ virtual Expected<std::vector<std::reference_wrapper<Device>>> get_physical_devices() const override;
+ virtual Expected<std::vector<std::string>> get_physical_devices_ids() const override;
+ virtual Expected<hailo_stream_interface_t> get_default_streams_interface() const override;
+ virtual hailo_status dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override;
+
+private:
+ std::shared_ptr<hrpc::Client> m_client;
+ uint32_t m_handle;
+};
+
+} /* namespace hailort */
+
+#endif /* _HAILO_VDEVICE_HRPC_CLIENT_HPP_ */
return status;
}
+ virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override
+ {
+ for (const auto &pair : m_devices) {
+ auto &device = pair.second;
+ const auto status = device->dma_map_dmabuf(dmabuf_fd, size, direction);
+ CHECK_SUCCESS(status);
+ }
+ return HAILO_SUCCESS;
+ }
+
+ virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override
+ {
+ hailo_status status = HAILO_SUCCESS;
+ for (const auto &pair : m_devices) {
+ auto &device = pair.second;
+ // Best effort, propagate first error
+ const auto unmap_status = device->dma_unmap_dmabuf(dmabuf_fd, size, direction);
+ if (HAILO_SUCCESS != unmap_status) {
+ LOGGER__ERROR("Failed unmapping dmabuf {} with status {}", dmabuf_fd, unmap_status);
+ if (HAILO_SUCCESS == status) {
+ status = unmap_status;
+ }
+ }
+ }
+
+ return status;
+ }
+
+ static Expected<HailoRTDriver::AcceleratorType> get_accelerator_type(hailo_device_id_t *device_ids, size_t device_count);
static hailo_status validate_params(const hailo_vdevice_params_t ¶ms);
+ static Expected<bool> device_ids_contains_eth(const hailo_vdevice_params_t ¶ms);
private:
VDeviceBase(std::map<device_id_t, std::unique_ptr<Device>> &&devices, CoreOpsSchedulerPtr core_ops_scheduler,
virtual hailo_status after_fork_in_child() override;
virtual hailo_status dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) override;
virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override;
private:
VDeviceClient(std::unique_ptr<HailoRtRpcClient> client, VDeviceIdentifier &&identifier, std::vector<std::unique_ptr<hailort::Device>> &&devices);
const std::string &network_name = "") override;
virtual hailo_status dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) override;
virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override;
private:
VDeviceHandle(uint32_t handle);
else()
message(FATAL_ERROR "Unexpeced platform target, stopping build")
endif()
+set(DRIVER_OS_DIR ${DRIVER_OS_DIR} PARENT_SCOPE)
set(DRIVER_SRC_FILES
${CMAKE_CURRENT_SOURCE_DIR}/driver/hailort_driver.cpp
${CMAKE_CURRENT_SOURCE_DIR}/vdma_stream.cpp
${CMAKE_CURRENT_SOURCE_DIR}/circular_stream_buffer_pool.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/pcie_session.cpp
+
${CMAKE_CURRENT_SOURCE_DIR}/dma_mapped_buffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pcie/pcie_device.cpp
${CMAKE_CURRENT_SOURCE_DIR}/integrated/integrated_device.cpp
${CMAKE_CURRENT_SOURCE_DIR}/channel/boundary_channel.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/channel/channels_group.cpp
${CMAKE_CURRENT_SOURCE_DIR}/channel/interrupts_dispatcher.cpp
${CMAKE_CURRENT_SOURCE_DIR}/channel/transfer_launcher.cpp
namespace hailort {
namespace vdma {
-
-
-Expected<BoundaryChannelPtr> BoundaryChannel::create(vdma::ChannelId channel_id, Direction direction,
- HailoRTDriver &driver, uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name,
- LatencyMeterPtr latency_meter)
+Expected<BoundaryChannelPtr> BoundaryChannel::create(HailoRTDriver &driver, vdma::ChannelId channel_id,
+ Direction direction, vdma::DescriptorList &&desc_list, TransferLauncher &transfer_launcher,
+ size_t ongoing_transfers, size_t pending_transfers, const std::string &stream_name, LatencyMeterPtr latency_meter)
{
hailo_status status = HAILO_UNINITIALIZED;
- auto channel_ptr = make_shared_nothrow<BoundaryChannel>(channel_id, direction, driver, descs_count,
- desc_page_size, stream_name, latency_meter, status);
+ auto channel_ptr = make_shared_nothrow<BoundaryChannel>(driver, channel_id, direction, std::move(desc_list),
+ transfer_launcher, ongoing_transfers, pending_transfers, stream_name, latency_meter, status);
CHECK_NOT_NULL_AS_EXPECTED(channel_ptr, HAILO_OUT_OF_HOST_MEMORY);
CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating BoundaryChannel");
return channel_ptr;
}
-BoundaryChannel::BoundaryChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver,
- uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name,
- LatencyMeterPtr latency_meter, hailo_status &status) :
+BoundaryChannel::BoundaryChannel(HailoRTDriver &driver, vdma::ChannelId channel_id, Direction direction,
+ DescriptorList &&desc_list, TransferLauncher &transfer_launcher,
+ size_t ongoing_transfers_queue_size, size_t pending_transfers_queue_size,
+ const std::string &stream_name, LatencyMeterPtr latency_meter, hailo_status &status) :
m_channel_id(channel_id),
m_direction(direction),
m_driver(driver),
- m_desc_list(nullptr),
+ m_transfer_launcher(transfer_launcher),
+ m_desc_list(std::move(desc_list)),
m_stream_name(stream_name),
+ m_descs(m_desc_list.count()),
m_is_channel_activated(false),
- m_ongoing_transfers((latency_meter != nullptr) ? ONGOING_TRANSFERS_SIZE/2 : ONGOING_TRANSFERS_SIZE),
+ m_channel_mutex(),
+ // CircularArrays with storage_size x can store x-1 elements, hence the +1
+ m_ongoing_transfers(ongoing_transfers_queue_size + 1),
+ m_pending_transfers(pending_transfers_queue_size + 1),
m_latency_meter(latency_meter),
- m_pending_latency_measurements(ONGOING_TRANSFERS_SIZE) // Make sure there will always be place for latency measure
+ m_pending_latency_measurements(ONGOING_TRANSFERS_SIZE), // Make sure there will always be place for latency measure
+ m_last_timestamp_num_processed(0),
+ m_bounded_buffer(nullptr)
{
if (Direction::BOTH == direction) {
LOGGER__ERROR("Boundary channels must be unidirectional");
return;
}
- CB_INIT(m_descs, descs_count);
-
- status = allocate_descriptor_list(descs_count, desc_page_size);
- if (HAILO_SUCCESS != status) {
- LOGGER__ERROR("Failed to allocate Vdma buffer for channel transfer! status={}", status);
- return;
- }
-
status = HAILO_SUCCESS;
}
-hailo_status BoundaryChannel::trigger_channel_completion(uint16_t hw_num_processed)
+// Function that based off the irq data returns the status to be sent to the callbak functions
+static hailo_status get_callback_status(vdma::ChannelId channel_id, const ChannelIrqData &irq_data)
{
- // NOTE: right now, we can retake the 'completion' descriptor for a new transfer before handling the interrupt.
- // we should have our own pointers indicating whats free instead of reading from HW.
+ hailo_status status = HAILO_UNINITIALIZED;
+ if (!irq_data.is_active) {
+ status = HAILO_STREAM_ABORT;
+ } else if (!irq_data.validation_success) {
+ LOGGER__WARNING("Channel {} validation failed", channel_id);
+ status = HAILO_INTERNAL_FAILURE;
+ } else if ((0 != irq_data.host_error) || (0 != irq_data.device_error)) {
+ LOGGER__WARNING("Channel {} completed with errors: host_error {} device_error {}",
+ channel_id, irq_data.host_error, irq_data.device_error);
+ status = HAILO_INTERNAL_FAILURE;
+ } else {
+ status = HAILO_SUCCESS;
+ }
+ return status;
+}
+hailo_status BoundaryChannel::trigger_channel_completion(const ChannelIrqData &irq_data)
+{
std::unique_lock<std::mutex> lock(m_channel_mutex);
if (!m_is_channel_activated) {
CHECK_SUCCESS(update_latency_meter());
}
- while (!m_ongoing_transfers.empty()) {
- // Reading previous_num_processed inside the loop since on_transfer_complete may increase this value.
- const auto previous_num_processed = static_cast<uint16_t>(CB_TAIL(m_descs));
- if (!is_transfer_complete(m_ongoing_transfers.front(), previous_num_processed, hw_num_processed)) {
- break;
- }
-
+ CHECK(irq_data.transfers_completed <= m_ongoing_transfers.size(), HAILO_INTERNAL_FAILURE,
+ "Invalid amount of completed transfers {} max {}", irq_data.transfers_completed, m_ongoing_transfers.size());
+
+ auto callback_status = get_callback_status(m_channel_id, irq_data);
+ // If channel is no longer active - all transfers should be completed
+ const size_t num_transfers_to_trigger = (HAILO_SUCCESS == callback_status) ? irq_data.transfers_completed :
+ m_ongoing_transfers.size();
+ for (size_t i = 0; i < num_transfers_to_trigger; i++) {
auto transfer = std::move(m_ongoing_transfers.front());
m_ongoing_transfers.pop_front();
- on_transfer_complete(lock, transfer, HAILO_SUCCESS);
+ // We increase desc num_proc (can happen only in this flow). After it is increased -
+ // 1. On D2H channels - the output can be read by the user.
+ // 2. On H2D channels - new input can be written to the buffer.
+ m_descs.set_tail((transfer.last_desc + 1) & m_descs.size_mask());
+
+ // We've freed up room in the descriptor list, so we can launch another transfer
+ if (!m_pending_transfers.empty()) {
+ m_transfer_launcher.enqueue_transfer([this]() {
+ std::unique_lock<std::mutex> lock(m_channel_mutex);
+ if (m_pending_transfers.empty()) {
+ return;
+ }
+ auto transfer_request = std::move(m_pending_transfers.front());
+ m_pending_transfers.pop_front();
+ const auto status = launch_transfer_impl(std::move(transfer_request));
+ if (status != HAILO_SUCCESS) {
+ on_request_complete(lock, transfer_request, status);
+ }
+ });
+ }
+
+ // Call the user callback
+ // We want to do this after launching transfers queued in m_pending_transfers, in order to keep the
+ // callback order consistent.
+ // Also, we want to make sure that the callbacks are called after the descriptors can be reused (so the user
+ // will be able to start new transfer).
+ on_request_complete(lock, transfer.request, callback_status);
}
return HAILO_SUCCESS;
CONTROL_PROTOCOL__host_buffer_info_t BoundaryChannel::get_boundary_buffer_info(uint32_t transfer_size) const
{
// Boundary channels always have scatter gather buffers
- return VdmaEdgeLayer::get_host_buffer_info(VdmaEdgeLayer::Type::SCATTER_GATHER, m_desc_list->dma_address(),
- m_desc_list->desc_page_size(), m_desc_list->count(), transfer_size);
+ return VdmaEdgeLayer::get_host_buffer_info(VdmaEdgeLayer::Type::SCATTER_GATHER, m_desc_list.dma_address(),
+ m_desc_list.desc_page_size(), m_desc_list.count(), transfer_size);
}
hailo_status BoundaryChannel::activate()
m_is_channel_activated = true;
assert(m_ongoing_transfers.empty());
m_last_timestamp_num_processed = 0;
- CB_RESET(m_descs);
+ m_descs.reset();
return HAILO_SUCCESS;
}
-hailo_status BoundaryChannel::deactivate()
+void BoundaryChannel::deactivate()
{
std::unique_lock<std::mutex> lock(m_channel_mutex);
m_is_channel_activated = false;
+}
+
+hailo_status BoundaryChannel::launch_transfer(TransferRequest &&transfer_request)
+{
+ std::unique_lock<std::mutex> lock(m_channel_mutex);
+ if (!m_is_channel_activated) {
+ return HAILO_STREAM_NOT_ACTIVATED;
+ }
+
+ if ((m_ongoing_transfers.size() < m_ongoing_transfers.capacity()) && (m_pending_transfers.size() == 0)) {
+ // There's room in the desc list and there are no pending transfers => execute on user's thread
+ // We can't use the user thread to launch the transfer if there are pending transfers, because we need to
+ // preserve the order of the transfers.
+ return launch_transfer_impl(std::move(transfer_request));
+ }
- // Note: OngoingTransfers held by m_ongoing_transfers may still hold copies of the current callback
- // which in turn holds a reference to *this. Since we deactivate the channel there's no risk that
- // these callbacks will be called and we don't need to reset this callback.
+ if (m_pending_transfers.size() >= m_pending_transfers.capacity()) {
+ return HAILO_QUEUE_IS_FULL;
+ }
+ // Defer to the transfer launcher
+ m_pending_transfers.push_back(std::move(transfer_request));
return HAILO_SUCCESS;
}
-hailo_status BoundaryChannel::launch_transfer(TransferRequest &&transfer_request)
+// Assumes that the m_channel_mutex is locked!
+hailo_status BoundaryChannel::launch_transfer_impl(TransferRequest &&transfer_request)
{
- std::unique_lock<std::mutex> lock(m_channel_mutex);
if (!m_is_channel_activated) {
return HAILO_STREAM_NOT_ACTIVATED;
}
- if (m_ongoing_transfers.size() >= get_max_ongoing_transfers(transfer_request.get_total_transfer_size())) {
+ if (m_ongoing_transfers.size() >= m_ongoing_transfers.capacity()) {
return HAILO_QUEUE_IS_FULL;
}
- auto num_available = static_cast<uint16_t>(CB_HEAD(m_descs));
+ auto num_available = static_cast<uint16_t>(m_descs.head());
const uint16_t first_desc = num_available;
uint16_t last_desc = std::numeric_limits<uint16_t>::max();
uint16_t total_descs_count = 0;
transfer_buffer.size()
});
- const auto desired_desc_num = m_desc_list->descriptors_in_buffer(transfer_buffer.size());
+ const auto desired_desc_num = m_desc_list.descriptors_in_buffer(transfer_buffer.size());
CHECK(desired_desc_num <= MAX_SG_DESCS_COUNT, HAILO_INTERNAL_FAILURE);
const uint16_t desc_num = static_cast<uint16_t>(desired_desc_num);
assert(total_descs_count + desc_num < MAX_SG_DESCS_COUNT);
total_descs_count = static_cast<uint16_t>(total_descs_count + desc_num);
- last_desc = static_cast<uint16_t>((current_num_available + desc_num - 1) & m_descs.size_mask);
- current_num_available = static_cast<uint16_t>((last_desc + 1) & m_descs.size_mask);
+ last_desc = static_cast<uint16_t>((current_num_available + desc_num - 1) & m_descs.size_mask());
+ current_num_available = static_cast<uint16_t>((last_desc + 1) & m_descs.size_mask());
}
auto first_desc_interrupts = InterruptsDomain::NONE;
}
const auto last_desc_interrupts = InterruptsDomain::HOST;
- int num_processed = CB_TAIL(m_descs);
- int num_free = CB_AVAIL(m_descs, num_available, num_processed);
+ int num_processed = m_descs.tail();
+ int num_free = m_descs.avail(num_available, num_processed);
if (total_descs_count > num_free) {
return HAILO_OUT_OF_DESCRIPTORS;
}
- m_ongoing_transfers.push_back(OngoingTransfer{std::move(transfer_request), last_desc});
if (m_latency_meter) {
assert(!m_pending_latency_measurements.full());
m_pending_latency_measurements.push_back(m_direction == Direction::H2D ? first_desc : last_desc);
}
- CB_ENQUEUE(m_descs, total_descs_count);
+ m_descs.enqueue(total_descs_count);
- TRY(const auto desc_programmed, m_driver.launch_transfer(
+ TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto desc_programmed, m_driver.launch_transfer(
m_channel_id,
- m_desc_list->handle(),
+ m_desc_list.handle(),
num_available,
driver_transfer_buffers,
should_bind,
));
CHECK(total_descs_count == desc_programmed, HAILO_INTERNAL_FAILURE,
"Inconsistent desc programed expecting {} got {}", total_descs_count, desc_programmed);
+ m_ongoing_transfers.push_back(OngoingTransfer{std::move(transfer_request), last_desc});
return HAILO_SUCCESS;
}
{
CHECK(m_bounded_buffer == nullptr, HAILO_INTERNAL_FAILURE,
"Buffer is already bound to channel {}", m_channel_id);
- const auto expected_size = static_cast<size_t>(m_desc_list->desc_page_size()) * m_desc_list->count();
+ const auto expected_size = static_cast<size_t>(m_desc_list.desc_page_size()) * m_desc_list.count();
CHECK(buffer->size() == expected_size, HAILO_INVALID_ARGUMENT,
- "Buffer size {} does not feet in desc list - descs count {} desc page size {}", buffer->size(),
- m_desc_list->count(), m_desc_list->desc_page_size());
+ "Buffer size {} does not fit in desc list - descs count {} desc page size {}", buffer->size(),
+ m_desc_list.count(), m_desc_list.desc_page_size());
static const size_t DEFAULT_BUFFER_OFFSET = 0;
- CHECK_SUCCESS(m_desc_list->configure_to_use_buffer(*buffer, buffer->size(), DEFAULT_BUFFER_OFFSET, m_channel_id));
+ CHECK_SUCCESS(m_desc_list.program(*buffer, buffer->size(), DEFAULT_BUFFER_OFFSET, m_channel_id));
m_bounded_buffer = buffer;
return HAILO_SUCCESS;
}
void BoundaryChannel::cancel_pending_transfers()
{
std::unique_lock<std::mutex> lock(m_channel_mutex);
+ // Cancel all ongoing transfers
while (!m_ongoing_transfers.empty()) {
auto transfer = std::move(m_ongoing_transfers.front());
m_ongoing_transfers.pop_front();
- on_transfer_complete(lock, transfer, HAILO_STREAM_ABORT);
+ on_request_complete(lock, transfer.request, HAILO_STREAM_ABORT);
+ }
+
+ // Then cancel all pending transfers (which were to happen after the ongoing transfers are done)
+ while (!m_pending_transfers.empty()) {
+ auto pending_transfer = std::move(m_pending_transfers.front());
+ m_pending_transfers.pop_front();
+
+ on_request_complete(lock, pending_transfer, HAILO_STREAM_ABORT);
}
}
-size_t BoundaryChannel::get_max_ongoing_transfers(size_t transfer_size) const
+size_t BoundaryChannel::get_max_ongoing_transfers(size_t /* transfer_size */) const
{
- // Add desc for boundary channel because might need extra for non aligned async API
- const auto descs_in_transfer = m_desc_list->descriptors_in_buffer(transfer_size) + 1;
- const auto descs_count = CB_SIZE(m_descs);
- size_t max_transfers_in_buffer = (descs_count - 1) / descs_in_transfer;
+ // TODO: Remove transfer_size from get_max_ongoing_transfers (HRT-13419)
+ return std::max(m_pending_transfers.capacity(), m_ongoing_transfers.capacity());
+}
- return std::min(max_transfers_in_buffer, m_ongoing_transfers.capacity());
+// TODO: try and get rid of this func and merge with get_max_ongoing_transfers (HRT-13557)
+size_t BoundaryChannel::get_max_aligned_transfers_in_desc_list(size_t transfer_size) const
+{
+ // Since this calc if for aligned transfers, we don't need to factor in the bounce buffer
+ static const auto NO_BOUNCE_BUFFER = false;
+ return m_desc_list.max_transfers(static_cast<uint32_t>(transfer_size), NO_BOUNCE_BUFFER);
}
hailo_status BoundaryChannel::update_latency_meter()
return HAILO_SUCCESS;
}
-bool BoundaryChannel::is_transfer_complete(const OngoingTransfer &transfer, uint16_t previous_num_processed,
- uint16_t current_num_processed) const
-{
- // Transfer is complete if its last descriptor is in [previous_num_processed, current_num_processed) or
- // the the buffer is empty (previous_num_processed == CB_HEAD(m_descs))
- return is_desc_between(previous_num_processed, current_num_processed, transfer.last_desc) ||
- (current_num_processed == CB_HEAD(m_descs));
-}
-
-void BoundaryChannel::on_transfer_complete(std::unique_lock<std::mutex> &lock,
- OngoingTransfer &transfer, hailo_status complete_status)
+void BoundaryChannel::on_request_complete(std::unique_lock<std::mutex> &lock, TransferRequest &request,
+ hailo_status complete_status)
{
- // We increase desc num_proc (can happen only in this flow). After it is increased -
- // 1. On D2H channels - the output can be read by the user.
- // 2. On H2D channels - new input can be written to the buffer.
- _CB_SET(m_descs.tail, (transfer.last_desc + 1) & m_descs.size_mask);
-
- // Finally, we notify user callbacks registered with the transfer.
- // We want to make sure that the callbacks are called after the descriptors can be reused (So the user will
- // be able to start new transfer).
lock.unlock();
- transfer.request.callback(complete_status);
+ request.callback(complete_status);
lock.lock();
}
}
}
-hailo_status BoundaryChannel::allocate_descriptor_list(uint32_t descs_count, uint16_t desc_page_size)
-{
- static const bool CIRCULAR = true;
- auto desc_list_exp = DescriptorList::create(descs_count, desc_page_size, CIRCULAR, m_driver);
- CHECK_EXPECTED_AS_STATUS(desc_list_exp);
-
- m_desc_list = make_shared_nothrow<DescriptorList>(desc_list_exp.release());
- CHECK_NOT_NULL(m_desc_list, HAILO_OUT_OF_HOST_MEMORY);
-
- return HAILO_SUCCESS;
-}
-
hailo_status BoundaryChannel::validate_bound_buffer(TransferRequest &transfer_request)
{
assert(m_bounded_buffer);
"When bound buffer is used, transfer request must contain only one buffer");
auto &transfer_buffer = transfer_request.transfer_buffers[0];
- const auto num_available = CB_HEAD(m_descs);
- const auto expected_offset = static_cast<size_t>(m_desc_list->desc_page_size()) * num_available;
+ const auto num_available = m_descs.head();
+ const auto expected_offset = static_cast<size_t>(m_desc_list.desc_page_size()) * num_available;
CHECK(transfer_buffer.offset() == expected_offset, HAILO_INTERNAL_FAILURE,
"Unexpected buffer offset, expected {} actual {}", expected_offset, transfer_buffer.offset());
- CHECK(transfer_buffer.base_buffer().data() == reinterpret_cast<const uint8_t*>(m_bounded_buffer->user_address()), HAILO_INTERNAL_FAILURE,
+ TRY(auto base_buffer, transfer_buffer.base_buffer());
+ CHECK(base_buffer.data() == reinterpret_cast<const uint8_t*>(m_bounded_buffer->user_address()), HAILO_INTERNAL_FAILURE,
"Got the wrong buffer");
- CHECK(transfer_buffer.base_buffer().size() == m_bounded_buffer->size(), HAILO_INTERNAL_FAILURE,
- "Got invalid buffer size {}, expected {}", transfer_buffer.base_buffer().size(), m_bounded_buffer->size());
+ CHECK(base_buffer.size() == m_bounded_buffer->size(), HAILO_INTERNAL_FAILURE,
+ "Got invalid buffer size {}, expected {}", base_buffer.size(), m_bounded_buffer->size());
return HAILO_SUCCESS;
}
#define _HAILO_VDMA_BOUNDARY_CHANNEL_HPP_
#include "vdma/channel/channel_id.hpp"
+#include "vdma/channel/transfer_launcher.hpp"
#include "vdma/memory/descriptor_list.hpp"
#include "stream_common/transfer_common.hpp"
public:
using Direction = HailoRTDriver::DmaDirection;
- static Expected<BoundaryChannelPtr> create(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver,
- uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name = "", LatencyMeterPtr latency_meter = nullptr);
+ static Expected<BoundaryChannelPtr> create(HailoRTDriver &driver, vdma::ChannelId channel_id, Direction direction,
+ vdma::DescriptorList &&desc_list, TransferLauncher &transfer_launcher, size_t ongoing_transfers,
+ size_t pending_transfers = 0, const std::string &stream_name = "", LatencyMeterPtr latency_meter = nullptr);
- BoundaryChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, uint32_t descs_count,
- uint16_t desc_page_size, const std::string &stream_name, LatencyMeterPtr latency_meter,
- hailo_status &status);
+ BoundaryChannel(HailoRTDriver &driver, vdma::ChannelId channel_id, Direction direction, DescriptorList &&desc_list,
+ TransferLauncher &transfer_launcher, size_t ongoing_transfers_queue_size, size_t pending_transfers_queue_size,
+ const std::string &stream_name, LatencyMeterPtr latency_meter, hailo_status &status);
BoundaryChannel(const BoundaryChannel &other) = delete;
BoundaryChannel &operator=(const BoundaryChannel &other) = delete;
BoundaryChannel(BoundaryChannel &&other) = delete;
BoundaryChannel &operator=(BoundaryChannel &&other) = delete;
virtual ~BoundaryChannel() = default;
- // Called after the FW activated the channel.
+ /**
+ * Activates the channel object, assume the vDMA channel registers are already in activated state.
+ */
hailo_status activate();
- // Called before the FW deactivated the channel.
- hailo_status deactivate();
-
- hailo_status trigger_channel_completion(uint16_t hw_num_processed);
+ /**
+ * Deactivates the channel object, assume the vDMA channel registers are already in deactivated state.
+ */
+ void deactivate();
// Calls all pending transfer callbacks (if they exist), marking them as canceled by passing
// HAILO_STREAM_ABORT as a status to the callbacks.
// unexpected results
void cancel_pending_transfers();
+ /**
+ * Called when some transfer (or transfers) is completed.
+ */
+ hailo_status trigger_channel_completion(const ChannelIrqData &irq_data);
+
hailo_status launch_transfer(TransferRequest &&transfer_request);
// To avoid buffer bindings, one can call this function to statically bind a full buffer to the channel. The buffer
// size should be exactly desc_page_size() * descs_count() of current descriptors list.
hailo_status bind_buffer(MappedBufferPtr buffer);
+ // TODO: rename BoundaryChannel::get_max_ongoing_transfers to BoundaryChannel::get_max_parallel_transfers (HRT-13513)
size_t get_max_ongoing_transfers(size_t transfer_size) const;
+ size_t get_max_aligned_transfers_in_desc_list(size_t transfer_size) const;
CONTROL_PROTOCOL__host_buffer_info_t get_boundary_buffer_info(uint32_t transfer_size) const;
return m_stream_name;
}
- std::shared_ptr<DescriptorList> get_desc_list()
+ DescriptorList &get_desc_list()
{
return m_desc_list;
}
-private:
+ bool should_measure_timestamp() const { return m_latency_meter != nullptr; }
+private:
hailo_status update_latency_meter();
- bool is_transfer_complete(const OngoingTransfer &transfer, uint16_t previous_num_processed,
- uint16_t current_num_processed) const;
- void on_transfer_complete(std::unique_lock<std::mutex> &lock, OngoingTransfer &transfer,
+ void on_request_complete(std::unique_lock<std::mutex> &lock, TransferRequest &request,
hailo_status complete_status);
+ hailo_status launch_transfer_impl(TransferRequest &&transfer_request);
static bool is_desc_between(uint16_t begin, uint16_t end, uint16_t desc);
- hailo_status allocate_descriptor_list(uint32_t descs_count, uint16_t desc_page_size);
hailo_status validate_bound_buffer(TransferRequest &transfer_request);
const vdma::ChannelId m_channel_id;
const Direction m_direction;
HailoRTDriver &m_driver;
- std::shared_ptr<DescriptorList> m_desc_list; // Host side descriptor list
+ TransferLauncher &m_transfer_launcher;
+ DescriptorList m_desc_list; // Host side descriptor list
const std::string m_stream_name;
- circbuf_t m_descs;
+ // Since all desc list sizes are a power of 2, we can use IsPow2Tag to optimize the circular buffer
+ CircularBuffer<IsPow2Tag> m_descs;
bool m_is_channel_activated;
std::mutex m_channel_mutex;
- CircularArray<OngoingTransfer> m_ongoing_transfers;
+ // * m_pending_transfers holds transfers that are waiting to be bound to the descriptor list.
+ // * m_ongoing_transfers holds transfers that have been bound to the descriptor list and
+ // are waiting to be completed.
+ // * Note that the capacity of the pending_transfers and ongoing_transfers circular
+ // buffers may not be a power of 2, hence the IsNotPow2Tag
+ CircularArray<OngoingTransfer, IsNotPow2Tag> m_ongoing_transfers;
+ CircularArray<TransferRequest, IsNotPow2Tag> m_pending_transfers;
// About HW latency measurements:
// - For each ongoing transfer, we push some num-proc value to the pending_latency_measurements array. When this
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file channels_group.cpp
+ **/
+
+#include "channels_group.hpp"
+
+namespace hailort {
+namespace vdma {
+
+ChannelsGroup::ChannelsGroup(std::initializer_list<BoundaryChannelPtr> channels)
+{
+ for (const auto &channel : channels) {
+ add_channel(channel);
+ }
+}
+
+void ChannelsGroup::add_channel(BoundaryChannelPtr channel)
+{
+ const auto id = channel->get_channel_id();
+ assert(nullptr == m_channels[id.engine_index][id.channel_index]);
+ m_channels[id.engine_index][id.channel_index] = channel;
+}
+
+ChannelsBitmap ChannelsGroup::bitmap() const
+{
+ ChannelsBitmap bitmap{};
+ for (size_t i = 0; i < m_channels.size(); i++) {
+ for (size_t j = 0; j < m_channels[i].size(); j++) {
+ if (m_channels[i][j]) {
+ bitmap[i] |= (1 << j);
+ }
+ }
+ }
+ return bitmap;
+}
+
+bool ChannelsGroup::should_measure_timestamp() const
+{
+ for (const auto &engine : m_channels) {
+ for (const auto &channel : engine) {
+ if (channel && channel->should_measure_timestamp()) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+Expected<BoundaryChannelPtr> ChannelsGroup::get_by_id(vdma::ChannelId channel_id)
+{
+ auto channel = m_channels[channel_id.engine_index][channel_id.channel_index];
+ if (!channel) {
+ return make_unexpected(HAILO_NOT_FOUND);
+ }
+ return channel;
+}
+
+Expected<BoundaryChannelPtr> ChannelsGroup::get_by_name(const std::string &stream_name)
+{
+ for (const auto &engine : m_channels) {
+ for (const auto &channel : engine) {
+ if (channel && (channel->stream_name() == stream_name)) {
+ return BoundaryChannelPtr{channel};
+ }
+ }
+ }
+ return make_unexpected(HAILO_NOT_FOUND);
+}
+
+void ChannelsGroup::process_interrupts(IrqData &&irq_data)
+{
+ assert(irq_data.channels_count <= ARRAY_ENTRIES(irq_data.channels_irq_data));
+ for (uint8_t irq_index = 0; irq_index < irq_data.channels_count; irq_index++) {
+ const auto &channel_irq_data = irq_data.channels_irq_data[irq_index];
+ auto status = process_channel_interrupt(channel_irq_data); // TODO: TODO: HRT-9429 done
+ if ((status != HAILO_SUCCESS) && (status != HAILO_STREAM_NOT_ACTIVATED)) {
+ LOGGER__ERROR("Trigger channel completion failed on channel {} with status {}", channel_irq_data.channel_id, status);
+ }
+ }
+}
+
+hailo_status ChannelsGroup::process_channel_interrupt(const ChannelIrqData &channel_irq_data)
+{
+ TRY(auto channel, get_by_id(channel_irq_data.channel_id), "Channel {} not found", channel_irq_data.channel_id);
+ return channel->trigger_channel_completion(channel_irq_data);
+}
+
+} /* namespace vdma */
+} /* namespace hailort */
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file channels_group.hpp
+ * @brief Contains a group of channels that are used together.
+ **/
+
+#ifndef _HAILO_CHANNELS_GROUP_HPP_
+#define _HAILO_CHANNELS_GROUP_HPP_
+
+#include "vdma/channel/boundary_channel.hpp"
+
+namespace hailort {
+namespace vdma {
+
+class ChannelsGroup final {
+public:
+ ChannelsGroup() = default;
+ ChannelsGroup(std::initializer_list<BoundaryChannelPtr> channels);
+
+ void add_channel(BoundaryChannelPtr channel);
+
+ ChannelsBitmap bitmap() const;
+ bool should_measure_timestamp() const;
+ Expected<BoundaryChannelPtr> get_by_id(vdma::ChannelId channel_id);
+ Expected<BoundaryChannelPtr> get_by_name(const std::string &stream_name);
+
+ void process_interrupts(IrqData &&irq_data);
+
+private:
+
+ hailo_status process_channel_interrupt(const ChannelIrqData &irq_data);
+
+ std::array<
+ std::array<BoundaryChannelPtr, MAX_VDMA_CHANNELS_COUNT>,
+ MAX_VDMA_ENGINES_COUNT
+ > m_channels;
+};
+
+} /* namespace vdma */
+} /* namespace hailort */
+
+#endif /* _HAILO_CHANNELS_GROUP_HPP_ */
CHECK_NOT_NULL(wait_context, HAILO_OUT_OF_HOST_MEMORY);
m_wait_context = std::move(wait_context);
- auto status = m_driver.get().vdma_interrupts_enable(m_wait_context->bitmap, enable_timestamp_measure);
- CHECK_SUCCESS(status, "Failed to enable vdma interrupts");
+ auto status = m_driver.get().vdma_enable_channels(m_wait_context->bitmap, enable_timestamp_measure);
+ CHECK_SUCCESS(status, "Failed to enable vdma channels");
}
m_cond.notify_one();
return HAILO_SUCCESS;
}
+hailo_status InterruptsDispatcher::start(const ChannelsGroup &channels_group)
+{
+ return start(channels_group.bitmap(), channels_group.should_measure_timestamp(),
+ [channels_group=channels_group](IrqData &&irq_data) mutable {
+ channels_group.process_interrupts(std::move(irq_data));
+ });
+}
+
hailo_status InterruptsDispatcher::stop()
{
std::unique_lock<std::mutex> lock(m_mutex);
m_wait_context = nullptr;
// Calling disable interrupts will cause the vdma_interrupts_wait to return.
- auto status = m_driver.get().vdma_interrupts_disable(bitmap);
+ auto status = m_driver.get().vdma_disable_channels(bitmap);
CHECK_SUCCESS(status, "Failed to disable vdma interrupts");
// Needs to make sure that the interrupts thread is disabled.
// vdma_interrupts_wait is a blocking function that returns in this scenarios:
// 1. We got a new interrupts, irq_data will be passed to the process_irq callback
- // 2. vdma_interrupts_disable will be called, vdma_interrupts_wait will return with an empty list.
+ // 2. vdma_disable_channels will be called, vdma_interrupts_wait will return with an empty list.
// 3. Other error returns - shouldn't really happen, we exit the interrupt thread.
lock.unlock();
auto irq_data = m_driver.get().vdma_interrupts_wait(wait_context.bitmap);
#define _HAILO_VDMA_INTERRUPTS_DISPATCHER_HPP_
#include "vdma/driver/hailort_driver.hpp"
+#include "vdma/channel/channels_group.hpp"
+
#include <thread>
#include <functional>
#include <condition_variable>
hailo_status start(const ChannelsBitmap &channels_bitmap, bool enable_timestamp_measure,
const ProcessIrqCallback &process_irq);
+ hailo_status start(const ChannelsGroup &channels_group);
hailo_status stop();
private:
m_thread_active = false;\r
\r
while (!m_queue.empty()) {\r
+ // No need signal that the transfer was aborted, it'll be done in BoundaryChannel::cancel_pending_transfers\r
m_queue.pop();\r
- // TODO: need to call the callbacks to signal that they were aborted? (HRT-13110)\r
- // like this:\r
- // auto transfer_request = m_queue.front();\r
- // m_queue.pop();\r
- // transfer_request.callback(HAILO_STREAM_ABORT);\r
- // or can it be done in BoundaryChannel::cancel_pending_transfers?\r
}\r
\r
// TODO: Keep stop flow used in interrupt thread? (HRT-13110)\r
class TransferLauncher final\r
{\r
public:\r
- // TODO: fix this to be a proper transfer object (HRT-13110)\r
using Transfer = std::function<void()>;\r
\r
static Expected<std::unique_ptr<TransferLauncher>> create();\r
m_transfer_size(transfer_size),
m_base_buffer(std::move(base_buffer)),
m_mappings(std::move(mappings)),
+ m_queue(static_cast<int>(descs_count)),
m_next_enqueue_desc_offset(0)
{
assert(is_powerof2(descs_count) && (descs_count > 0));
assert(m_base_buffer.size() == (m_desc_page_size * descs_count));
- CB_INIT(m_queue, descs_count);
- m_queue.head = static_cast<int>(descs_count - 1);
+ m_queue.set_head(static_cast<int>(descs_count) - 1);
}
size_t CircularStreamBufferPool::max_queue_size() const
{
- return (m_queue.size - 1) / DIV_ROUND_UP(m_transfer_size, m_desc_page_size);
+ return (m_queue.size() - 1) / DIV_ROUND_UP(m_transfer_size, m_desc_page_size);
}
size_t CircularStreamBufferPool::buffers_ready_to_dequeue() const
{
- const size_t descs_available = CB_PROG(m_queue, CB_HEAD(m_queue), CB_TAIL(m_queue));
+ const size_t descs_available = m_queue.prog(m_queue.head(), m_queue.tail());
return descs_available / descs_in_transfer();
}
{
CHECK_AS_EXPECTED(buffers_ready_to_dequeue() > 0, HAILO_INTERNAL_FAILURE, "CircularStreamBufferPool is empty");
- const size_t offset_in_buffer = CB_TAIL(m_queue) * m_desc_page_size;
- CB_DEQUEUE(m_queue, descs_in_transfer());
+ const size_t offset_in_buffer = m_queue.tail() * m_desc_page_size;
+ m_queue.dequeue(static_cast<int>(descs_in_transfer()));
return TransferBuffer {
MemoryView(m_base_buffer),
m_transfer_size,
hailo_status CircularStreamBufferPool::enqueue(TransferBuffer &&buffer_info)
{
const size_t descs_required = descs_in_transfer();
- const size_t descs_available = CB_AVAIL(m_queue, CB_HEAD(m_queue), CB_TAIL(m_queue));
+ const size_t descs_available = m_queue.avail(m_queue.head(), m_queue.tail());
CHECK(descs_available >= descs_required, HAILO_INTERNAL_FAILURE, "Can enqueue without previous dequeue");
- CHECK(buffer_info.base_buffer().data() == m_base_buffer.data(), HAILO_INTERNAL_FAILURE, "Got the wrong buffer");
+ TRY(auto base_buffer, buffer_info.base_buffer());
+ CHECK(base_buffer.data() == m_base_buffer.data(), HAILO_INTERNAL_FAILURE, "Got the wrong buffer");
CHECK(buffer_info.size() == m_transfer_size, HAILO_INTERNAL_FAILURE, "Got invalid buffer size {}, expected {}",
buffer_info.size(), m_transfer_size);
"Out of order enqueue is not supported in CircularStreamBufferPool. Got offset {}, expected {}",
buffer_info.offset(), expected_offset);
- CB_ENQUEUE(m_queue, descs_required);
- m_next_enqueue_desc_offset = (m_next_enqueue_desc_offset + descs_required) & m_queue.size_mask;
+ m_queue.enqueue(static_cast<int>(descs_required));
+ m_next_enqueue_desc_offset = (m_next_enqueue_desc_offset + descs_required) & m_queue.size_mask();
return HAILO_SUCCESS;
}
void CircularStreamBufferPool::reset_pointers()
{
- CB_RESET(m_queue);
- m_queue.head = static_cast<int>(m_queue.size - 1);
+ m_queue.reset();
+ m_queue.set_head(static_cast<int>(m_queue.size()) - 1);
m_next_enqueue_desc_offset = 0;
}
const size_t m_transfer_size;
- // m_mapped_buffer.size() must be CB_SIZE(m_queue) * m_desc_page_size
+ // m_mapped_buffer.size() must be m_queue.size() * m_desc_page_size
Buffer m_base_buffer;
DmaMappedBuffer m_mappings;
// Head/tail based queue that manages the buffer pool.
// The head and tail are in m_desc_page_size granularity.
//
- // If CB_HEAD(m_queue) == CB_TAIL(m_queue) the pool is empty.
+ // If m_queue.head() == m_queue.tail() the pool is empty.
// Otherwise, the buffers that can be in use starts from
- // CB_TAIL(m_queue) * m_desc_page_size (inclusive)
+ // m_queue.tail() * m_desc_page_size (inclusive)
// until
- // CB_HEAD(m_queue) * m_desc_page_size (exclusive)
- circbuf_t m_queue;
+ // m_queue.head() * m_desc_page_size (exclusive)
+ CircularBuffer<IsPow2Tag> m_queue;
// Used to validate that the buffers are enqueued in order.
size_t m_next_enqueue_desc_offset;
(HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE | HAILO_VDMA_INTERRUPTS_DOMAIN_HOST), "Driver and libhailort parameters mismatch");
+static const std::string INTEGRATED_NNC_DRIVER_PATH = "/dev/hailo_integrated_nnc";
+static const std::string PCIE_EP_DRIVER_PATH = "/dev/hailo_pci_ep";
+
+
#define CHECK_IOCTL_RESULT(err, message) do { \
auto __err = (err); \
CHECK(0 == __err, HAILO_DRIVER_FAIL, message " errno: {}", __err); \
} while (0)
-static hailo_dma_data_direction direction_to_dma_data_direction(HailoRTDriver::DmaDirection direction) {
- switch (direction) {
- case HailoRTDriver::DmaDirection::H2D:
- return HAILO_DMA_TO_DEVICE;
- case HailoRTDriver::DmaDirection::D2H:
- return HAILO_DMA_FROM_DEVICE;
- case HailoRTDriver::DmaDirection::BOTH:
- return HAILO_DMA_BIDIRECTIONAL;
+static hailo_dma_buffer_type driver_dma_buffer_type_to_dma_buffer_type(HailoRTDriver::DmaBufferType buffer_type) {
+ switch (buffer_type) {
+ case HailoRTDriver::DmaBufferType::USER_PTR_BUFFER:
+ return HAILO_DMA_USER_PTR_BUFFER;
+ case HailoRTDriver::DmaBufferType::DMABUF_BUFFER:
+ return HAILO_DMA_DMABUF_BUFFER;
}
assert(false);
- // On release build Return value that will make ioctls to fail.
- return HAILO_DMA_NONE;
+ return HAILO_DMA_BUFFER_MAX_ENUM;
}
static enum hailo_cpu_id translate_cpu_id(hailo_cpu_id_t cpu_id)
return HAILO_TRANSFER_MEMORY_DMA_ENGINE1;
case MemoryType::DMA_ENGINE2:
return HAILO_TRANSFER_MEMORY_DMA_ENGINE2;
+ case MemoryType::PCIE_EP_CONFIG:
+ return HAILO_TRANSFER_MEMORY_PCIE_EP_CONFIG;
+ case MemoryType::PCIE_EP_BRIDGE:
+ return HAILO_TRANSFER_MEMORY_PCIE_EP_BRIDGE;
}
assert(false);
return HAILO_TRANSFER_MEMORY_MAX_ENUM;
}
+static hailo_dma_data_direction direction_to_dma_data_direction(HailoRTDriver::DmaDirection direction) {
+ switch (direction) {
+ case HailoRTDriver::DmaDirection::H2D:
+ return HAILO_DMA_TO_DEVICE;
+ case HailoRTDriver::DmaDirection::D2H:
+ return HAILO_DMA_FROM_DEVICE;
+ case HailoRTDriver::DmaDirection::BOTH:
+ return HAILO_DMA_BIDIRECTIONAL;
+ }
+
+ assert(false);
+ // On release build Return value that will make ioctls to fail.
+ return HAILO_DMA_NONE;
+}
+
// TODO: validate wraparounds for buffer/mapping handles in the driver (HRT-9509)
const uintptr_t HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE = INVALID_DRIVER_HANDLE_VALUE;
const size_t HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE = INVALID_DRIVER_HANDLE_VALUE;
#error "unsupported platform!"
#endif
-Expected<std::unique_ptr<HailoRTDriver>> HailoRTDriver::create(const DeviceInfo &device_info)
+Expected<std::unique_ptr<HailoRTDriver>> HailoRTDriver::create(const std::string &device_id, const std::string &dev_path)
{
- TRY(auto fd, open_device_file(device_info.dev_path));
+ TRY(auto fd, open_device_file(dev_path));
hailo_status status = HAILO_UNINITIALIZED;
- std::unique_ptr<HailoRTDriver> driver(new (std::nothrow) HailoRTDriver(device_info, std::move(fd), status));
+ std::unique_ptr<HailoRTDriver> driver(new (std::nothrow) HailoRTDriver(device_id, std::move(fd), status));
CHECK_NOT_NULL_AS_EXPECTED(driver, HAILO_OUT_OF_HOST_MEMORY);
CHECK_SUCCESS_AS_EXPECTED(status);
return driver;
}
+Expected<std::unique_ptr<HailoRTDriver>> HailoRTDriver::create_pcie(const std::string &device_id)
+{
+ TRY(const auto scan_results, scan_devices());
+
+ auto device_found = std::find_if(scan_results.cbegin(), scan_results.cend(),
+ [&device_id](const auto &compared_scan_result) {
+ return (device_id == compared_scan_result.device_id);
+ });
+ CHECK(device_found != scan_results.cend(), HAILO_INVALID_ARGUMENT, "Requested device not found");
+
+ return create(device_found->device_id, device_found->dev_path);
+}
+
+Expected<std::unique_ptr<HailoRTDriver>> HailoRTDriver::create_integrated_nnc()
+{
+ return create(INTEGRATED_NNC_DEVICE_ID, INTEGRATED_NNC_DRIVER_PATH);
+}
+
+bool HailoRTDriver::is_integrated_nnc_loaded()
+{
+#if defined(_MSC_VER)
+ // windows is not supported for integrated_nnc driver
+ return false;
+#else
+ return (access(INTEGRATED_NNC_DRIVER_PATH.c_str(), F_OK) == 0);
+#endif // defined(_MSC_VER)
+}
+
+Expected<std::unique_ptr<HailoRTDriver>> HailoRTDriver::create_pcie_ep()
+{
+ return create(PCIE_EP_DEVICE_ID, PCIE_EP_DRIVER_PATH);
+}
+
+bool HailoRTDriver::is_pcie_ep_loaded()
+{
+#if defined(_MSC_VER)
+ // windows is not supported for pcie_ep driver
+ return false;
+#else
+ return (access(PCIE_EP_DRIVER_PATH.c_str(), F_OK) == 0);
+#endif // defined(_MSC_VER)
+}
+
static hailo_status validate_driver_version(const hailo_driver_info &driver_info)
{
hailo_version_t library_version{};
return HAILO_SUCCESS;
}
-HailoRTDriver::HailoRTDriver(const DeviceInfo &device_info, FileDescriptor &&fd, hailo_status &status) :
+HailoRTDriver::HailoRTDriver(const std::string &device_id, FileDescriptor &&fd, hailo_status &status) :
m_fd(std::move(fd)),
- m_device_info(device_info),
+ m_device_id(device_id),
m_allocate_driver_buffer(false)
{
hailo_driver_info driver_info{};
case HAILO_DMA_TYPE_DRAM:
m_dma_type = DmaType::DRAM;
break;
+ case HAILO_DMA_TYPE_PCI_EP:
+ m_dma_type = DmaType::PCIE_EP;
+ break;
default:
LOGGER__ERROR("Invalid dma type returned from ioctl {}", device_properties.dma_type);
status = HAILO_DRIVER_FAIL;
}
}
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_all_devices()
+{
+ std::vector<HailoRTDriver::DeviceInfo> devices_info;
+
+ TRY(auto nnc_devices, scan_nnc_devices());
+ if (!nnc_devices.empty()) {
+ devices_info.insert(devices_info.end(), nnc_devices.begin(), nnc_devices.end());
+ }
+
+ TRY(auto soc_devices, scan_soc_devices());
+ if (!soc_devices.empty()) {
+ devices_info.insert(devices_info.end(), soc_devices.begin(), soc_devices.end());
+ }
+
+ return devices_info;
+}
+
Expected<std::vector<HailoRTDriver::DeviceInfo>> HailoRTDriver::scan_devices()
{
- auto device_names = list_devices();
- CHECK_EXPECTED(device_names, "Failed listing pcie devices");
+ return scan_all_devices();
+}
+Expected<std::vector<HailoRTDriver::DeviceInfo>> HailoRTDriver::scan_devices(AcceleratorType acc_type)
+{
std::vector<HailoRTDriver::DeviceInfo> devices_info;
- for (const auto &device_name : device_names.value()) {
- auto device_info = query_device_info(device_name);
- CHECK_EXPECTED(device_info, "failed parsing device info for {}", device_name);
- devices_info.push_back(device_info.release());
+
+ if (AcceleratorType::SOC_ACCELERATOR == acc_type) {
+ TRY(devices_info, scan_soc_devices());
+ } else if (AcceleratorType::NNC_ACCELERATOR == acc_type) {
+ TRY(devices_info, scan_nnc_devices());
}
+
return devices_info;
}
return HAILO_SUCCESS;
}
-hailo_status HailoRTDriver::vdma_interrupts_enable(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure)
+hailo_status HailoRTDriver::vdma_enable_channels(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure)
{
CHECK(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given");
- hailo_vdma_interrupts_enable_params params{};
+ hailo_vdma_enable_channels_params params{};
std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine);
params.enable_timestamps_measure = enable_timestamps_measure;
- CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_INTERRUPTS_ENABLE, ¶ms), "Failed to enabled vdma interrupts");
+ CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_ENABLE_CHANNELS, ¶ms), "Failed to enable vdma channels");
return HAILO_SUCCESS;
}
-hailo_status HailoRTDriver::vdma_interrupts_disable(const ChannelsBitmap &channels_bitmap)
+hailo_status HailoRTDriver::vdma_disable_channels(const ChannelsBitmap &channels_bitmap)
{
CHECK(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given");
- hailo_vdma_interrupts_disable_params params{};
+ hailo_vdma_disable_channels_params params{};
std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine);
- CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_INTERRUPTS_DISABLE, ¶ms), "Failed to disable vdma interrupts");
+ CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_DISABLE_CHANNELS, ¶ms), "Failed to disable vdma channels");
return HAILO_SUCCESS;
}
irq.channels_irq_data[i].channel_id.engine_index = engine_index;
irq.channels_irq_data[i].channel_id.channel_index = channel_index;
irq.channels_irq_data[i].is_active = params.irq_data[i].is_active;
- irq.channels_irq_data[i].desc_num_processed = params.irq_data[i].host_num_processed;
+ irq.channels_irq_data[i].transfers_completed = params.irq_data[i].transfers_completed;
irq.channels_irq_data[i].host_error = params.irq_data[i].host_error;
irq.channels_irq_data[i].device_error = params.irq_data[i].device_error;
irq.channels_irq_data[i].validation_success = params.irq_data[i].validation_success;
return HAILO_SUCCESS;
}
-Expected<HailoRTDriver::VdmaBufferHandle> HailoRTDriver::vdma_buffer_map(void *user_address, size_t required_size,
- DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle) {
+Expected<HailoRTDriver::VdmaBufferHandle> HailoRTDriver::vdma_buffer_map_dmabuf(int dmabuf_fd, size_t required_size, DmaDirection data_direction,
+ DmaBufferType buffer_type)
+{
+ CHECK_AS_EXPECTED (DmaBufferType::DMABUF_BUFFER == buffer_type, HAILO_INVALID_ARGUMENT,
+ "Error, Invalid buffer type given, buffer type {}", buffer_type);
+
+ return vdma_buffer_map(dmabuf_fd, required_size, data_direction, INVALID_MAPPED_BUFFER_DRIVER_IDENTIFIER,
+ buffer_type);
+}
+
+Expected<HailoRTDriver::VdmaBufferHandle> HailoRTDriver::vdma_buffer_map(uintptr_t user_address, size_t required_size,
+ DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle,
+ DmaBufferType buffer_type) {
std::unique_lock<std::mutex> mapping_lock(m_mapped_buffer_lock);
auto mapped_buffer = std::find_if(m_mapped_buffer.begin(), m_mapped_buffer.end(),
return Expected<VdmaBufferHandle>(mapped_buffer->handle);
} else {
// Buffer not mapped, map it now
- auto handle = vdma_buffer_map_ioctl(user_address, required_size, data_direction, driver_buff_handle);
+ auto handle = vdma_buffer_map_ioctl(user_address, required_size, data_direction,
+ driver_buff_handle, buffer_type);
CHECK_EXPECTED(handle);
const auto mapping_count = 1;
return HAILO_SUCCESS;
}
-hailo_status HailoRTDriver::vdma_buffer_unmap(void *user_address, size_t size, DmaDirection data_direction)
+hailo_status HailoRTDriver::vdma_buffer_unmap(uintptr_t user_address, size_t size, DmaDirection data_direction)
{
std::unique_lock<std::mutex> mapping_lock(m_mapped_buffer_lock);
auto mapped_buffer = std::find_if(m_mapped_buffer.begin(), m_mapped_buffer.end(),
#endif
}
-Expected<DescriptorsListInfo> HailoRTDriver::descriptors_list_create(size_t desc_count, uint16_t desc_page_size,
- bool is_circular)
-{
- uintptr_t desc_handle = INVALID_DRIVER_HANDLE_VALUE;
- uint64_t dma_address = 0;
- TRY(std::tie(desc_handle, dma_address),
- descriptors_list_create_ioctl(desc_count, desc_page_size, is_circular));
-
- auto user_address = descriptors_list_create_mmap(desc_handle, desc_count);
- if (!user_address) {
- auto status = descriptors_list_release_ioctl(desc_handle);
- if (HAILO_SUCCESS != status) {
- LOGGER__ERROR("Failed releasing descriptors list, status {}", status);
- // continue
- }
- return make_unexpected(user_address.status());
- }
-
- return DescriptorsListInfo{desc_handle, dma_address, desc_count, user_address.release()};
-}
-
-hailo_status HailoRTDriver::descriptors_list_release(const DescriptorsListInfo &descriptors_list_info)
+hailo_status HailoRTDriver::descriptors_list_program(uintptr_t desc_handle, VdmaBufferHandle buffer_handle,
+ size_t buffer_size, size_t buffer_offset, uint8_t channel_index, uint32_t starting_desc, bool should_bind,
+ InterruptsDomain last_desc_interrupts)
{
- hailo_status status = HAILO_SUCCESS;
-
- auto unmap_status = descriptors_list_create_munmap(descriptors_list_info.user_address, descriptors_list_info.desc_count);
- if (HAILO_SUCCESS != unmap_status) {
- LOGGER__ERROR("Descriptors list unmap failed with {}", unmap_status);
- status = unmap_status;
- // continue
- }
+ hailo_desc_list_program_params params{};
+ params.buffer_handle = buffer_handle;
+ params.buffer_size = buffer_size;
+ params.buffer_offset = buffer_offset;
+ params.desc_handle = desc_handle;
+ params.channel_index = channel_index;
+ params.starting_desc = starting_desc;
- auto release_status = descriptors_list_release_ioctl(descriptors_list_info.handle);
- if (HAILO_SUCCESS != release_status) {
- LOGGER__ERROR("Descriptors list release status failed with {}", release_status);
- status = release_status;
- // continue
- }
+ params.should_bind = should_bind;
+ params.last_interrupts_domain = (hailo_vdma_interrupts_domain)last_desc_interrupts;
- return status;
-}
+#ifdef NDEBUG
+ params.is_debug = false;
+#else
+ params.is_debug = true;
+#endif
-hailo_status HailoRTDriver::descriptors_list_bind_vdma_buffer(uintptr_t desc_handle, VdmaBufferHandle buffer_handle,
- size_t buffer_size, size_t buffer_offset, uint8_t channel_index, uint32_t starting_desc)
-{
- hailo_desc_list_bind_vdma_buffer_params config_info{};
- config_info.buffer_handle = buffer_handle;
- config_info.buffer_size = buffer_size;
- config_info.buffer_offset = buffer_offset;
- config_info.desc_handle = desc_handle;
- config_info.channel_index = channel_index;
- config_info.starting_desc = starting_desc;
-
- CHECK_IOCTL_RESULT(run_ioctl(HAILO_DESC_LIST_BIND_VDMA_BUFFER, &config_info), "Failed bind buffer to desc list");
+ CHECK_IOCTL_RESULT(run_ioctl(HAILO_DESC_LIST_PROGRAM, ¶ms), "Failed bind buffer to desc list");
return HAILO_SUCCESS;
}
params.is_debug = true;
#endif
- CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_LAUNCH_TRANSFER, ¶ms), "Failed launch transfer");
+ int err = run_ioctl(HAILO_VDMA_LAUNCH_TRANSFER, ¶ms);
+ if ((0 != err) && (-ECONNRESET == params.launch_transfer_status)) {
+ return make_unexpected(HAILO_STREAM_ABORT);
+ }
+ CHECK(0 == err, HAILO_DRIVER_FAIL, "Failed launch transfer errno: {}", err);
+
return Expected<uint32_t>(params.descs_programed);
}
return params.in_use ? HAILO_DEVICE_IN_USE : HAILO_SUCCESS;
}
+Expected<std::pair<vdma::ChannelId, vdma::ChannelId>> HailoRTDriver::soc_connect(uintptr_t input_buffer_desc_handle,
+ uintptr_t output_buffer_desc_handle)
+{
+ hailo_soc_connect_params params{};
+ params.input_desc_handle = input_buffer_desc_handle;
+ params.output_desc_handle = output_buffer_desc_handle;
+ CHECK_IOCTL_RESULT(run_ioctl(HAILO_SOC_CONNECT, ¶ms), "Failed soc_connect");
+ vdma::ChannelId input_channel{0, params.input_channel_index};
+ vdma::ChannelId output_channel{0, params.output_channel_index};
+ return std::make_pair(input_channel, output_channel);
+}
+
+Expected<std::pair<vdma::ChannelId, vdma::ChannelId>> HailoRTDriver::pci_ep_accept(uintptr_t input_buffer_desc_handle, uintptr_t output_buffer_desc_handle)
+{
+ hailo_pci_ep_accept_params params{};
+ params.input_desc_handle = input_buffer_desc_handle;
+ params.output_desc_handle = output_buffer_desc_handle;
+ CHECK_IOCTL_RESULT(run_ioctl(HAILO_PCI_EP_ACCEPT, ¶ms), "Failed pci_ep accept");
+ vdma::ChannelId input_channel{0, params.input_channel_index};
+ vdma::ChannelId output_channel{0, params.output_channel_index};
+ return std::make_pair(input_channel, output_channel);
+}
+
+hailo_status HailoRTDriver::close_connection(vdma::ChannelId input_channel, vdma::ChannelId output_channel,
+ PcieSessionType session_type)
+{
+ if (PcieSessionType::SERVER == session_type) {
+ hailo_pci_ep_close_params params{};
+ params.input_channel_index = input_channel.channel_index;
+ params.output_channel_index = output_channel.channel_index;
+ CHECK_IOCTL_RESULT(run_ioctl(HAILO_PCI_EP_CLOSE, ¶ms), "Failed pci_ep_close");
+ return HAILO_SUCCESS;
+ } else if (PcieSessionType::CLIENT == session_type) {
+ hailo_soc_close_params params{};
+ params.input_channel_index = input_channel.channel_index;
+ params.output_channel_index = output_channel.channel_index;
+ CHECK_IOCTL_RESULT(run_ioctl(HAILO_SOC_CLOSE, ¶ms), "Failed soc_close");
+ return HAILO_SUCCESS;
+ } else {
+ LOGGER__ERROR("close_connection not supported with session type {}", session_type);
+ return HAILO_NOT_SUPPORTED;
+ }
+}
+
#if defined(__linux__)
static bool is_blocking_ioctl(unsigned long request)
{
}
#if defined(__linux__) || defined(_WIN32)
-Expected<HailoRTDriver::VdmaBufferHandle> HailoRTDriver::vdma_buffer_map_ioctl(void *user_address, size_t required_size,
- DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle)
+Expected<HailoRTDriver::VdmaBufferHandle> HailoRTDriver::vdma_buffer_map_ioctl(uintptr_t user_address, size_t required_size,
+ DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle,
+ DmaBufferType buffer_type)
{
hailo_vdma_buffer_map_params map_user_buffer_info{};
map_user_buffer_info.user_address = user_address;
map_user_buffer_info.size = required_size;
map_user_buffer_info.data_direction = direction_to_dma_data_direction(data_direction);
+ map_user_buffer_info.buffer_type = driver_dma_buffer_type_to_dma_buffer_type(buffer_type);
map_user_buffer_info.allocated_buffer_handle = driver_buff_handle;
map_user_buffer_info.mapped_handle = 0;
return std::move(map_user_buffer_info.mapped_handle);
}
#elif defined(__QNX__)
-Expected<HailoRTDriver::VdmaBufferHandle> HailoRTDriver::vdma_buffer_map_ioctl(void *user_address, size_t required_size,
- DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle)
+Expected<HailoRTDriver::VdmaBufferHandle> HailoRTDriver::vdma_buffer_map_ioctl(uintptr_t user_address, size_t required_size,
+ DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle,
+ DmaBufferType buffer_type)
{
// Mapping is done by the driver_buff_handle (shm file descriptor), and not by address.
(void)user_address;
.shared_memory_handle = shm_handle,
.size = required_size,
.data_direction = direction_to_dma_data_direction(data_direction),
+ .buffer_type = driver_dma_buffer_type_to_dma_buffer_type(buffer_type),
.allocated_buffer_handle = INVALID_DRIVER_HANDLE_VALUE,
.mapped_handle = 0
};
return HAILO_SUCCESS;
}
-Expected<std::pair<uintptr_t, uint64_t>> HailoRTDriver::descriptors_list_create_ioctl(size_t desc_count,
+Expected<DescriptorsListInfo> HailoRTDriver::descriptors_list_create(size_t desc_count,
uint16_t desc_page_size, bool is_circular)
{
CHECK(is_powerof2(desc_page_size), HAILO_INVALID_ARGUMENT, "Invalid desc page size {}", desc_page_size);
CHECK_IOCTL_RESULT(run_ioctl(HAILO_DESC_LIST_CREATE, &create_desc_info), "Failed create desc list");
- return std::make_pair(create_desc_info.desc_handle, create_desc_info.dma_address);
+ return DescriptorsListInfo{create_desc_info.desc_handle, create_desc_info.dma_address};
}
-hailo_status HailoRTDriver::descriptors_list_release_ioctl(uintptr_t desc_handle)
+hailo_status HailoRTDriver::descriptors_list_release(const DescriptorsListInfo &desc_info)
{
struct hailo_desc_list_release_params params{};
- params.desc_handle = desc_handle;
+ params.desc_handle = desc_info.handle;
CHECK_IOCTL_RESULT(run_ioctl(HAILO_DESC_LIST_RELEASE, ¶ms), "Failed release desc list");
return HAILO_SUCCESS;
}
-#if defined(__linux__)
-Expected<void *> HailoRTDriver::descriptors_list_create_mmap(uintptr_t desc_handle, size_t desc_count)
-{
- // We lock m_driver_lock before calling mmap. Read m_driver_lock doc in the header
- std::unique_lock<std::mutex> lock(m_driver_lock);
-
- const size_t buffer_size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR;
- void *address = mmap(nullptr, buffer_size, PROT_WRITE | PROT_READ, MAP_SHARED, m_fd, (off_t)desc_handle);
- if (MAP_FAILED == address) {
- LOGGER__ERROR("Failed to map descriptors list buffer with errno: {}", errno);
- return make_unexpected(HAILO_DRIVER_FAIL);
- }
- return address;
-}
-
-hailo_status HailoRTDriver::descriptors_list_create_munmap(void *address, size_t desc_count)
-{
- const size_t buffer_size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR;
- if (0 != munmap(address, buffer_size)) {
- LOGGER__ERROR("munmap of address {}, length: {} failed with errno: {}", address, buffer_size, errno);
- return HAILO_DRIVER_FAIL;
- }
- return HAILO_SUCCESS;
-}
-
-#elif defined(__QNX__)
-
-Expected<void *> HailoRTDriver::descriptors_list_create_mmap(uintptr_t desc_handle, size_t desc_count)
-{
- const size_t buffer_size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR;
- struct hailo_non_linux_desc_list_mmap_params map_vdma_list_params {
- .desc_handle = desc_handle,
- .size = buffer_size,
- .user_address = nullptr,
- };
-
- CHECK_IOCTL_RESULT(run_ioctl(HAILO_NON_LINUX_DESC_LIST_MMAP, &map_vdma_list_params), "Failed mmap descriptors list");
-
- void *address = mmap(nullptr, buffer_size, PROT_WRITE | PROT_READ | PROT_NOCACHE, MAP_SHARED | MAP_PHYS, NOFD,
- (off_t)map_vdma_list_params.user_address);
- CHECK_AS_EXPECTED(MAP_FAILED != address, HAILO_INTERNAL_FAILURE, "Failed to mmap buffer fd with errno:{}", errno);
-
- return address;
-}
-
-hailo_status HailoRTDriver::descriptors_list_create_munmap(void *address, size_t desc_count)
-{
- const size_t buffer_size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR;
- if (0 != munmap(address, buffer_size)) {
- LOGGER__ERROR("munmap of address {}, length: {} failed with errno: {}", address, buffer_size, errno);
- return HAILO_DRIVER_FAIL;
- }
- return HAILO_SUCCESS;
-}
-
-#elif defined(_WIN32)
-Expected<void *> HailoRTDriver::descriptors_list_create_mmap(uintptr_t desc_handle, size_t desc_count)
-{
- hailo_non_linux_desc_list_mmap_params params{};
- params.desc_handle = desc_handle;
- params.size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR;
- CHECK_IOCTL_RESULT(run_ioctl(HAILO_NON_LINUX_DESC_LIST_MMAP, ¶ms), "Failed mmap desc list");
- void *user_address = params.user_address;
- return user_address;
-}
-
-hailo_status HailoRTDriver::descriptors_list_create_munmap(void *, size_t )
-{
- // On windows, the unmap is done on the release ioctl
- return HAILO_SUCCESS;
-}
-#else
-#error "unsupported platform!"
-#endif
-
#if defined(__linux__)
Expected<std::pair<uintptr_t, uint64_t>> HailoRTDriver::continous_buffer_alloc_ioctl(size_t size)
constexpr uint8_t MIN_D2H_CHANNEL_INDEX = MAX_H2D_CHANNEL_INDEX + 1;
constexpr uint8_t MAX_D2H_CHANNEL_INDEX = 31;
-constexpr size_t SIZE_OF_SINGLE_DESCRIPTOR = 0x10;
-
// NOTE: don't change members from this struct without updating all code using it (platform specific)
struct ChannelInterruptTimestamp {
std::chrono::nanoseconds timestamp;
struct ChannelIrqData {
vdma::ChannelId channel_id;
bool is_active;
- uint16_t desc_num_processed;
+ uint8_t transfers_completed;
uint8_t host_error;
uint8_t device_error;
bool validation_success;
struct DescriptorsListInfo {
uintptr_t handle; // Unique identifier for the driver.
uint64_t dma_address;
- size_t desc_count;
- void *user_address;
};
struct ContinousBufferInfo {
{
public:
+ enum class AcceleratorType {
+ NNC_ACCELERATOR,
+ SOC_ACCELERATOR,
+ ACC_TYPE_MAX_VALUE
+ };
+
struct DeviceInfo {
std::string dev_path;
std::string device_id;
+ enum AcceleratorType accelerator_type;
};
enum class DmaDirection {
BOTH
};
+ enum class DmaBufferType {
+ USER_PTR_BUFFER = 0,
+ DMABUF_BUFFER
+ };
+
enum class DmaSyncDirection {
TO_HOST = 0,
TO_DEVICE
enum class DmaType {
PCIE,
- DRAM
+ DRAM,
+ PCIE_EP
};
enum class MemoryType {
DMA_ENGINE0,
DMA_ENGINE1,
DMA_ENGINE2,
+
+ // PCIe EP driver memories
+ PCIE_EP_CONFIG,
+ PCIE_EP_BRIDGE,
+ };
+
+ enum class PcieSessionType {
+ CLIENT, // (soc)
+ SERVER // (A53 - pci_ep)
};
using VdmaBufferHandle = size_t;
- static Expected<std::unique_ptr<HailoRTDriver>> create(const DeviceInfo &device_info);
+ static Expected<std::unique_ptr<HailoRTDriver>> create(const std::string &device_id, const std::string &dev_path);
- ~HailoRTDriver();
+ static Expected<std::unique_ptr<HailoRTDriver>> create_pcie(const std::string &device_id);
+
+ static Expected<std::unique_ptr<HailoRTDriver>> create_integrated_nnc();
+ static bool is_integrated_nnc_loaded();
+
+ static Expected<std::unique_ptr<HailoRTDriver>> create_pcie_ep();
+ static bool is_pcie_ep_loaded();
static Expected<std::vector<DeviceInfo>> scan_devices();
+ static Expected<std::vector<DeviceInfo>> scan_devices(AcceleratorType accelerator_type);
+
+ ~HailoRTDriver();
hailo_status read_memory(MemoryType memory_type, uint64_t address, void *buf, size_t size);
hailo_status write_memory(MemoryType memory_type, uint64_t address, const void *buf, size_t size);
- hailo_status vdma_interrupts_enable(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure);
- hailo_status vdma_interrupts_disable(const ChannelsBitmap &channel_id);
+ hailo_status vdma_enable_channels(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure);
+ hailo_status vdma_disable_channels(const ChannelsBitmap &channel_id);
Expected<IrqData> vdma_interrupts_wait(const ChannelsBitmap &channels_bitmap);
Expected<ChannelInterruptTimestampList> vdma_interrupts_read_timestamps(vdma::ChannelId channel_id);
hailo_status reset_nn_core();
+ /**
+ * Maps a dmabuf to physical memory.
+ *
+ * @param[in] dmabuf_fd - File decsriptor to the dmabuf.
+ * @param[in] required_size - size of dmabug we are mapping.
+ * @param[in] data_direction - direction is used for optimization.
+ * @param[in] buffer_type - buffer type must be DMABUF
+ */
+ Expected<VdmaBufferHandle> vdma_buffer_map_dmabuf(int dmabuf_fd, size_t required_size, DmaDirection data_direction,
+ DmaBufferType buffer_type);
+
/**
* Pins a page aligned user buffer to physical memory, creates an IOMMU mapping (pci_mag_sg).
* The buffer is used for streaming D2H or H2D using DMA.
* @param[in] driver_buff_handle - handle to driver allocated buffer - INVALID_DRIVER_BUFFER_HANDLE_VALUE in case
* of user allocated buffer
*/
- Expected<VdmaBufferHandle> vdma_buffer_map(void *user_address, size_t required_size, DmaDirection data_direction,
- const vdma_mapped_buffer_driver_identifier &driver_buff_handle);
+ Expected<VdmaBufferHandle> vdma_buffer_map(uintptr_t user_address, size_t required_size, DmaDirection data_direction,
+ const vdma_mapped_buffer_driver_identifier &driver_buff_handle, DmaBufferType buffer_type);
/**
* Unmaps user buffer mapped using HailoRTDriver::map_buffer.
*/
hailo_status vdma_buffer_unmap(VdmaBufferHandle handle);
- hailo_status vdma_buffer_unmap(void *user_address, size_t size, DmaDirection data_direction);
+ hailo_status vdma_buffer_unmap(uintptr_t user_address, size_t size, DmaDirection data_direction);
hailo_status vdma_buffer_sync(VdmaBufferHandle buffer, DmaSyncDirection sync_direction, size_t offset, size_t count);
hailo_status descriptors_list_release(const DescriptorsListInfo &descriptors_list_info);
/**
- * Configure vdma channel descriptors to point to the given user address.
+ * Program the given descriptors list to point to the given buffer.
*/
- hailo_status descriptors_list_bind_vdma_buffer(uintptr_t desc_handle, VdmaBufferHandle buffer_handle,
+ hailo_status descriptors_list_program(uintptr_t desc_handle, VdmaBufferHandle buffer_handle,
size_t buffer_size, size_t buffer_offset, uint8_t channel_index,
- uint32_t starting_desc);
+ uint32_t starting_desc, bool should_bind, InterruptsDomain last_desc_interrupts);
struct TransferBuffer {
VdmaBufferHandle buffer_handle;
*/
hailo_status mark_as_used();
+ Expected<std::pair<vdma::ChannelId, vdma::ChannelId>> soc_connect(uintptr_t input_buffer_desc_handle,
+ uintptr_t output_buffer_desc_handle);
+
+ Expected<std::pair<vdma::ChannelId, vdma::ChannelId>> pci_ep_accept(uintptr_t input_buffer_desc_handle,
+ uintptr_t output_buffer_desc_handle);
+
+ hailo_status close_connection(vdma::ChannelId input_channel, vdma::ChannelId output_channel,
+ PcieSessionType session_type);
+
const std::string &device_id() const
{
- return m_device_info.device_id;
+ return m_device_id;
}
inline DmaType dma_type() const
static const uint8_t INVALID_VDMA_CHANNEL_INDEX;
static const vdma_mapped_buffer_driver_identifier INVALID_MAPPED_BUFFER_DRIVER_IDENTIFIER;
+ static constexpr const char *INTEGRATED_NNC_DEVICE_ID = "[integrated]";
+ static constexpr const char *PCIE_EP_DEVICE_ID = "[pci_ep]";
+
private:
template<typename PointerType>
int run_ioctl(uint32_t ioctl_code, PointerType param);
hailo_status read_memory_ioctl(MemoryType memory_type, uint64_t address, void *buf, size_t size);
hailo_status write_memory_ioctl(MemoryType memory_type, uint64_t address, const void *buf, size_t size);
- Expected<VdmaBufferHandle> vdma_buffer_map_ioctl(void *user_address, size_t required_size,
- DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle);
+ Expected<VdmaBufferHandle> vdma_buffer_map_ioctl(uintptr_t user_address, size_t required_size,
+ DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle,
+ DmaBufferType buffer_type);
hailo_status vdma_buffer_unmap_ioctl(VdmaBufferHandle handle);
- Expected<std::pair<uintptr_t, uint64_t>> descriptors_list_create_ioctl(size_t desc_count, uint16_t desc_page_size,
- bool is_circular);
- hailo_status descriptors_list_release_ioctl(uintptr_t desc_handle);
- Expected<void *> descriptors_list_create_mmap(uintptr_t desc_handle, size_t desc_count);
- hailo_status descriptors_list_create_munmap(void *address, size_t desc_count);
-
Expected<std::pair<uintptr_t, uint64_t>> continous_buffer_alloc_ioctl(size_t size);
hailo_status continous_buffer_free_ioctl(uintptr_t desc_handle);
Expected<void *> continous_buffer_mmap(uintptr_t desc_handle, size_t size);
hailo_status continous_buffer_munmap(void *address, size_t size);
- HailoRTDriver(const DeviceInfo &device_info, FileDescriptor &&fd, hailo_status &status);
+ HailoRTDriver(const std::string &device_id, FileDescriptor &&fd, hailo_status &status);
bool is_valid_channel_id(const vdma::ChannelId &channel_id);
bool is_valid_channels_bitmap(const ChannelsBitmap &bitmap)
FileDescriptor m_fd;
DeviceInfo m_device_info;
+ std::string m_device_id;
uint16_t m_desc_max_page_size;
DmaType m_dma_type;
bool m_allocate_driver_buffer;
// TODO HRT-11937: when ioctl is combined, move caching to driver
struct MappedBufferInfo {
VdmaBufferHandle handle;
- void *address;
+ uintptr_t address;
DmaDirection direction;
size_t size;
vdma_mapped_buffer_driver_identifier driver_buff_handle;
{
Expected<FileDescriptor> open_device_file(const std::string &path);
-Expected<std::vector<std::string>> list_devices();
Expected<HailoRTDriver::DeviceInfo> query_device_info(const std::string &device_name);
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_nnc_devices();
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_soc_devices();
#ifndef _WIN32
#define HAILO_CLASS_PATH ("/sys/class/hailo_chardev")
#define HAILO_BOARD_LOCATION_FILENAME ("board_location")
+#define HAILO_BOARD_ACCELERATOR_TYPE_FILENAME ("accelerator_type")
Expected<FileDescriptor> open_device_file(const std::string &path)
{
return devices;
}
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_devices()
+{
+ TRY_V(auto devices, list_devices(), "Failed listing pcie devices");
+
+ std::vector<HailoRTDriver::DeviceInfo> devices_info;
+ for (const auto &device_name : devices) {
+ TRY(auto device_info, query_device_info(device_name), "Failed parsing device info for {}", device_name);
+ devices_info.push_back(device_info);
+ }
+
+ return devices_info;
+}
+
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_soc_devices()
+{
+ TRY(auto all_devices_info, scan_devices());
+
+ // TODO- HRT-14078: change to be based on different classes
+ std::vector<HailoRTDriver::DeviceInfo> soc_devices_info;
+ for (const auto &device_info : all_devices_info) {
+ if (HailoRTDriver::AcceleratorType::SOC_ACCELERATOR == device_info.accelerator_type) {
+ soc_devices_info.push_back(device_info);
+ }
+ }
+
+ return soc_devices_info;
+}
+
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_nnc_devices()
+{
+ TRY(auto all_devices_info, scan_devices());
+
+ // TODO- HRT-14078: change to be based on different classes
+ std::vector<HailoRTDriver::DeviceInfo> nnc_devices_info;
+ for (const auto &device_info : all_devices_info) {
+ if (HailoRTDriver::AcceleratorType::NNC_ACCELERATOR == device_info.accelerator_type) {
+ nnc_devices_info.push_back(device_info);
+ }
+ }
+
+ return nnc_devices_info;
+}
+
+Expected<std::string> get_line_from_file(const std::string &file_path)
+{
+ std::ifstream file(file_path);
+ CHECK_AS_EXPECTED(file.good(), HAILO_DRIVER_FAIL, "Failed open {}", file_path);
+
+ std::string line;
+ std::getline(file, line);
+ CHECK_AS_EXPECTED(file.eof(), HAILO_DRIVER_FAIL, "Failed read {}", file_path);
+
+ return line;
+}
+
Expected<HailoRTDriver::DeviceInfo> query_device_info(const std::string &device_name)
{
const std::string device_id_path = std::string(HAILO_CLASS_PATH) + "/" +
device_name + "/" + HAILO_BOARD_LOCATION_FILENAME;
- std::ifstream device_id_file(device_id_path);
- CHECK_AS_EXPECTED(device_id_file.good(), HAILO_DRIVER_FAIL, "Failed open {}", device_id_path);
+ TRY(auto device_id , get_line_from_file(device_id_path));
+
+ const std::string accelerator_type_path = std::string(HAILO_CLASS_PATH) + "/" +
+ device_name + "/" + HAILO_BOARD_ACCELERATOR_TYPE_FILENAME;
+ TRY(auto accelerator_type_s, get_line_from_file(accelerator_type_path));
- std::string device_id;
- std::getline(device_id_file, device_id);
- CHECK_AS_EXPECTED(device_id_file.eof(), HAILO_DRIVER_FAIL, "Failed read {}", device_id_path);
+ int accelerator_type = std::stoi(accelerator_type_s);
HailoRTDriver::DeviceInfo device_info = {};
device_info.dev_path = std::string("/dev/") + device_name;
device_info.device_id = device_id;
+ device_info.accelerator_type = static_cast<HailoRTDriver::AcceleratorType>(accelerator_type);
return device_info;
}
return FileDescriptor(handle);
}
-Expected<std::vector<std::string>> list_devices()
-{
- GUID guid = GUID_DEVINTERFACE_HailoKM;
+Expected<std::vector<std::string>> list_devices(GUID guid)
+{
ULONG len = 0;
CONFIGRET cr = CM_Get_Device_Interface_List_SizeA(
&len,
return names;
}
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_devices(GUID guid)
+{
+ TRY (auto names, list_devices(guid), "Failed listing pcie devices");
+
+ std::vector<HailoRTDriver::DeviceInfo> devices_info;
+ for (const auto &name : names) {
+ auto device_info = query_device_info(name);
+ CHECK_EXPECTED(device_info, "Failed parsing device info for {}", name);
+ if (GUID_DEVINTERFACE_HailoKM_NNC == guid) {
+ device_info->accelerator_type = HailoRTDriver::AcceleratorType::NNC_ACCELERATOR;
+ } else if (GUID_DEVINTERFACE_HailoKM_SOC == guid) {
+ device_info->accelerator_type = HailoRTDriver::AcceleratorType::SOC_ACCELERATOR;
+ }
+ devices_info.push_back(device_info.release());
+ }
+
+ return devices_info;
+}
+
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_soc_devices()
+{
+ GUID guid = GUID_DEVINTERFACE_HailoKM_SOC;
+ return scan_devices(guid);
+}
+
+Expected<std::vector<HailoRTDriver::DeviceInfo>> scan_nnc_devices()
+{
+ GUID guid = GUID_DEVINTERFACE_HailoKM_NNC;
+ return scan_devices(guid);
+}
+
static Expected<uint32_t> parse_uint32_property(const std::wstring &dev_interface,
const DEVPROPKEY* key)
{
}
COMPATIBLE_PARAM_CAST(hailo_memory_transfer_params, MemoryTransfer);
-COMPATIBLE_PARAM_CAST(hailo_vdma_interrupts_enable_params, VdmaInterruptsEnable)
-COMPATIBLE_PARAM_CAST(hailo_vdma_interrupts_disable_params, VdmaInterruptsDisable)
+COMPATIBLE_PARAM_CAST(hailo_vdma_enable_channels_params, VdmaEnableChannels)
+COMPATIBLE_PARAM_CAST(hailo_vdma_disable_channels_params, VdmaDisableChannels)
COMPATIBLE_PARAM_CAST(hailo_vdma_interrupts_read_timestamp_params, VdmaInterruptsReadTimestamps)
COMPATIBLE_PARAM_CAST(hailo_vdma_interrupts_wait_params, VdmaInterruptsWait)
COMPATIBLE_PARAM_CAST(hailo_vdma_buffer_sync_params, VdmaBufferSync)
COMPATIBLE_PARAM_CAST(hailo_vdma_buffer_unmap_params, VdmaBufferUnmap)
COMPATIBLE_PARAM_CAST(hailo_desc_list_create_params, DescListCreate)
COMPATIBLE_PARAM_CAST(hailo_desc_list_release_params, DescListReleaseParam)
-COMPATIBLE_PARAM_CAST(hailo_desc_list_bind_vdma_buffer_params, DescListBind)
+COMPATIBLE_PARAM_CAST(hailo_desc_list_program_params, DescListProgram)
COMPATIBLE_PARAM_CAST(hailo_d2h_notification, D2HNotification)
COMPATIBLE_PARAM_CAST(hailo_device_properties, DeviceProperties)
COMPATIBLE_PARAM_CAST(hailo_driver_info, DriverInfo)
-COMPATIBLE_PARAM_CAST(hailo_non_linux_desc_list_mmap_params, DescListMmap)
COMPATIBLE_PARAM_CAST(hailo_read_log_params, ReadLog)
COMPATIBLE_PARAM_CAST(hailo_mark_as_in_use_params, MarkAsInUse)
COMPATIBLE_PARAM_CAST(hailo_vdma_launch_transfer_params, LaunchTransfer)
+COMPATIBLE_PARAM_CAST(hailo_soc_connect_params, ConnectParams)
+COMPATIBLE_PARAM_CAST(hailo_soc_close_params, SocCloseParams)
+COMPATIBLE_PARAM_CAST(hailo_pci_ep_accept_params, AcceptParams)
+COMPATIBLE_PARAM_CAST(hailo_pci_ep_close_params, PciEpCloseParams)
// Special handle for nullptr_t. This case occurs when there is no parameters passed.
tCompatibleHailoIoctlData WindowsIoctlParamCast<nullptr_t>::to_compatible(nullptr_t data)
#include "md5.h"
#include <memory>
-static const std::string INTEGRATED_NNC_DRIVER_PATH = "/dev/hailo_integrated_nnc";
-
namespace hailort
{
+// 16 MB
+#define INTEGRATED_DEVICE_INFINITE_ACTION_LIST_POOL_SIZE (16777216)
+
bool IntegratedDevice::is_loaded()
{
-#if defined(_MSC_VER)
- // windows is not supported for core driver
- return false;
-#else
- return (access(INTEGRATED_NNC_DRIVER_PATH.c_str(), F_OK) == 0);
-#endif // defined(_MSC_VER)
+ return HailoRTDriver::is_integrated_nnc_loaded();
+}
+
+Expected<std::pair<void*, uint64_t>> IntegratedDevice::allocate_infinite_action_list_buffer(size_t size)
+{
+ CHECK_AS_EXPECTED(0 == (size % OsUtils::get_page_size()), HAILO_INVALID_ARGUMENT,
+ "Infinte action list buffer size must be a multiple of page size");
+ CHECK_AS_EXPECTED(m_device_infinite_action_list_pool_allocation_offset + size <= m_device_infinite_action_list_pool.size(),
+ HAILO_INVALID_ARGUMENT, "Buffer pool size is too small for requested infinte action list buffer");
+
+ auto user_addres = static_cast<void*>(reinterpret_cast<uint8_t*>(m_device_infinite_action_list_pool.user_address()) +
+ m_device_infinite_action_list_pool_allocation_offset);
+ auto dma_address = m_device_infinite_action_list_pool.dma_address() + m_device_infinite_action_list_pool_allocation_offset;
+
+ m_device_infinite_action_list_pool_allocation_offset += size;
+
+ return std::make_pair(user_addres, dma_address);
}
Expected<std::unique_ptr<IntegratedDevice>> IntegratedDevice::create()
{
hailo_status status = HAILO_UNINITIALIZED;
- const HailoRTDriver::DeviceInfo device_info {INTEGRATED_NNC_DRIVER_PATH, DEVICE_ID};
- auto driver = HailoRTDriver::create(device_info);
- CHECK_EXPECTED(driver, "Failed to initialize HailoRTDriver");
+ TRY(auto driver, HailoRTDriver::create_integrated_nnc());
+
+ // Create pool of memory for infinite action list so can all be in the LUT memory area
+ // TODO: remove this when infinite action list allocates from its own pool of CMA memory
+ TRY(auto infinite_action_list_pool, vdma::ContinuousBuffer::create(INTEGRATED_DEVICE_INFINITE_ACTION_LIST_POOL_SIZE,
+ *driver));
+
+ // Verify pool is in mapped range
+ CHECK_AS_EXPECTED(DDRActionListBufferBuilder::verify_dma_addr(infinite_action_list_pool), HAILO_INTERNAL_FAILURE,
+ "Failed to allocate continous buffer pool M4 mapped memory region");
+
- auto device = std::unique_ptr<IntegratedDevice>(new (std::nothrow) IntegratedDevice(driver.release(), status));
+ auto device = std::unique_ptr<IntegratedDevice>(new (std::nothrow) IntegratedDevice(std::move(driver),
+ std::move(infinite_action_list_pool), status));
CHECK_AS_EXPECTED((nullptr != device), HAILO_OUT_OF_HOST_MEMORY);
CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating IntegratedDevice");
return device;
}
-IntegratedDevice::IntegratedDevice(std::unique_ptr<HailoRTDriver> &&driver, hailo_status &status) :
- VdmaDevice::VdmaDevice(std::move(driver), Device::Type::INTEGRATED)
+IntegratedDevice::IntegratedDevice(std::unique_ptr<HailoRTDriver> &&driver, vdma::ContinuousBuffer &&pool,
+ hailo_status &status) :
+ VdmaDevice::VdmaDevice(std::move(driver), Device::Type::INTEGRATED, status),
+ m_device_infinite_action_list_pool(std::move(pool)),
+ m_device_infinite_action_list_pool_allocation_offset(0)
{
+ if (status != HAILO_SUCCESS) {
+ LOGGER__ERROR("Failed to create VdmaDevice");
+ return;
+ }
+
status = update_fw_state();
if (HAILO_SUCCESS != status) {
LOGGER__ERROR("update_fw_state() failed with status {}", status);
#include "hailo/hailort.h"
#include "vdma/vdma_device.hpp"
+#include "vdma/memory/continuous_buffer.hpp"
#include <memory>
}
}
- static constexpr const char *DEVICE_ID = "[integrated]";
+ static constexpr const char *DEVICE_ID = HailoRTDriver::INTEGRATED_NNC_DEVICE_ID;
+
+ Expected<std::pair<void*, uint64_t>> allocate_infinite_action_list_buffer(size_t size);
protected:
virtual hailo_status reset_impl(CONTROL_PROTOCOL__reset_type_t reset_type) override;
private:
- IntegratedDevice(std::unique_ptr<HailoRTDriver> &&driver, hailo_status &status);
+ IntegratedDevice(std::unique_ptr<HailoRTDriver> &&driver, vdma::ContinuousBuffer &&pool, hailo_status &status);
+
+ vdma::ContinuousBuffer m_device_infinite_action_list_pool;
+ size_t m_device_infinite_action_list_pool_allocation_offset;
};
return m_buffer_info.dma_address;
}
+void* ContinuousBuffer::user_address() const
+{
+ return m_buffer_info.user_address;
+}
+
hailo_status ContinuousBuffer::read(void *buf_dst, size_t count, size_t offset)
{
CHECK((count + offset) <= m_buffer_info.size, HAILO_INSUFFICIENT_BUFFER,
virtual hailo_status read(void *buf_dst, size_t count, size_t offset) override;
virtual hailo_status write(const void *buf_src, size_t count, size_t offset) override;
+ void *user_address() const;
uint64_t dma_address() const;
private:
ContinuousBuffer(HailoRTDriver &driver, const ContinousBufferInfo &buffer_info);
}
Expected<uint32_t> ContinuousEdgeLayer::program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain,
- size_t desc_offset)
+ size_t desc_offset, size_t buffer_offset, bool should_bind)
{
(void)last_desc_interrupts_domain;
(void)desc_offset;
+ (void)buffer_offset;
+ (void)should_bind;
// The descriptors in continuous mode are programmed by the hw, nothing to do here.
return descriptors_in_buffer(transfer_size);
virtual uint32_t descs_count() const override;
virtual Expected<uint32_t> program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain,
- size_t desc_offset) override;
+ size_t desc_offset, size_t buffer_offset = 0, bool should_bind = false) override;
private:
ContinuousEdgeLayer(std::shared_ptr<ContinuousBuffer> &&buffer, size_t size, size_t offset,
#include "utils.h"
-
-#define DESC_STATUS_REQ (1 << 0)
-#define DESC_STATUS_REQ_ERR (1 << 1)
-#define DESC_REQUREST_IRQ_PROCESSED (1 << 2)
-#define DESC_REQUREST_IRQ_ERR (1 << 3)
-
-#define PCIE_DMA_HOST_INTERRUPTS_BITMASK (1 << 5)
-#define PCIE_DMA_DEVICE_INTERRUPTS_BITMASK (1 << 4)
-
-#define DRAM_DMA_HOST_INTERRUPTS_BITMASK (1 << 4)
-#define DRAM_DMA_DEVICE_INTERRUPTS_BITMASK (1 << 5)
-
-#define DESC_PAGE_SIZE_SHIFT (8)
-#define DESC_PAGE_SIZE_MASK (0xFFFFFF00)
-#define DESC_IRQ_MASK (0x0000003C)
-
namespace hailort {
namespace vdma {
DescriptorList::DescriptorList(uint32_t desc_count, uint16_t desc_page_size, bool is_circular, HailoRTDriver &driver,
hailo_status &status) :
m_desc_list_info(),
+ m_desc_count(desc_count),
m_is_circular(is_circular),
m_driver(driver),
m_desc_page_size(desc_page_size)
DescriptorList::DescriptorList(DescriptorList &&other) noexcept :
m_desc_list_info(),
+ m_desc_count(other.m_desc_count),
m_is_circular(std::move(other.m_is_circular)),
m_driver(other.m_driver),
m_desc_page_size(other.m_desc_page_size)
{
m_desc_list_info.handle = std::exchange(other.m_desc_list_info.handle, 0);
m_desc_list_info.dma_address = std::exchange(other.m_desc_list_info.dma_address, 0);
- m_desc_list_info.desc_count = std::move(other.m_desc_list_info.desc_count);
- m_desc_list_info.user_address = std::exchange(other.m_desc_list_info.user_address, nullptr);
}
-hailo_status DescriptorList::configure_to_use_buffer(MappedBuffer& buffer, size_t buffer_size,
- size_t buffer_offset, ChannelId channel_id, uint32_t starting_desc)
+hailo_status DescriptorList::program(MappedBuffer& buffer, size_t buffer_size,
+ size_t buffer_offset, ChannelId channel_id, uint32_t starting_desc, bool should_bind /* = true */,
+ InterruptsDomain last_desc_interrupts /* = InterruptsDomain::NONE */)
{
const auto desc_list_capacity = m_desc_page_size * count();
CHECK(buffer_size <= desc_list_capacity, HAILO_INVALID_ARGUMENT,
"Can't bind a buffer larger than the descriptor list's capacity. Buffer size {}, descriptor list capacity {}",
buffer_size, desc_list_capacity);
- return m_driver.descriptors_list_bind_vdma_buffer(m_desc_list_info.handle, buffer.handle(), buffer_size,
- buffer_offset, channel_id.channel_index, starting_desc);
-}
-
-Expected<uint16_t> DescriptorList::program_last_descriptor(size_t transfer_size,
- InterruptsDomain last_desc_interrupts_domain, size_t desc_offset)
-{
- assert(transfer_size > 0);
- const auto required_descriptors = descriptors_in_buffer(transfer_size);
- // Required_descriptors + desc_offset can't reach m_count.
- if ((!m_is_circular) && ((required_descriptors + desc_offset) > count())){
- LOGGER__ERROR("Requested transfer size ({}) result in more descriptors than available ({})", transfer_size, count());
- return make_unexpected(HAILO_OUT_OF_DESCRIPTORS);
- }
-
- // Program last descriptor of the transfer size
- /* write residue page with the remaining buffer size*/
- auto resuide = transfer_size - (required_descriptors - 1) * m_desc_page_size;
- assert(IS_FIT_IN_UINT16(resuide));
- size_t last_desc = (desc_offset + required_descriptors - 1) % count();
- program_single_descriptor(last_desc, static_cast<uint16_t>(resuide), last_desc_interrupts_domain);
-
- return std::move(static_cast<uint16_t>(required_descriptors));
+ return m_driver.descriptors_list_program(m_desc_list_info.handle, buffer.handle(), buffer_size,
+ buffer_offset, channel_id.channel_index, starting_desc, should_bind, last_desc_interrupts);
}
uint32_t DescriptorList::descriptors_in_buffer(size_t buffer_size) const
return get_nearest_powerof_2(descs_count, MIN_SG_DESCS_COUNT);
}
-uint32_t DescriptorList::get_interrupts_bitmask(InterruptsDomain interrupts_domain)
-{
- uint32_t host_bitmask = 0;
- uint32_t device_bitmask = 0;
-
- switch (m_driver.dma_type()) {
- case HailoRTDriver::DmaType::PCIE:
- host_bitmask = PCIE_DMA_HOST_INTERRUPTS_BITMASK;
- device_bitmask = PCIE_DMA_DEVICE_INTERRUPTS_BITMASK;
- break;
- case HailoRTDriver::DmaType::DRAM:
- host_bitmask = DRAM_DMA_HOST_INTERRUPTS_BITMASK;
- device_bitmask = DRAM_DMA_DEVICE_INTERRUPTS_BITMASK;
- break;
- default:
- assert(false);
- }
-
- uint32_t bitmask = 0;
- if (host_interuptes_enabled(interrupts_domain)) {
- bitmask |= host_bitmask;
- }
- if (device_interuptes_enabled(interrupts_domain)) {
- bitmask |= device_bitmask;
- }
-
- return bitmask;
-}
-
-void DescriptorList::program_single_descriptor(size_t desc_index, uint16_t page_size,
- InterruptsDomain interrupts_domain)
-{
- auto &descriptor = (*this)[desc_index];
-
- // Update the descriptor's PAGE_SIZE field in the control register with the maximum size of the DMA page.
- // Make all edits to the local variable local_pagesize_desc_ctrl that is on the stack to save read/writes to DDR
- auto local_pagesize_desc_ctrl = static_cast<uint32_t>(page_size << DESC_PAGE_SIZE_SHIFT) & DESC_PAGE_SIZE_MASK;
-
- if (InterruptsDomain::NONE != interrupts_domain) {
- // Update the desc_control
- local_pagesize_desc_ctrl |= (DESC_REQUREST_IRQ_PROCESSED | DESC_REQUREST_IRQ_ERR |
- get_interrupts_bitmask(interrupts_domain));
-#ifndef NDEBUG
- local_pagesize_desc_ctrl |= (DESC_STATUS_REQ | DESC_STATUS_REQ_ERR);
-#endif
- }
-
- descriptor.PageSize_DescControl = local_pagesize_desc_ctrl;
-
-#ifndef NDEBUG
- // Clear status
- descriptor.RemainingPageSize_Status = 0;
-#endif
-}
-
} /* namespace vdma */
} /* namespace hailort */
#include "vdma/memory/mapped_buffer.hpp"
#include "vdma/driver/hailort_driver.hpp"
-#include "os/mmap_buffer.hpp"
-
namespace hailort {
namespace vdma {
static_assert(DEFAULT_SG_PAGE_SIZE > 0, "DEFAULT_SG_PAGE_SIZE must be larger then 0");
-static constexpr auto DESCRIPTOR_STATUS_MASK = 0xFF;
-static constexpr auto DESCRIPTOR_STATUS_DONE_BIT = 0;
-static constexpr auto DESCRIPTOR_STATUS_ERROR_BIT = 1;
-
-struct VdmaDescriptor
-{
- // Struct layout is taken from PLDA spec for vDMA, and cannot be changed.
- uint32_t PageSize_DescControl;
- uint32_t AddrL_rsvd_DataID;
- uint32_t AddrH;
- uint32_t RemainingPageSize_Status;
-
-#ifndef NDEBUG
- // Easy accessors (only on debug since we mark DESC_STATUS_REQ and DESC_STATUS_REQ_ERR are set only on debug).
- uint8_t status() const
- {
- return RemainingPageSize_Status & DESCRIPTOR_STATUS_MASK;
- }
-
- bool is_done() const
- {
- return is_bit_set(status(), DESCRIPTOR_STATUS_DONE_BIT);
- }
-
- bool is_error() const
- {
- return is_bit_set(status(), DESCRIPTOR_STATUS_ERROR_BIT);
- }
-#endif /* NDEBUG */
-};
-
-static_assert(SIZE_OF_SINGLE_DESCRIPTOR == sizeof(VdmaDescriptor), "Invalid size of descriptor");
-
-inline bool host_interuptes_enabled(InterruptsDomain interrupts_domain)
-{
- return 0 != (static_cast<uint32_t>(interrupts_domain) & static_cast<uint32_t>(InterruptsDomain::HOST));
-}
-
-inline bool device_interuptes_enabled(InterruptsDomain interrupts_domain)
-{
- return 0 != (static_cast<uint32_t>(interrupts_domain) & static_cast<uint32_t>(InterruptsDomain::DEVICE));
-}
-
class DescriptorList
{
public:
uint32_t count() const
{
- assert(m_desc_list_info.desc_count <= std::numeric_limits<uint32_t>::max());
- return static_cast<uint32_t>(m_desc_list_info.desc_count);
+ return m_desc_count;
}
uint64_t dma_address() const
return m_desc_list_info.dma_address;
}
- VdmaDescriptor& operator[](size_t i)
- {
- assert(i < count());
- return desc_list()[i];
- }
-
uint16_t desc_page_size() const
{
return m_desc_page_size;
return m_desc_list_info.handle;
}
- uint16_t max_transfers(uint32_t transfer_size)
+ uint16_t max_transfers(uint32_t transfer_size, bool include_bounce_buffer = false) const
{
+ const auto descs_needed = descriptors_in_buffer(transfer_size) + (include_bounce_buffer ? 1 : 0);
// We need to keep at least 1 free desc at all time.
- return static_cast<uint16_t>((count() - 1) / descriptors_in_buffer(transfer_size));
+ return static_cast<uint16_t>((count() - 1) / descs_needed);
}
// Map descriptors starting at offset to the start of buffer, wrapping around the descriptor list as needed
// On hailo8, we allow configuring buffer without specific channel index (default is INVALID_VDMA_CHANNEL_INDEX).
- hailo_status configure_to_use_buffer(MappedBuffer& buffer, size_t buffer_size, size_t buffer_offset,
- ChannelId channel_id, uint32_t starting_desc = 0);
- // All descritors are initialized to have size of m_desc_page_size - so all we do is set the last descritor for the
- // Interrupt - and then after transfer has finished clear the previously used first and last decsriptors.
- // This saves us write/ reads to the desscriptor list which is DMA memory.
- Expected<uint16_t> program_last_descriptor(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain,
- size_t desc_offset);
- void program_single_descriptor(size_t desc_index, uint16_t page_size, InterruptsDomain interrupts_domain);
+ hailo_status program(MappedBuffer& buffer, size_t buffer_size, size_t buffer_offset,
+ ChannelId channel_id, uint32_t starting_desc = 0, bool should_bind = true,
+ InterruptsDomain last_desc_interrupts = InterruptsDomain::NONE);
uint32_t descriptors_in_buffer(size_t buffer_size) const;
static uint32_t descriptors_in_buffer(size_t buffer_size, uint16_t desc_page_size);
DescriptorList(uint32_t desc_count, uint16_t desc_page_size, bool is_circular, HailoRTDriver &driver,
hailo_status &status);
- VdmaDescriptor *desc_list() { return reinterpret_cast<VdmaDescriptor*>(m_desc_list_info.user_address); }
-
- uint32_t get_interrupts_bitmask(InterruptsDomain interrupts_domain);
-
-
DescriptorsListInfo m_desc_list_info;
+ const uint32_t m_desc_count;
const bool m_is_circular;
HailoRTDriver &m_driver;
const uint16_t m_desc_page_size;
#if defined(_MSC_VER)
#include "os/windows/virtual_alloc_guard.hpp"
+#else
+#include <sys/mman.h>
#endif /* defined(_MSC_VER) */
#error "unsupported platform!"
#endif
+
} /* namespace vdma */
} /* namespace hailort */
#include "mapped_buffer.hpp"
-#include "vdma/vdma_device.hpp"
-
namespace hailort {
namespace vdma {
Expected<MappedBufferPtr> MappedBuffer::create_shared(DmaAbleBufferPtr buffer, HailoRTDriver &driver,
HailoRTDriver::DmaDirection data_direction)
{
- auto status = HAILO_UNINITIALIZED;
- auto result = make_shared_nothrow<MappedBuffer>(driver, buffer, data_direction, status);
- CHECK_SUCCESS_AS_EXPECTED(status);
+ TRY(auto buffer_handle, driver.vdma_buffer_map(reinterpret_cast<uintptr_t>(buffer->user_address()), buffer->size(), data_direction,
+ buffer->buffer_identifier(), HailoRTDriver::DmaBufferType::USER_PTR_BUFFER));
+
+ auto result = make_shared_nothrow<MappedBuffer>(driver, buffer, data_direction, buffer_handle);
CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY);
return result;
return create_shared(buffer.release(), driver, data_direction);
}
-MappedBuffer::MappedBuffer(HailoRTDriver &driver, DmaAbleBufferPtr buffer,
- HailoRTDriver::DmaDirection data_direction, hailo_status &status) :
- m_driver(driver),
- m_buffer(buffer),
- m_mapping_handle(HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE),
- m_data_direction(data_direction)
+Expected<MappedBufferPtr> MappedBuffer::create_shared_from_dmabuf(int dmabuf_fd, size_t size, HailoRTDriver &driver,
+ HailoRTDriver::DmaDirection data_direction)
{
- auto expected_handle = driver.vdma_buffer_map(m_buffer->user_address(), m_buffer->size(), m_data_direction,
- m_buffer->buffer_identifier());
- if (!expected_handle) {
- LOGGER__ERROR("Mapping address {} to dma failed", m_buffer->user_address());
- status = expected_handle.status();
- return;
- }
+ TRY(auto buffer_handle, driver.vdma_buffer_map_dmabuf(dmabuf_fd, size, data_direction,
+ HailoRTDriver::DmaBufferType::DMABUF_BUFFER));
- m_mapping_handle = expected_handle.release();
- status = HAILO_SUCCESS;
+ // TODO: if need user address for dmabuf use DmaBufDmaAbleBuffer
+ auto result = make_shared_nothrow<MappedBuffer>(driver, nullptr, data_direction, buffer_handle);
+ CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY);
+
+ return result;
}
+MappedBuffer::MappedBuffer(HailoRTDriver &driver, DmaAbleBufferPtr buffer, HailoRTDriver::DmaDirection data_direction,
+ HailoRTDriver::VdmaBufferHandle vdma_buffer_handle) :
+ m_driver(driver),
+ m_buffer(buffer),
+ m_mapping_handle(vdma_buffer_handle),
+ m_data_direction(data_direction)
+{}
+
MappedBuffer::~MappedBuffer()
{
if (HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE != m_mapping_handle) {
static Expected<MappedBufferPtr> create_shared_by_allocation(size_t size, HailoRTDriver &driver,
HailoRTDriver::DmaDirection data_direction);
+ // Receive an fd to a dmabuf object and map it in our driver and create MappedBuffer
+ static Expected<MappedBufferPtr> create_shared_from_dmabuf(int dmabuf_fd, size_t size, HailoRTDriver &driver,
+ HailoRTDriver::DmaDirection data_direction);
+
MappedBuffer(HailoRTDriver &driver, DmaAbleBufferPtr buffer, HailoRTDriver::DmaDirection data_direction,
- hailo_status &status);
+ HailoRTDriver::VdmaBufferHandle vdma_buffer_handle);
MappedBuffer(MappedBuffer &&other) noexcept;
MappedBuffer(const MappedBuffer &other) = delete;
MappedBuffer &operator=(const MappedBuffer &other) = delete;
assert((desc_count * desc_page_size) <= std::numeric_limits<uint32_t>::max());
- auto status = desc_list_exp->configure_to_use_buffer(*(buffer->get_mapped_buffer()), size , offset, channel_id);
+ auto status = desc_list_exp->program(*(buffer->get_mapped_buffer()), size , offset, channel_id);
CHECK_SUCCESS_AS_EXPECTED(status);
- return SgEdgeLayer(std::move(buffer), desc_list_exp.release(), size, offset);
+ return SgEdgeLayer(std::move(buffer), desc_list_exp.release(), size, offset, channel_id);
}
SgEdgeLayer::SgEdgeLayer(std::shared_ptr<SgBuffer> &&buffer, DescriptorList &&desc_list,
- size_t size, size_t offset) :
+ size_t size, size_t offset, ChannelId channel_id) :
VdmaEdgeLayer(std::move(buffer), size, offset),
- m_desc_list(std::move(desc_list))
+ m_desc_list(std::move(desc_list)),
+ m_channel_id(channel_id)
{}
uint64_t SgEdgeLayer::dma_address() const
}
Expected<uint32_t> SgEdgeLayer::program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain,
- size_t desc_offset)
+ size_t desc_offset, size_t buffer_offset, bool should_bind)
{
- return m_desc_list.program_last_descriptor(transfer_size, last_desc_interrupts_domain, desc_offset);
+ CHECK_SUCCESS(m_desc_list.program(*get_mapped_buffer(), transfer_size, buffer_offset, m_channel_id,
+ static_cast<uint32_t>(desc_offset), should_bind, last_desc_interrupts_domain));
+ return descriptors_in_buffer(transfer_size);
}
}
virtual uint32_t descs_count() const override;
virtual Expected<uint32_t> program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain,
- size_t desc_offset) override;
+ size_t desc_offset, size_t buffer_offset = 0, bool should_bind = false) override;
private:
SgEdgeLayer(std::shared_ptr<SgBuffer> &&buffer, DescriptorList &&desc_list,
- size_t size, size_t offset);
+ size_t size, size_t offset, ChannelId channel_id);
+
+ vdma::MappedBufferPtr get_mapped_buffer()
+ {
+ return std::static_pointer_cast<SgBuffer>(m_buffer)->get_mapped_buffer();
+ }
- // Initialization Dependency: The descriptor list points into the mapped buffer so it must be freed before it
- std::shared_ptr<SgBuffer> m_buffer;
DescriptorList m_desc_list;
+ const ChannelId m_channel_id;
};
} /* vdma */
return m_size;\r
}\r
\r
+ size_t backing_buffer_size() const\r
+ {\r
+ return m_buffer->size();\r
+ }\r
+\r
uint32_t descriptors_in_buffer(size_t buffer_size) const\r
{\r
assert(buffer_size < std::numeric_limits<uint32_t>::max());\r
hailo_status write(const void *buf_src, size_t count, size_t offset);\r
\r
virtual Expected<uint32_t> program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain,\r
- size_t desc_offset) = 0;\r
+ size_t desc_offset, size_t buffer_offset = 0, bool should_bind = false) = 0;\r
\r
CONTROL_PROTOCOL__host_buffer_info_t get_host_buffer_info(uint32_t transfer_size);\r
static CONTROL_PROTOCOL__host_buffer_info_t get_host_buffer_info(Type type, uint64_t dma_address,\r
auto device_info = find_device_info(pcie_device_info);
CHECK_EXPECTED(device_info);
- auto driver = HailoRTDriver::create(*device_info);
+ auto driver = HailoRTDriver::create(device_info->device_id, device_info->dev_path);
CHECK_EXPECTED(driver);
hailo_status status = HAILO_UNINITIALIZED;
}
PcieDevice::PcieDevice(std::unique_ptr<HailoRTDriver> &&driver, hailo_status &status) :
- VdmaDevice(std::move(driver), Device::Type::PCIE)
+ VdmaDevice(std::move(driver), Device::Type::PCIE, status)
{
+ if (status != HAILO_SUCCESS) {
+ LOGGER__ERROR("Failed to create VdmaDevice");
+ return;
+ }
+
if (m_driver->is_fw_loaded()) {
status = update_fw_state();
if (HAILO_SUCCESS != status) {
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file pcie_session.cpp
+ **/
+
+#include "pcie_session.hpp"
+#include "vdma/channel/channels_group.hpp"
+
+namespace hailort
+{
+
+static constexpr uint64_t MAX_ONGOING_TRANSFERS = 128;
+
+Expected<PcieSession> PcieSession::connect(std::shared_ptr<HailoRTDriver> driver, pcie_connection_port_t port)
+{
+ TRY(auto input_desc_list, create_desc_list(*driver));
+ TRY(auto output_desc_list, create_desc_list(*driver));
+
+ TRY(auto channel_pair, driver->soc_connect(input_desc_list.handle(), output_desc_list.handle()));
+
+ (void)port;
+
+ return PcieSession::create(driver, channel_pair.first, channel_pair.second, std::move(input_desc_list),
+ std::move(output_desc_list), PcieSessionType::CLIENT);
+}
+
+Expected<PcieSession> PcieSession::accept(std::shared_ptr<HailoRTDriver> driver, pcie_connection_port_t port)
+{
+ TRY(auto input_desc_list, create_desc_list(*driver));
+ TRY(auto output_desc_list, create_desc_list(*driver));
+
+ TRY(auto channel_pair, driver->pci_ep_accept(input_desc_list.handle(), output_desc_list.handle()));
+
+ (void)port;
+
+ return PcieSession::create(driver, channel_pair.first, channel_pair.second, std::move(input_desc_list),
+ std::move(output_desc_list), PcieSessionType::SERVER);
+}
+
+Expected<PcieSession> PcieSession::create(std::shared_ptr<HailoRTDriver> driver, vdma::ChannelId input_channel_id,
+ vdma::ChannelId output_channel_id, vdma::DescriptorList &&input_desc_list, vdma::DescriptorList &&output_desc_list,
+ PcieSessionType session_type)
+{
+ // TODO: HRT-14038 - remove this to support multiple connections. Until then, mark as used to allow ctrl+c handle
+ CHECK_SUCCESS(driver->mark_as_used());
+
+ TRY(auto interrupts_dispatcher, vdma::InterruptsDispatcher::create(*driver));
+ TRY(auto transfer_launcher, vdma::TransferLauncher::create());
+
+ auto create_channel = [&](vdma::ChannelId id, vdma::BoundaryChannel::Direction dir, vdma::DescriptorList &&desc_list) {
+ return vdma::BoundaryChannel::create(*driver, id, dir, std::move(desc_list), *transfer_launcher,
+ MAX_ONGOING_TRANSFERS);
+ };
+
+ TRY(auto input_channel, create_channel(input_channel_id, vdma::BoundaryChannel::Direction::H2D, std::move(input_desc_list)));
+ TRY(auto output_channel, create_channel(output_channel_id, vdma::BoundaryChannel::Direction::D2H, std::move(output_desc_list)));
+
+ CHECK_SUCCESS(interrupts_dispatcher->start(vdma::ChannelsGroup{input_channel, output_channel}));
+ CHECK_SUCCESS(transfer_launcher->start());
+
+ CHECK_SUCCESS(input_channel->activate());
+ CHECK_SUCCESS(output_channel->activate());
+
+ return PcieSession(std::move(driver), std::move(interrupts_dispatcher), std::move(transfer_launcher),
+ std::move(input_channel), std::move(output_channel), session_type);
+}
+
+hailo_status PcieSession::write(const void *buffer, size_t size, std::chrono::milliseconds timeout)
+{
+ return launch_transfer_sync(*m_input, const_cast<void *>(buffer), size, timeout);
+}
+
+hailo_status PcieSession::read(void *buffer, size_t size, std::chrono::milliseconds timeout)
+{
+ return launch_transfer_sync(*m_output, buffer, size, timeout);
+}
+
+hailo_status PcieSession::write_async(const void *buffer, size_t size, std::function<void(hailo_status)> &&callback)
+{
+ return launch_transfer_async(*m_input, const_cast<void *>(buffer), size, std::move(callback));
+}
+
+hailo_status PcieSession::read_async(void *buffer, size_t size, std::function<void(hailo_status)> &&callback)
+{
+ return launch_transfer_async(*m_output, buffer, size, std::move(callback));
+}
+
+hailo_status PcieSession::close()
+{
+ hailo_status status = HAILO_SUCCESS; // Success orietnted
+
+ // First, close all host resources, disallow new transfers
+ m_input->deactivate();
+ m_output->deactivate();
+
+ auto stop_status = m_interrupts_dispatcher->stop();
+ if (HAILO_SUCCESS != stop_status) {
+ LOGGER__ERROR("Failed to stop interrupts dispatcher with status {}", stop_status);
+ status = stop_status;
+ }
+
+ // Then, close the connection to ABORT any vDMA channel.
+ stop_status = m_driver->close_connection(m_input->get_channel_id(), m_output->get_channel_id(), m_session_type);
+ if (HAILO_SUCCESS != stop_status) {
+ LOGGER__ERROR("Failed to close connection with status {}", stop_status);
+ status = stop_status;
+ }
+
+ // Finally, cancel any pending transfer (must happen after the vDMA channel was aborted).
+ m_input->cancel_pending_transfers();
+ m_output->cancel_pending_transfers();
+
+ stop_status = m_transfer_launcher->stop();
+ if (HAILO_SUCCESS != stop_status) {
+ LOGGER__ERROR("Failed to stop transfer launcher with status {}", stop_status);
+ status = stop_status;
+ }
+
+ return status;
+}
+
+uint64_t PcieSession::max_transfer_size()
+{
+ // The max transfer size, is the size feet in MAX_SG_DESCS_COUNT -1.
+ // We don't use the last 8 descritpors to max sure max_transfer_size is aligened to 4K
+ return vdma::DEFAULT_SG_PAGE_SIZE * (MAX_SG_DESCS_COUNT - 8);
+}
+
+hailo_status PcieSession::launch_transfer_sync(vdma::BoundaryChannel &channel,
+ void *buffer, size_t size, std::chrono::milliseconds timeout)
+{
+ std::mutex mutex;
+ std::condition_variable cv;
+ hailo_status transfer_status = HAILO_UNINITIALIZED;
+ auto callback = [&](hailo_status status) mutable {
+ {
+ std::unique_lock<std::mutex> lock(mutex);
+ assert(status != HAILO_UNINITIALIZED);
+ transfer_status = status;
+ }
+ cv.notify_one();
+ };
+
+ auto status = launch_transfer_async(channel, buffer, size, std::move(callback));
+ if (HAILO_STREAM_ABORT == status) {
+ return status;
+ }
+ CHECK_SUCCESS(status);
+
+ std::unique_lock<std::mutex> lock(mutex);
+ CHECK(cv.wait_for(lock, timeout, [&] { return transfer_status != HAILO_UNINITIALIZED; }),
+ HAILO_TIMEOUT, "Timeout waiting for transfer completion");
+ return transfer_status;
+}
+
+hailo_status PcieSession::launch_transfer_async(vdma::BoundaryChannel &channel,
+ void *buffer, size_t size, std::function<void(hailo_status)> &&callback)
+{
+ TransferRequest request{
+ {TransferBuffer(MemoryView(buffer, size))},
+ std::move(callback)
+ };
+
+ return channel.launch_transfer(std::move(request));
+}
+
+Expected<vdma::DescriptorList> PcieSession::create_desc_list(HailoRTDriver &driver)
+{
+ const bool circular = true;
+ TRY(auto desc_list, vdma::DescriptorList::create(MAX_SG_DESCS_COUNT, vdma::DEFAULT_SG_PAGE_SIZE, circular, driver));
+ return desc_list;
+}
+
+} /* namespace hailort */
--- /dev/null
+/**
+ * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file pcie_session.hpp
+ * @brief Wrapper for 2 BoundaryChannels, one for input and one for output, used as a the transport layer of a
+ * session over PCIe.
+ **/
+
+#ifndef _HAILO_PCIE_SESSION_HPP_
+#define _HAILO_PCIE_SESSION_HPP_
+
+#include "hailo/hailort.h"
+#include "vdma/channel/boundary_channel.hpp"
+#include "vdma/channel/interrupts_dispatcher.hpp"
+#include "vdma/channel/transfer_launcher.hpp"
+
+namespace hailort
+{
+
+// A special magic number used to match each accept() with the corresponding connect().
+// By using this magic, multiple servers can be implemented and run simultaneously on the same device.
+using pcie_connection_port_t = uint32_t;
+using PcieSessionType = HailoRTDriver::PcieSessionType;
+
+/**
+ * a PcieSession object need to be constructed both at the device side (via accept) or the host side (via connect).
+ * After the session is created on both sides, the session can be used to send and receive data (based on the desired
+ * protocol).
+ *
+ * This session object is a low-level object offering fast zero-copy data transfer over PCIe with negligible overhead.
+ * To achieve this, the object have the following limitations:
+ * 1. Buffers Alignment -
+ * a. The input/output buffers must be page aligned.
+ * b. The output buffer should own the full cache line size, otherwise invalidating the cache can cause memory
+ * corruption. Simple solution is to allocate the buffer using mmap.
+ * 2. Buffer Sizes
+ * a. The buffer must be a multiple of 8 bytes.
+ * b. The max size of each buffer is (desc_page_size * (max_desc_count - 1)) = 32 MB.
+ * 3. Read/Write synchronization - The size pattern of the writes in one edge must be the same as the size pattern
+ * of reads in the other edge.
+ * For example, if the host writes this pattern:
+ * WRITE 8 bytes
+ * WRITE 32 bytes
+ * WRITE 16 bytes
+ * The device must read the same pattern:
+ * READ 8 bytes
+ * READ 32 bytes
+ * READ 16 bytes
+ *
+ * The protocol must ensure this behavior (for example by sending a const size header before each write).
+ */
+class PcieSession final {
+public:
+
+ static Expected<PcieSession> connect(std::shared_ptr<HailoRTDriver> driver, pcie_connection_port_t port);
+ static Expected<PcieSession> accept(std::shared_ptr<HailoRTDriver> driver, pcie_connection_port_t port);
+
+ hailo_status write(const void *buffer, size_t size, std::chrono::milliseconds timeout);
+ hailo_status read(void *buffer, size_t size, std::chrono::milliseconds timeout);
+
+ hailo_status write_async(const void *buffer, size_t size, std::function<void(hailo_status)> &&callback);
+ hailo_status read_async(void *buffer, size_t size, std::function<void(hailo_status)> &&callback);
+
+ hailo_status close();
+
+ inline PcieSessionType session_type() const
+ {
+ return m_session_type;
+ }
+
+ static uint64_t max_transfer_size();
+
+private:
+
+ using ChannelIdsPair = std::pair<vdma::ChannelId, vdma::ChannelId>;
+ using DescriptorsListPair = std::pair<std::reference_wrapper<vdma::DescriptorList>, std::reference_wrapper<vdma::DescriptorList>>;
+
+ static Expected<PcieSession> create(std::shared_ptr<HailoRTDriver> driver, vdma::ChannelId input_channel,
+ vdma::ChannelId output_channel, vdma::DescriptorList &&input_desc_list, vdma::DescriptorList &&output_desc_list,
+ PcieSessionType session_type);
+
+ PcieSession(std::shared_ptr<HailoRTDriver> &&driver,
+ std::unique_ptr<vdma::InterruptsDispatcher> &&interrupts_dispatcher,
+ std::unique_ptr<vdma::TransferLauncher> &&transfer_launcher,
+ vdma::BoundaryChannelPtr &&input, vdma::BoundaryChannelPtr &&output, PcieSessionType session_type) :
+ m_driver(std::move(driver)),
+ m_interrupts_dispatcher(std::move(interrupts_dispatcher)),
+ m_transfer_launcher(std::move(transfer_launcher)),
+ m_input(std::move(input)),
+ m_output(std::move(output)),
+ m_session_type(session_type)
+ {}
+
+ static hailo_status launch_transfer_sync(vdma::BoundaryChannel &channel,
+ void *buffer, size_t size, std::chrono::milliseconds timeout);
+ static hailo_status launch_transfer_async(vdma::BoundaryChannel &channel,
+ void *buffer, size_t size, std::function<void(hailo_status)> &&callback);
+ static Expected<vdma::DescriptorList> create_desc_list(HailoRTDriver &driver);
+
+ std::shared_ptr<HailoRTDriver> m_driver;
+
+ std::unique_ptr<vdma::InterruptsDispatcher> m_interrupts_dispatcher;
+ std::unique_ptr<vdma::TransferLauncher> m_transfer_launcher;
+
+ vdma::BoundaryChannelPtr m_input;
+ vdma::BoundaryChannelPtr m_output;
+
+ PcieSessionType m_session_type;
+};
+
+} /* namespace hailort */
+
+#endif /* _HAILO_PCIE_SESSION_HPP_ */
#include "vdma/vdma_config_core_op.hpp"
#include "network_group/network_group_internal.hpp"
#include "net_flow/pipeline/vstream_internal.hpp"
-
+#include "device_common/control.hpp"
namespace hailort
{
Expected<VdmaConfigCoreOp> VdmaConfigCoreOp::create(ActiveCoreOpHolder &active_core_op_holder,
const ConfigureNetworkParams &config_params,
std::shared_ptr<ResourcesManager> resources_manager,
+ std::shared_ptr<CacheManager> cache_manager,
std::shared_ptr<CoreOpMetadata> metadata)
{
auto status = HAILO_UNINITIALIZED;
- VdmaConfigCoreOp object(active_core_op_holder, config_params,
- std::move(resources_manager), metadata, status);
+ VdmaConfigCoreOp core_op(active_core_op_holder, config_params, std::move(resources_manager), cache_manager,
+ metadata, status);
CHECK_SUCCESS_AS_EXPECTED(status);
- return object;
+ return core_op;
}
-VdmaConfigCoreOp::VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder,
- const ConfigureNetworkParams &config_params,
- std::shared_ptr<ResourcesManager> &&resources_manager,
- std::shared_ptr<CoreOpMetadata> metadata, hailo_status &status) :
- CoreOp(config_params, metadata, active_core_op_holder, status),
- m_resources_manager(std::move(resources_manager))
+Expected<std::shared_ptr<VdmaConfigCoreOp>> VdmaConfigCoreOp::create_shared(ActiveCoreOpHolder &active_core_op_holder,
+ const ConfigureNetworkParams &config_params,
+ std::shared_ptr<ResourcesManager> resources_manager,
+ std::shared_ptr<CacheManager> cache_manager,
+ std::shared_ptr<CoreOpMetadata> metadata)
+{
+ TRY(auto core_op, create(active_core_op_holder, config_params, resources_manager, cache_manager, metadata));
+ auto core_op_ptr = make_shared_nothrow<VdmaConfigCoreOp>(std::move(core_op));
+ CHECK_NOT_NULL_AS_EXPECTED(core_op_ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+ return core_op_ptr;
+}
+
+VdmaConfigCoreOp::VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder, const ConfigureNetworkParams &config_params,
+ std::shared_ptr<ResourcesManager> &&resources_manager, std::shared_ptr<CacheManager> cache_manager,
+ std::shared_ptr<CoreOpMetadata> metadata, hailo_status &status) :
+ CoreOp(config_params, metadata, active_core_op_holder, status),
+ m_resources_manager(std::move(resources_manager)),
+ m_cache_manager(cache_manager)
{}
hailo_status VdmaConfigCoreOp::activate_impl(uint16_t dynamic_batch_size)
{
- auto status = HAILO_UNINITIALIZED;
- auto start_time = std::chrono::steady_clock::now();
+ auto status = register_cache_update_callback();
+ CHECK_SUCCESS(status, "Failed to register cache update callback");
+ auto start_time = std::chrono::steady_clock::now();
if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE != dynamic_batch_size) {
CHECK(dynamic_batch_size <= get_smallest_configured_batch_size(get_config_params()),
HAILO_INVALID_ARGUMENT, "Batch size given is {} although max is {}", dynamic_batch_size,
//TODO: HRT-13019 - Unite with the calculation in core_op.cpp
const auto elapsed_time_ms = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - start_time).count();
- TRACE(ActivateCoreOpTrace, std::string(m_resources_manager->get_dev_id()), vdevice_core_op_handle(), elapsed_time_ms);
+ TRACE(ActivateCoreOpTrace, std::string(m_resources_manager->get_dev_id()), vdevice_core_op_handle(), elapsed_time_ms, dynamic_batch_size);
return HAILO_SUCCESS;
}
std::chrono::steady_clock::now() - start_time).count();
TRACE(DeactivateCoreOpTrace, std::string(m_resources_manager->get_dev_id()), vdevice_core_op_handle(), elapsed_time_ms);
+ status = unregister_cache_update_callback();
+ CHECK_SUCCESS(status, "Failed to unregister cache update callback");
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status VdmaConfigCoreOp::register_cache_update_callback()
+{
+ const auto cache_offset_env_var = get_env_variable(HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR);
+ if (cache_offset_env_var.has_value() && has_caches()) {
+ std::string policy;
+ int32_t offset_delta = 0;
+ TRY(const auto cache_write_size, get_cache_write_size());
+ if (cache_offset_env_var.value() == HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DEFAULT) {
+ offset_delta = cache_write_size;
+ policy = "cache write size (default)";
+ } else if (cache_offset_env_var.value() == HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DISABLED) {
+ LOGGER__INFO("Skipping cache offset updates");
+ return HAILO_SUCCESS;
+ } else {
+ offset_delta = std::stoi(cache_offset_env_var.value());
+ policy = "environment variable";
+ CHECK(offset_delta <= static_cast<int32_t>(cache_write_size), HAILO_INVALID_ARGUMENT, "Invalid cache offset delta");
+ }
+
+ auto &vdma_device = m_resources_manager->get_device();
+ vdma_device.set_notification_callback([this, offset_delta](Device &, const hailo_notification_t ¬ification, void *) {
+ if (HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET != notification.id) {
+ LOGGER__ERROR("Notification id passed to callback is invalid");
+ return;
+ }
+
+ const auto status = this->update_cache_offset(static_cast<int32_t>(offset_delta));
+ if (HAILO_SUCCESS != status) {
+ LOGGER__ERROR("Failed to update cache offset");
+ }
+ }, HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET, nullptr);
+
+ LOGGER__INFO("Cache offsets will automatically be updated by {} [{}]", offset_delta, policy);
+ }
+
+ return HAILO_SUCCESS;
+}
+
+hailo_status VdmaConfigCoreOp::unregister_cache_update_callback()
+{
+ auto &vdma_device = m_resources_manager->get_device();
+ auto status = vdma_device.remove_notification_callback(HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET);
+ CHECK_SUCCESS(status);
return HAILO_SUCCESS;
}
return m_resources_manager->read_intermediate_buffer(key);
}
+Expected<Buffer> VdmaConfigCoreOp::get_cache_buffer(uint32_t cache_id)
+{
+ return m_resources_manager->read_cache_buffer(cache_id);
+}
+
+Expected<std::map<uint32_t, Buffer>> VdmaConfigCoreOp::get_cache_buffers()
+{
+ return m_resources_manager->read_cache_buffers();
+}
+
+bool VdmaConfigCoreOp::has_caches() const
+{
+ return m_resources_manager->get_cache_buffers().size() > 0;
+}
+
+Expected<uint32_t> VdmaConfigCoreOp::get_cache_read_size() const
+{
+ // Input to the core == cache read
+ size_t input_size = 0;
+ for (auto &cache_buffer : m_resources_manager->get_cache_buffers()) {
+ const auto curr_input_size = cache_buffer.second.input_size();
+ if (input_size == 0) {
+ input_size = curr_input_size;
+ } else {
+ CHECK(input_size == curr_input_size, HAILO_INVALID_ARGUMENT, "Cache buffers have different input sizes");
+ }
+ }
+
+ return static_cast<uint32_t>(input_size);
+}
+
+Expected<uint32_t> VdmaConfigCoreOp::get_cache_write_size() const
+{
+ // Output from the core == cache write
+ size_t output_size = 0;
+ for (auto &cache_buffer : m_resources_manager->get_cache_buffers()) {
+ const auto curr_output_size = cache_buffer.second.output_size();
+ if (output_size == 0) {
+ output_size = curr_output_size;
+ } else {
+ CHECK(output_size == curr_output_size, HAILO_INVALID_ARGUMENT, "Cache buffers have different output sizes");
+ }
+ }
+
+ return static_cast<uint32_t>(output_size);
+}
+
+
+hailo_status VdmaConfigCoreOp::init_cache(uint32_t read_offset, int32_t write_offset_delta)
+{
+ CHECK(has_caches(), HAILO_INVALID_OPERATION, "No caches in core-op");
+ return m_cache_manager->init_caches(read_offset, write_offset_delta);
+}
+
+Expected<hailo_cache_info_t> VdmaConfigCoreOp::get_cache_info() const
+{
+ CHECK(has_caches(), HAILO_INVALID_OPERATION, "No caches in core-op");
+
+ return hailo_cache_info_t{
+ m_cache_manager->get_cache_size(),
+ m_cache_manager->get_read_offset_bytes(),
+ m_cache_manager->get_write_offset_bytes_delta()
+ };
+}
+
+hailo_status VdmaConfigCoreOp::update_cache_offset(int32_t offset_delta_bytes)
+{
+ CHECK(has_caches(), HAILO_INVALID_OPERATION, "No caches in core-op");
+
+ // TODO: figure out how to do this s.t. it'll work with the sched (HRT-14287)
+ // auto status = wait_for_activation(std::chrono::milliseconds(0));
+ // CHECK_SUCCESS(status, "Core op must be activated before updating cache offset");
+
+ // Update the offsets in the cache manager
+ auto status = m_cache_manager->update_cache_offset(offset_delta_bytes);
+ CHECK_SUCCESS(status);
+
+ // Signal to the fw that the cache offset has been updated
+ status = Control::context_switch_signal_cache_updated(m_resources_manager->get_device());
+ CHECK_SUCCESS(status);
+
+ return HAILO_SUCCESS;
+}
+
} /* namespace hailort */
#include "vdma/channel/boundary_channel.hpp"
#include "core_op/resource_manager/resource_manager.hpp"
+#include "core_op/resource_manager/cache_manager.hpp"
#include "core_op/active_core_op_holder.hpp"
#include "control_protocol.h"
static Expected<VdmaConfigCoreOp> create(ActiveCoreOpHolder &active_core_op_holder,
const ConfigureNetworkParams &config_params,
std::shared_ptr<ResourcesManager> resources_managers,
+ std::shared_ptr<CacheManager> cache_manager,
+ std::shared_ptr<CoreOpMetadata> metadata);
+
+ static Expected<std::shared_ptr<VdmaConfigCoreOp>> create_shared(ActiveCoreOpHolder &active_core_op_holder,
+ const ConfigureNetworkParams &config_params,
+ std::shared_ptr<ResourcesManager> resources_manager,
+ std::shared_ptr<CacheManager> cache_manager,
std::shared_ptr<CoreOpMetadata> metadata);
std::shared_ptr<ResourcesManager> &get_resources_manager()
hailo_status cancel_pending_transfers();
+ hailo_status register_cache_update_callback();
+ hailo_status unregister_cache_update_callback();
+
virtual Expected<hailo_stream_interface_t> get_default_streams_interface() override;
virtual Expected<std::shared_ptr<LatencyMetersMap>> get_latency_meters() override;
virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name) override;
virtual Expected<HwInferResults> run_hw_infer_estimator() override;
virtual Expected<Buffer> get_intermediate_buffer(const IntermediateBufferKey &) override;
+ virtual Expected<Buffer> get_cache_buffer(uint32_t cache_id) override;
+ virtual Expected<std::map<uint32_t, Buffer>> get_cache_buffers() override;
+ virtual bool has_caches() const override;
+ virtual Expected<uint32_t> get_cache_read_size() const override;
+ virtual Expected<uint32_t> get_cache_write_size() const override;
+ virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override;
+ virtual Expected<hailo_cache_info_t> get_cache_info() const;
+ virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override;
virtual ~VdmaConfigCoreOp() = default;
VdmaConfigCoreOp(const VdmaConfigCoreOp &other) = delete;
VdmaConfigCoreOp &operator=(const VdmaConfigCoreOp &other) = delete;
VdmaConfigCoreOp &operator=(VdmaConfigCoreOp &&other) = delete;
VdmaConfigCoreOp(VdmaConfigCoreOp &&other) noexcept : CoreOp(std::move(other)),
- m_resources_manager(std::move(other.m_resources_manager))
+ m_resources_manager(std::move(other.m_resources_manager)),
+ m_cache_manager(std::move(other.m_cache_manager))
{}
private:
-VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder,
+ VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder,
const ConfigureNetworkParams &config_params,
std::shared_ptr<ResourcesManager> &&resources_manager,
+ std::shared_ptr<CacheManager> cache_manager,
std::shared_ptr<CoreOpMetadata> metadata, hailo_status &status);
std::shared_ptr<ResourcesManager> m_resources_manager;
+ std::shared_ptr<CacheManager> m_cache_manager;
};
} /* namespace hailort */
const auto core_op_handle = next ? next->vdevice_core_op_handle() : INVALID_CORE_OP_HANDLE;
const auto elapsed_time_ms = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - start_time).count();
- TRACE(SwitchCoreOpTrace, device_id, core_op_handle, elapsed_time_ms);
+ TRACE(SwitchCoreOpTrace, device_id, core_op_handle, elapsed_time_ms, batch_size);
return HAILO_SUCCESS;
}
// TODO: HRT-13253 don't use resources manager instead call m_vdma_device directly. The device should store the
// current active core op.
if (next != nullptr) {
+ CHECK_SUCCESS(next->register_cache_update_callback(), "Failed to register cache update callback");
CHECK_SUCCESS(next->get_resources_manager()->enable_state_machine(batch_size), "Failed to enable state machine");
// In the case of switch NG, we call FW switch to next NG without marking the current NG as deactivated.
// Added setter to mark the current NG as deactivated.
if (current != nullptr) {
CHECK_SUCCESS(current->cancel_pending_transfers(), "Failed canceling pending transfers from previous core-op");
+ CHECK_SUCCESS(current->unregister_cache_update_callback(), "Failed unregistering cache updates from previous core-op");
}
return HAILO_SUCCESS;
static constexpr std::chrono::milliseconds DEFAULT_TIMEOUT(50000);
#endif /* ifndef HAILO_EMULATOR */
-VdmaDevice::VdmaDevice(std::unique_ptr<HailoRTDriver> &&driver, Device::Type type) :
+VdmaDevice::VdmaDevice(std::unique_ptr<HailoRTDriver> &&driver, Device::Type type, hailo_status &status) :
DeviceBase::DeviceBase(type),
m_driver(std::move(driver)),
m_is_configured(false)
{
activate_notifications(get_dev_id());
+
+ status = HAILO_SUCCESS;
}
Expected<std::unique_ptr<VdmaDevice>> VdmaDevice::create(const std::string &device_id)
status = clear_configured_apps();
CHECK_SUCCESS_AS_EXPECTED(status);
+ assert(nullptr == m_cache_manager);
+ TRY(m_cache_manager, CacheManager::create_shared(get_driver()));
+
assert(nullptr == m_vdma_interrupts_dispatcher);
- TRY(m_vdma_interrupts_dispatcher, vdma::InterruptsDispatcher::create(std::ref(*m_driver)));
+ TRY(m_vdma_interrupts_dispatcher, vdma::InterruptsDispatcher::create(get_driver()));
assert(nullptr == m_vdma_transfer_launcher);
TRY(m_vdma_transfer_launcher, vdma::TransferLauncher::create());
assert(core_ops_metadata.size() == 1);
auto core_op_metadata = core_ops_metadata[0];
- /* build HEF supported features */
- auto resource_manager = ResourcesManagerBuilder::build(current_core_op_index,
- *this, get_driver(), config_params, core_op_metadata, static_cast<HEFHwArch>(hef.pimpl->get_device_arch()),
- hef.pimpl->get_shef_file_handle());
- CHECK_EXPECTED(resource_manager);
-
+ auto status = m_cache_manager->create_caches_from_core_op(core_op_metadata);
+ CHECK_SUCCESS(status);
- auto core_op = VdmaConfigCoreOp::create(m_active_core_op_holder, config_params,
- resource_manager.release(), core_op_metadata);
+ TRY(auto resource_manager, ResourcesManagerBuilder::build(current_core_op_index,
+ *this, get_driver(), m_cache_manager, config_params, core_op_metadata,
+ static_cast<HEFHwArch>(hef.pimpl->get_device_arch())));
- auto core_op_ptr = make_shared_nothrow<VdmaConfigCoreOp>(core_op.release());
- CHECK_AS_EXPECTED(nullptr != core_op_ptr, HAILO_OUT_OF_HOST_MEMORY);
+ TRY(auto core_op_ptr, VdmaConfigCoreOp::create_shared(m_active_core_op_holder, config_params,
+ resource_manager, m_cache_manager, core_op_metadata));
// TODO: move this func into VdmaConfigCoreOp c'tor
- auto status = core_op_ptr->create_streams_from_config_params(*this);
- CHECK_SUCCESS_AS_EXPECTED(status);
+ status = core_op_ptr->create_streams_from_config_params(*this);
+ CHECK_SUCCESS(status);
- // Check that all boundary streams were created
+ // Check that all boundary streams were created
status = hef.pimpl->validate_boundary_streams_were_created(core_op_metadata->core_op_name(), core_op_ptr);
- CHECK_SUCCESS_AS_EXPECTED(status);
+ CHECK_SUCCESS(status);
core_ops.emplace_back(core_op_ptr);
m_core_ops.emplace_back(core_op_ptr);
buffer_identifier = buffer->buffer_identifier();
}
- CHECK_EXPECTED(m_driver->vdma_buffer_map(address, size, to_hailo_driver_direction(data_direction), buffer_identifier));
+ CHECK_EXPECTED(m_driver->vdma_buffer_map(reinterpret_cast<uintptr_t>(address), size, to_hailo_driver_direction(data_direction), buffer_identifier,
+ HailoRTDriver::DmaBufferType::USER_PTR_BUFFER));
return HAILO_SUCCESS;
}
return HAILO_SUCCESS;
}
- return m_driver->vdma_buffer_unmap(address, size, to_hailo_driver_direction(data_direction));
+ return m_driver->vdma_buffer_unmap(reinterpret_cast<uintptr_t>(address), size, to_hailo_driver_direction(data_direction));
+}
+
+hailo_status VdmaDevice::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t data_direction)
+{
+ // Dont use BufferStorageResourceManager for dmabuf seeing as dmabufs are not allocated from hailoRT
+ auto buffer_identifier = HailoRTDriver::INVALID_MAPPED_BUFFER_DRIVER_IDENTIFIER;
+ CHECK_EXPECTED(m_driver->vdma_buffer_map(dmabuf_fd, size, to_hailo_driver_direction(data_direction), buffer_identifier,
+ HailoRTDriver::DmaBufferType::DMABUF_BUFFER));
+ return HAILO_SUCCESS;
+}
+
+hailo_status VdmaDevice::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t data_direction)
+{
+ return m_driver->vdma_buffer_unmap(dmabuf_fd, size, to_hailo_driver_direction(data_direction));
}
Expected<ConfiguredNetworkGroupVector> VdmaDevice::create_networks_group_vector(Hef &hef, const NetworkGroupsParamsMap &configure_params)
#include "vdma/channel/interrupts_dispatcher.hpp"
#include "vdma/channel/transfer_launcher.hpp"
#include "vdma/driver/hailort_driver.hpp"
-
+#include "core_op/resource_manager/cache_manager.hpp"
namespace hailort
{
virtual hailo_status dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) override;
virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override;
+ virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override;
protected:
- VdmaDevice(std::unique_ptr<HailoRTDriver> &&driver, Type type);
+ VdmaDevice(std::unique_ptr<HailoRTDriver> &&driver, Type type, hailo_status &status);
virtual Expected<D2H_EVENT_MESSAGE_t> read_notification() override;
virtual hailo_status disable_notifications() override;
virtual Expected<ConfiguredNetworkGroupVector> add_hef(Hef &hef, const NetworkGroupsParamsMap &configure_params) override;
std::unique_ptr<HailoRTDriver> m_driver;
+ CacheManagerPtr m_cache_manager;
// TODO - HRT-13234, move to DeviceBase
std::vector<std::shared_ptr<CoreOp>> m_core_ops;
std::vector<std::shared_ptr<ConfiguredNetworkGroup>> m_network_groups; // TODO: HRT-9547 - Remove when ConfiguredNetworkGroup will be kept in global context
#include "vdma/circular_stream_buffer_pool.hpp"
#include "utils/profiler/tracer_macros.hpp"
#include "utils/buffer_storage.hpp"
-#include "common/os_utils.hpp"
namespace hailort
VdmaInputStream::~VdmaInputStream()
{
- // We want to stop the vdma channel before closing the stream in the firmware
- // because sending data to a closed stream may terminate the dma engine
- const auto status = m_channel->deactivate();
- if (HAILO_SUCCESS != status) {
- LOGGER__ERROR("Failed to deactivate stream with error status {}", status);
- }
+ m_channel->deactivate();
}
hailo_stream_interface_t VdmaInputStream::get_interface() const
Expected<std::unique_ptr<StreamBufferPool>> VdmaInputStream::allocate_buffer_pool()
{
- TRY(auto circular_pool, CircularStreamBufferPool::create(m_device, HAILO_DMA_BUFFER_DIRECTION_H2D,
- m_channel->get_desc_list()->desc_page_size(), m_channel->get_desc_list()->count(), get_frame_size()));
+ const auto frame_size = get_frame_size();
+ const auto max_transfers_in_desc_list = m_channel->get_max_aligned_transfers_in_desc_list(frame_size);
+ const auto max_ongoing_transfers = m_channel->get_max_ongoing_transfers(frame_size);
+ if (max_transfers_in_desc_list < max_ongoing_transfers) {
+ // In this case we don't bind, since the descriptor list isn't big enough to hold all the buffers.
+ // (otherwise pending_transfers_queue_in_use would be false)
+ TRY(auto stream_buffer_pool, QueuedStreamBufferPool::create(max_ongoing_transfers, frame_size,
+ BufferStorageParams::create_dma()));
+
+ return std::unique_ptr<StreamBufferPool>(std::move(stream_buffer_pool));
+ } else {
+ // We can fit all the buffers in the descriptor list, so we can bind them statically.
+ TRY(auto circular_pool, CircularStreamBufferPool::create(m_device, HAILO_DMA_BUFFER_DIRECTION_H2D,
+ m_channel->get_desc_list().desc_page_size(), m_channel->get_desc_list().count(), frame_size));
- // Bind the buffer to the channel to avoid the need to do it on every transfer.
- TRY(auto pool_dma_able_buffer, circular_pool->get_base_buffer().storage().get_dma_able_buffer());
- TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared(pool_dma_able_buffer, m_device.get_driver(),
- HailoRTDriver::DmaDirection::H2D));
- CHECK_SUCCESS(m_channel->bind_buffer(mapped_buffer));
+ // Bind the buffer to the channel to avoid the need to do it on every transfer.
+ TRY(auto pool_dma_able_buffer, circular_pool->get_base_buffer().storage().get_dma_able_buffer());
+ TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared(pool_dma_able_buffer, m_device.get_driver(),
+ HailoRTDriver::DmaDirection::H2D));
+ CHECK_SUCCESS(m_channel->bind_buffer(mapped_buffer));
- return std::unique_ptr<StreamBufferPool>(std::move(circular_pool));
+ return std::unique_ptr<StreamBufferPool>(std::move(circular_pool));
+ }
}
size_t VdmaInputStream::get_max_ongoing_transfers() const
{
const auto dma_alignment = OsUtils::get_dma_able_alignment();
std::vector<TransferBuffer> transfer_buffers;
- const auto buffer_address = transfer_request.transfer_buffers[0].base_buffer().data();
+ TRY(auto base_buffer, transfer_request.transfer_buffers[0].base_buffer());
+ const auto buffer_address = base_buffer.data();
const auto buffer_size = transfer_request.transfer_buffers[0].size();
TRY(const auto dma_able_bounce_buffer, m_bounce_buffers_pool->dequeue());
{
TRACE(FrameDequeueH2DTrace, m_device.get_dev_id(), m_core_op_handle, name());
- const auto dma_able_alignment = OsUtils::get_dma_able_alignment();
- if (reinterpret_cast<size_t>(transfer_request.transfer_buffers[0].base_buffer().data()) % dma_able_alignment == 0) {
+ if (transfer_request.transfer_buffers[0].type() == TransferBufferType::DMABUF) {
return m_channel->launch_transfer(std::move(transfer_request));
} else {
- auto unaligned_transfer_request = align_transfer_request(std::move(transfer_request));
- CHECK_EXPECTED_AS_STATUS(unaligned_transfer_request);
- return m_channel->launch_transfer(unaligned_transfer_request.release());
+ TRY(auto is_request_aligned, transfer_request.is_request_aligned());
+ if (is_request_aligned) {
+ return m_channel->launch_transfer(std::move(transfer_request));
+ } else {
+ auto realigned_transfer_request = align_transfer_request(std::move(transfer_request));
+ CHECK_EXPECTED_AS_STATUS(realigned_transfer_request);
+ return m_channel->launch_transfer(realigned_transfer_request.release());
+ }
}
}
hailo_status VdmaInputStream::deactivate_stream_impl()
{
- return m_channel->deactivate();
+ m_channel->deactivate();
+ return HAILO_SUCCESS;
}
hailo_status VdmaInputStream::cancel_pending_transfers()
m_channel(std::move(channel)),
m_interface(interface),
m_transfer_size(get_transfer_size(m_stream_info, get_layer_info())),
- m_core_op_handle(INVALID_CORE_OP_HANDLE)
+ m_core_op_handle(INVALID_CORE_OP_HANDLE),
+ m_d2h_callback(default_d2h_callback)
{
// Check status for base class c'tor
if (HAILO_SUCCESS != status) {
VdmaOutputStream::~VdmaOutputStream()
{
- // We want to stop the vdma channel before closing the stream in the firmware
- // because sending data to a closed stream may terminate the dma engine
- const auto status = m_channel->deactivate();
- if (HAILO_SUCCESS != status) {
- LOGGER__ERROR("Failed to deactivate stream with error status {}", status);
- }
+ m_channel->deactivate();
}
hailo_stream_interface_t VdmaOutputStream::get_interface() const
Expected<std::unique_ptr<StreamBufferPool>> VdmaOutputStream::allocate_buffer_pool()
{
- TRY(auto circular_pool, CircularStreamBufferPool::create(m_device, HAILO_DMA_BUFFER_DIRECTION_D2H,
- m_channel->get_desc_list()->desc_page_size(), m_channel->get_desc_list()->count(), m_transfer_size));
-
- // Bind the buffer to the channel to avoid the need to do it on every transfer.
- TRY(auto pool_dma_able_buffer, circular_pool->get_base_buffer().storage().get_dma_able_buffer());
- TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared(pool_dma_able_buffer, m_device.get_driver(),
- HailoRTDriver::DmaDirection::D2H));
- CHECK_SUCCESS(m_channel->bind_buffer(mapped_buffer));
-
- return std::unique_ptr<StreamBufferPool>(std::move(circular_pool));
+ const auto max_transfers_in_desc_list = m_channel->get_max_aligned_transfers_in_desc_list(m_transfer_size);
+ const auto max_ongoing_transfers = m_channel->get_max_ongoing_transfers(m_transfer_size);
+ if (max_transfers_in_desc_list < max_ongoing_transfers) {
+ // In this case we don't bind, since the descriptor list isn't big enough to hold all the buffers.
+ // (otherwise pending_transfers_queue_in_use would be false)
+ TRY(auto stream_buffer_pool, QueuedStreamBufferPool::create(max_ongoing_transfers, m_transfer_size,
+ BufferStorageParams::create_dma()));
+
+ return std::unique_ptr<StreamBufferPool>(std::move(stream_buffer_pool));
+ } else {
+ // We can fit all the buffers in the descriptor list, so we can bind them statically.
+ TRY(auto circular_pool, CircularStreamBufferPool::create(m_device, HAILO_DMA_BUFFER_DIRECTION_D2H,
+ m_channel->get_desc_list().desc_page_size(), m_channel->get_desc_list().count(), m_transfer_size));
+
+ // Bind the buffer to the channel to avoid the need to do it on every transfer.
+ TRY(auto pool_dma_able_buffer, circular_pool->get_base_buffer().storage().get_dma_able_buffer());
+ TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared(pool_dma_able_buffer, m_device.get_driver(),
+ HailoRTDriver::DmaDirection::D2H));
+ CHECK_SUCCESS(m_channel->bind_buffer(mapped_buffer));
+
+ return std::unique_ptr<StreamBufferPool>(std::move(circular_pool));
+ }
}
size_t VdmaOutputStream::get_max_ongoing_transfers() const
CHECK_EXPECTED(bounce_buffer_exp);
auto bounce_buffer = bounce_buffer_exp.release();
- auto wrapped_callback = [unaligned_user_buffer = transfer_request.transfer_buffers[0].base_buffer(),
+ TRY(auto base_buffer, transfer_request.transfer_buffers[0].base_buffer());
+ auto wrapped_callback = [unaligned_user_buffer = base_buffer,
bounce_buffer=bounce_buffer, user_callback=transfer_request.callback](hailo_status callback_status) {
memcpy(const_cast<uint8_t*>(unaligned_user_buffer.data()), bounce_buffer->data(), unaligned_user_buffer.size());
user_callback(callback_status);
hailo_status VdmaOutputStream::read_async_impl(TransferRequest &&transfer_request)
{
- if ((INVALID_CORE_OP_HANDLE != m_core_op_handle) && (HAILO_FORMAT_ORDER_HAILO_NMS != m_stream_info.format.order)) {
+ if (HAILO_FORMAT_ORDER_HAILO_NMS != m_stream_info.format.order) {
// On NMS stream we trace EnqueueD2H inside nms_stream
- transfer_request.callback = [original_callback=transfer_request.callback, this](hailo_status status) {
- if (HAILO_SUCCESS == status) {
+ transfer_request.callback = [original_callback=transfer_request.callback, d2h_callback=m_d2h_callback, this](hailo_status status) {
+ if ((HAILO_SUCCESS == status) && (INVALID_CORE_OP_HANDLE != m_core_op_handle)) {
TRACE(FrameEnqueueD2HTrace, m_device.get_dev_id(), m_core_op_handle, name());
}
+ d2h_callback(status);
original_callback(status);
};
}
- const auto dma_able_alignment = OsUtils::get_dma_able_alignment();
- if (reinterpret_cast<size_t>(transfer_request.transfer_buffers[0].base_buffer().data()) % dma_able_alignment == 0) {
+ if (transfer_request.transfer_buffers[0].type() == TransferBufferType::DMABUF) {
return m_channel->launch_transfer(std::move(transfer_request));
} else {
- // In case of read unaligned - currently doesnt support using users buffer - so allocate complete new buffer size of user's buffer
- LOGGER__WARNING("read_async() was provided an unaligned buffer (address=0x{:x}), which causes performance degradation. Use buffers algined to {} bytes for optimal performance",
- reinterpret_cast<size_t>(transfer_request.transfer_buffers[0].base_buffer().data()), dma_able_alignment);
-
- auto realigned_transfer_request = align_transfer_request(std::move(transfer_request));
- CHECK_EXPECTED_AS_STATUS(realigned_transfer_request);
- return m_channel->launch_transfer(realigned_transfer_request.release());
+ TRY(auto is_request_aligned, transfer_request.is_request_aligned());
+ if (is_request_aligned) {
+ return m_channel->launch_transfer(std::move(transfer_request));
+ } else {
+ // In case of read unaligned - don't support using users buffer - so well allocate complete new buffer size of user's buffer
+ TRY(auto base_buffer, transfer_request.transfer_buffers[0].base_buffer());
+ LOGGER__WARNING("read_async() was provided an unaligned buffer (address=0x{:x}), which causes performance degradation. Use buffers algined to {} bytes for optimal performance",
+ reinterpret_cast<size_t>(base_buffer.data()), OsUtils::get_dma_able_alignment());
+
+ auto realigned_transfer_request = align_transfer_request(std::move(transfer_request));
+ CHECK_EXPECTED_AS_STATUS(realigned_transfer_request);
+ return m_channel->launch_transfer(realigned_transfer_request.release());
+ }
}
}
hailo_status VdmaOutputStream::deactivate_stream_impl()
{
- return m_channel->deactivate();
+ m_channel->deactivate();
+ return HAILO_SUCCESS;
}
void VdmaOutputStream::set_vdevice_core_op_handle(vdevice_core_op_handle_t core_op_handle)
return HAILO_SUCCESS;
}
+void VdmaOutputStream::set_d2h_callback(std::function<void(hailo_status)> callback)
+{
+ m_d2h_callback = callback;
+}
+
} /* namespace hailort */
// TODO - HRT-11739 - remove vdevice related members/functions (get/set_vdevice_core_op_handle)
virtual inline vdevice_core_op_handle_t get_vdevice_core_op_handle() override { return m_core_op_handle; };
virtual hailo_status cancel_pending_transfers() override;
+ void set_d2h_callback(std::function<void(hailo_status)> callback);
private:
+ static void default_d2h_callback(hailo_status) {};
static uint32_t get_transfer_size(const hailo_stream_info_t &stream_info, const LayerInfo &layer_info);
Expected<TransferRequest> align_transfer_request(TransferRequest &&transfer_request);
const hailo_stream_interface_t m_interface;
const uint32_t m_transfer_size;
vdevice_core_op_handle_t m_core_op_handle;
+ std::function<void(hailo_status)> m_d2h_callback;
};
int32 new_core_op_handle = 2;
string device_id = 3;
double duration = 4; //millisec
+ int32 dynamic_batch_size = 5;
}
message ProtoProfilerDeactivateCoreOpTrace {
bool background_removal = 5;
uint32 background_removal_index = 6;
bool cross_classes = 7;
+ bool bbox_only = 8;
}
message ProtoYolov8MatchingLayersNames {
@ECHO OFF
set BASE_URI=https://hailo-hailort.s3.eu-west-2.amazonaws.com
-set HRT_VERSION=4.17.1
+set HRT_VERSION=4.18.0
set FW_DIR=Hailo8/%HRT_VERSION%/FW
set FW=hailo8_fw.%HRT_VERSION%_eth.bin
set -e
readonly BASE_URI="https://hailo-hailort.s3.eu-west-2.amazonaws.com"
-readonly HRT_VERSION=4.17.1
+readonly HRT_VERSION=4.18.0
readonly FW_AWS_DIR="Hailo8/${HRT_VERSION}/FW"
readonly FW="hailo8_fw.${HRT_VERSION}_eth.bin"
:: cmd
@ECHO OFF
set BASE_URI=https://hailo-hailort.s3.eu-west-2.amazonaws.com
-set HRT_VERSION=4.17.1
+set HRT_VERSION=4.18.0
set REMOTE_HEF_DIR=Hailo8/%HRT_VERSION%/HEFS
set LOCAL_EXAMPLES_HEF_DIR=..\libhailort\examples\hefs
set LOCAL_TUTORIALS_HEF_DIR=..\libhailort\bindings\python\platform\hailo_tutorials\hefs
set -e
readonly BASE_URI="https://hailo-hailort.s3.eu-west-2.amazonaws.com"
-readonly HRT_VERSION=4.17.1
+readonly HRT_VERSION=4.18.0
readonly REMOTE_HEF_DIR="Hailo8/${HRT_VERSION}/HEFS"
readonly LOCAL_EXAMPLES_HEF_DIR="../libhailort/examples/hefs"
readonly LOCAL_TUTORIALS_HEF_DIR="../libhailort/bindings/python/platform/hailo_tutorials/hefs"
projects / platform / upstream / hailort.git / commitdiff