From 01e4c7f5a7463cc61ef1b2d22c31dd80a3a07d95 Mon Sep 17 00:00:00 2001 From: HailoRT-Automation <98901220+HailoRT-Automation@users.noreply.github.com> Date: Tue, 9 Jul 2024 23:47:13 +0300 Subject: [PATCH] v4.18.0 --- CMakeLists.txt | 10 + common/include/context_switch_defs.h | 20 + common/include/control_protocol.h | 29 +- common/include/d2h_events.h | 12 + common/include/firmware_header.h | 3 +- common/include/firmware_status.h | 4 + hailort/.gitignore | 1 - hailort/CMakeLists.txt | 14 +- hailort/LICENSE-3RD-PARTY.md | 2 +- hailort/common/circular_buffer.hpp | 188 ++- hailort/common/event_internal.cpp | 19 +- hailort/common/file_utils.cpp | 193 ++- hailort/common/file_utils.hpp | 68 + hailort/common/filesystem.hpp | 2 +- hailort/common/logger_macros.hpp | 7 + hailort/common/os/posix/ethernet_utils.cpp | 5 +- hailort/common/os/posix/filesystem.cpp | 15 +- hailort/common/os/posix/process.cpp | 23 +- hailort/common/os/posix/socket.cpp | 9 +- hailort/common/os/posix/traffic_control.cpp | 34 +- hailort/common/os/windows/ethernet_utils.cpp | 37 +- hailort/common/os/windows/filesystem.cpp | 11 +- hailort/common/os/windows/socket.cpp | 15 +- hailort/common/socket.hpp | 2 +- hailort/common/string_utils.cpp | 7 +- hailort/common/utils.hpp | 148 ++ hailort/drivers/common/hailo_ioctl_common.h | 240 ++- .../drivers/win/include/hailo_pcie_version.h | 10 - hailort/hailo_firmware_eula | 12 + hailort/hailort_server/CMakeLists.txt | 55 + hailort/hailort_server/hailort_server.cpp | 539 +++++++ hailort/hailort_server/hailort_server.hpp | 44 + hailort/hailort_server/hailort_server.sh | 39 + hailort/hailort_service/cng_buffer_pool.cpp | 81 +- hailort/hailort_service/cng_buffer_pool.hpp | 16 +- .../hailort_service/hailort_rpc_service.cpp | 90 +- .../service_resource_manager.hpp | 19 +- .../vdevice_callbacks_queue.hpp | 22 +- hailort/hailortcli/benchmark_command.cpp | 33 +- hailort/hailortcli/board_config_command.cpp | 11 +- hailort/hailortcli/command.cpp | 19 +- hailort/hailortcli/command.hpp | 2 + .../download_action_list_command.cpp | 148 +- .../download_action_list_command.hpp | 5 + hailort/hailortcli/fw_config_command.cpp | 37 +- hailort/hailortcli/fw_config_serializer.cpp | 149 +- hailort/hailortcli/fw_control_command.cpp | 10 +- hailort/hailortcli/fw_logger_command.cpp | 64 +- hailort/hailortcli/fw_logger_command.hpp | 5 + hailort/hailortcli/hailortcli.cpp | 11 +- .../measure_nnc_performance_command.cpp | 41 +- hailort/hailortcli/mon_command.cpp | 19 +- hailort/hailortcli/parse_hef_command.cpp | 21 +- hailort/hailortcli/run2/io_wrappers.hpp | 27 +- hailort/hailortcli/run2/live_stats.cpp | 6 +- .../run2/measurement_live_track.cpp | 14 +- hailort/hailortcli/run2/network_runner.cpp | 242 ++- hailort/hailortcli/run2/network_runner.hpp | 8 +- hailort/hailortcli/run2/run2_command.cpp | 81 +- hailort/hailortcli/run_command.cpp | 390 ++--- hailort/hailortcli/scan_command.cpp | 11 +- hailort/hailortcli/sensor_config_command.cpp | 11 +- .../hailortcli/udp_rate_limiter_command.cpp | 18 +- hailort/hrpc/CMakeLists.txt | 22 + hailort/hrpc/client.cpp | 125 ++ hailort/hrpc/client.hpp | 68 + .../hrpc/os/pcie/raw_connection_internal.cpp | 196 +++ .../hrpc/os/pcie/raw_connection_internal.hpp | 78 + .../hrpc/os/posix/raw_connection_internal.cpp | 122 ++ .../hrpc/os/posix/raw_connection_internal.hpp | 54 + .../os/windows/raw_connection_internal.cpp | 57 + .../os/windows/raw_connection_internal.hpp | 48 + hailort/hrpc/raw_connection.cpp | 47 + hailort/hrpc/raw_connection.hpp | 58 + hailort/hrpc/rpc_connection.cpp | 82 + hailort/hrpc/rpc_connection.hpp | 54 + hailort/hrpc/server.cpp | 123 ++ hailort/hrpc/server.hpp | 78 + hailort/hrpc_protocol/CMakeLists.txt | 24 + hailort/hrpc_protocol/rpc.proto | 207 +++ hailort/hrpc_protocol/serializer.cpp | 834 ++++++++++ hailort/hrpc_protocol/serializer.hpp | 253 ++++ hailort/libhailort/CMakeLists.txt | 9 +- .../bindings/gstreamer/CMakeLists.txt | 4 +- .../bindings/gstreamer/gst-hailo/common.hpp | 62 +- .../gst-hailo/gsthailo_allocator.cpp | 59 + .../gst-hailo/gsthailo_allocator.hpp | 50 + .../gst-hailo/gsthailo_dmabuf_allocator.cpp | 91 ++ .../gst-hailo/gsthailo_dmabuf_allocator.hpp | 61 + .../gstreamer/gst-hailo/gsthailonet.cpp | 426 +++--- .../gstreamer/gst-hailo/gsthailonet.hpp | 123 +- .../libhailort/bindings/python/CMakeLists.txt | 1 - .../platform/hailo_platform/__init__.py | 24 +- .../hailo_platform/pyhailort/pyhailort.py | 973 +++++++++++- .../tools/hailocli/hailocli_commands.py | 7 +- .../notebooks/HRT_0_Inference_Tutorial.ipynb | 8 +- ...rence_Tutorial_Multi_Process_Service.ipynb | 8 +- ...RT_3_Inference_Single_Model_Tutorial.ipynb | 102 ++ ...c_Inference_Multiple_Models_Tutorial.ipynb | 195 +++ .../bindings/python/platform/setup.py | 123 +- .../bindings/python/src/CMakeLists.txt | 64 +- .../bindings/python/src/bindings_common.hpp | 15 +- .../bindings/python/src/device_api.cpp | 27 +- .../bindings/python/src/device_api.hpp | 5 +- .../bindings/python/src/hef_api.cpp | 9 +- .../bindings/python/src/hef_api.hpp | 4 +- .../bindings/python/src/infer_model_api.cpp | 477 ++++++ .../bindings/python/src/infer_model_api.hpp | 195 +++ .../bindings/python/src/network_group_api.cpp | 13 +- .../bindings/python/src/network_group_api.hpp | 24 +- .../bindings/python/src/pyhailort.cpp | 68 +- .../bindings/python/src/vdevice_api.cpp | 36 + .../bindings/python/src/vdevice_api.hpp | 48 +- .../bindings/python/src/vstream_api.cpp | 39 +- .../bindings/python/src/vstream_api.hpp | 10 +- hailort/libhailort/doc/CMakeLists.txt | 2 +- .../data_quantization_example/CMakeLists.txt | 2 +- .../c/infer_pipeline_example/CMakeLists.txt | 2 +- .../c/multi_device_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../multi_network_vstream_example.c | 4 +- .../CMakeLists.txt | 2 +- .../power_measurement_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../c/raw_streams_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../switch_network_groups_example.c | 4 +- .../CMakeLists.txt | 2 +- .../c/vstreams_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../async_infer_advanced_example.cpp | 77 +- .../async_infer_basic_example/CMakeLists.txt | 2 +- .../async_infer_basic_example.cpp | 5 + .../cpp/infer_pipeline_example/CMakeLists.txt | 2 +- .../cpp/multi_device_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../cpp/multi_process_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../power_measurement_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../cpp/raw_streams_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../cpp/vstreams_example/CMakeLists.txt | 2 +- hailort/libhailort/hef.proto | 15 + hailort/libhailort/include/hailo/device.hpp | 42 + hailort/libhailort/include/hailo/hailort.h | 124 +- .../include/hailo/hailort_common.hpp | 11 +- .../include/hailo/hailort_dma-heap.h | 55 + hailort/libhailort/include/hailo/hef.hpp | 2 +- .../libhailort/include/hailo/infer_model.hpp | 135 +- .../include/hailo/network_group.hpp | 26 +- hailort/libhailort/include/hailo/stream.hpp | 82 +- hailort/libhailort/include/hailo/vdevice.hpp | 50 + hailort/libhailort/src/CMakeLists.txt | 6 +- hailort/libhailort/src/core_op/CMakeLists.txt | 2 + hailort/libhailort/src/core_op/core_op.cpp | 181 +-- hailort/libhailort/src/core_op/core_op.hpp | 11 +- .../ddr_action_list_buffer_builder.cpp | 37 +- .../ddr_action_list_buffer_builder.hpp | 12 +- .../core_op/resource_manager/cache_buffer.cpp | 109 ++ .../core_op/resource_manager/cache_buffer.hpp | 62 + .../resource_manager/cache_manager.cpp | 331 ++++ .../resource_manager/cache_manager.hpp | 88 ++ .../resource_manager/config_buffer.cpp | 104 +- .../resource_manager/config_buffer.hpp | 4 +- .../resource_manager/intermediate_buffer.cpp | 145 +- .../resource_manager/intermediate_buffer.hpp | 20 +- .../internal_buffer_manager.cpp | 30 +- .../internal_buffer_manager.hpp | 10 +- .../internal_buffer_planner.cpp | 44 +- .../internal_buffer_planner.hpp | 2 +- .../resource_manager/periph_calculator.cpp | 14 +- .../resource_manager/resource_manager.cpp | 346 +++-- .../resource_manager/resource_manager.hpp | 31 +- .../resource_manager_builder.cpp | 557 +++---- .../resource_manager_builder.hpp | 7 +- .../libhailort/src/device_common/control.cpp | 116 +- .../libhailort/src/device_common/control.hpp | 4 + .../src/device_common/control_protocol.cpp | 67 + .../src/device_common/control_protocol.hpp | 10 +- .../src/device_common/d2h_events_parser.cpp | 22 +- .../libhailort/src/device_common/device.cpp | 111 +- .../src/device_common/device_internal.cpp | 112 +- .../src/device_common/device_internal.hpp | 1 + hailort/libhailort/src/eth/eth_device.cpp | 82 +- hailort/libhailort/src/eth/eth_stream.cpp | 96 +- .../libhailort/src/eth/hcp_config_core_op.cpp | 39 + .../libhailort/src/eth/hcp_config_core_op.hpp | 6 + .../src/eth/network_rate_calculator.cpp | 20 +- hailort/libhailort/src/eth/udp.cpp | 5 +- hailort/libhailort/src/hailort.cpp | 34 +- .../src/hef/context_switch_actions.cpp | 238 ++- .../src/hef/context_switch_actions.hpp | 93 +- .../libhailort/src/hef/core_op_metadata.cpp | 158 +- .../libhailort/src/hef/core_op_metadata.hpp | 21 +- hailort/libhailort/src/hef/hef.cpp | 1336 +++++++++-------- hailort/libhailort/src/hef/hef_internal.hpp | 95 +- hailort/libhailort/src/hef/layer_info.hpp | 9 +- .../libhailort/src/net_flow/CMakeLists.txt | 3 + .../src/net_flow/ops/softmax_post_process.cpp | 44 +- .../ops/yolov5_bbox_only_post_process.cpp | 8 +- .../src/net_flow/ops/yolov5_post_process.hpp | 12 +- .../net_flow/ops/yolov5_seg_post_process.cpp | 26 +- .../ops/yolov8_bbox_only_post_process.cpp | 127 ++ .../ops/yolov8_bbox_only_post_process.hpp | 90 ++ .../src/net_flow/ops/yolov8_post_process.cpp | 2 - .../src/net_flow/ops/yolov8_post_process.hpp | 53 +- .../yolov8_bbox_only_op_metadata.hpp | 48 + .../ops_metadata/yolov8_op_metadata.hpp | 12 +- .../net_flow/pipeline/async_infer_runner.cpp | 133 +- .../net_flow/pipeline/async_infer_runner.hpp | 5 +- .../pipeline/async_pipeline_builder.cpp | 552 +++---- .../configured_infer_model_hrpc_client.cpp | 484 ++++++ .../configured_infer_model_hrpc_client.hpp | 151 ++ .../src/net_flow/pipeline/edge_elements.cpp | 47 +- .../src/net_flow/pipeline/edge_elements.hpp | 4 +- .../src/net_flow/pipeline/filter_elements.cpp | 134 +- .../src/net_flow/pipeline/infer_model.cpp | 462 ++++-- .../pipeline/infer_model_hrpc_client.cpp | 149 ++ .../pipeline/infer_model_hrpc_client.hpp | 54 + .../pipeline/infer_model_internal.hpp | 167 ++- .../net_flow/pipeline/multi_io_elements.cpp | 130 +- .../net_flow/pipeline/multi_io_elements.hpp | 4 +- .../src/net_flow/pipeline/pipeline.cpp | 276 ++-- .../src/net_flow/pipeline/pipeline.hpp | 63 +- .../src/net_flow/pipeline/queue_elements.cpp | 6 +- .../src/net_flow/pipeline/queue_elements.hpp | 2 +- .../src/net_flow/pipeline/vstream.cpp | 4 +- .../src/net_flow/pipeline/vstream_builder.cpp | 14 +- .../src/network_group/network_group.cpp | 64 +- .../network_group/network_group_internal.hpp | 23 +- .../src/os/posix/linux/dma_buffer_utils.cpp | 67 +- .../src/os/posix/qnx/dma_buffer_utils.cpp | 15 +- .../src/os/windows/dma_buffer_utils.cpp | 15 +- .../src/service/hailort_rpc_client.cpp | 41 +- .../src/service/network_group_client.cpp | 23 +- .../src/stream_common/nms_stream.cpp | 6 +- .../queued_stream_buffer_pool.cpp | 5 +- .../stream_common/remote_process_stream.cpp | 6 +- .../stream_common/remote_process_stream.hpp | 5 +- .../src/stream_common/stream_internal.cpp | 24 +- .../src/stream_common/stream_internal.hpp | 2 + .../src/stream_common/transfer_common.cpp | 63 +- .../src/stream_common/transfer_common.hpp | 29 +- .../libhailort/src/transform/transform.cpp | 33 +- hailort/libhailort/src/utils/buffer.cpp | 3 + .../libhailort/src/utils/buffer_storage.cpp | 53 +- .../libhailort/src/utils/buffer_storage.hpp | 29 + .../libhailort/src/utils/dma_buffer_utils.hpp | 12 +- .../libhailort/src/utils/hailort_common.cpp | 28 +- .../libhailort/src/utils/hailort_logger.cpp | 25 +- .../libhailort/src/utils/hailort_logger.hpp | 18 + .../libhailort/src/utils/profiler/handler.hpp | 6 +- .../profiler/scheduler_profiler_handler.cpp | 20 +- .../src/utils/thread_safe_queue.hpp | 5 + hailort/libhailort/src/vdevice/CMakeLists.txt | 3 + .../scheduler/scheduled_core_op_state.cpp | 6 +- .../scheduler/scheduled_core_op_state.hpp | 2 +- .../src/vdevice/scheduler/scheduler.cpp | 10 +- .../src/vdevice/scheduler/scheduler.hpp | 2 +- hailort/libhailort/src/vdevice/vdevice.cpp | 174 ++- .../src/vdevice/vdevice_core_op.cpp | 82 +- .../src/vdevice/vdevice_core_op.hpp | 8 + .../src/vdevice/vdevice_hrpc_client.cpp | 155 ++ .../src/vdevice/vdevice_hrpc_client.hpp | 52 + .../src/vdevice/vdevice_internal.hpp | 34 + hailort/libhailort/src/vdma/CMakeLists.txt | 4 + .../src/vdma/channel/boundary_channel.cpp | 243 +-- .../src/vdma/channel/boundary_channel.hpp | 56 +- .../src/vdma/channel/channels_group.cpp | 93 ++ .../src/vdma/channel/channels_group.hpp | 45 + .../vdma/channel/interrupts_dispatcher.cpp | 16 +- .../vdma/channel/interrupts_dispatcher.hpp | 3 + .../src/vdma/channel/transfer_launcher.cpp | 7 +- .../src/vdma/channel/transfer_launcher.hpp | 1 - .../src/vdma/circular_stream_buffer_pool.cpp | 25 +- .../src/vdma/circular_stream_buffer_pool.hpp | 10 +- .../src/vdma/driver/hailort_driver.cpp | 377 +++-- .../src/vdma/driver/hailort_driver.hpp | 100 +- .../src/vdma/driver/os/driver_os_specific.hpp | 3 +- .../os/posix/linux/driver_os_specific.cpp | 68 +- .../driver/os/windows/driver_os_specific.cpp | 47 +- .../src/vdma/integrated/integrated_device.cpp | 57 +- .../src/vdma/integrated/integrated_device.hpp | 10 +- .../src/vdma/memory/continuous_buffer.cpp | 5 + .../src/vdma/memory/continuous_buffer.hpp | 1 + .../src/vdma/memory/continuous_edge_layer.cpp | 4 +- .../src/vdma/memory/continuous_edge_layer.hpp | 2 +- .../src/vdma/memory/descriptor_list.cpp | 105 +- .../src/vdma/memory/descriptor_list.hpp | 76 +- .../src/vdma/memory/dma_able_buffer.cpp | 3 + .../src/vdma/memory/mapped_buffer.cpp | 41 +- .../src/vdma/memory/mapped_buffer.hpp | 6 +- .../src/vdma/memory/sg_edge_layer.cpp | 15 +- .../src/vdma/memory/sg_edge_layer.hpp | 12 +- .../src/vdma/memory/vdma_edge_layer.hpp | 7 +- .../libhailort/src/vdma/pcie/pcie_device.cpp | 9 +- hailort/libhailort/src/vdma/pcie_session.cpp | 177 +++ hailort/libhailort/src/vdma/pcie_session.hpp | 115 ++ .../src/vdma/vdma_config_core_op.cpp | 174 ++- .../src/vdma/vdma_config_core_op.hpp | 26 +- .../src/vdma/vdma_config_manager.cpp | 4 +- hailort/libhailort/src/vdma/vdma_device.cpp | 53 +- hailort/libhailort/src/vdma/vdma_device.hpp | 7 +- hailort/libhailort/src/vdma/vdma_stream.cpp | 140 +- hailort/libhailort/src/vdma/vdma_stream.hpp | 3 + hailort/libhailort/tracer_profiler.proto | 1 + hailort/rpc/hailort_rpc.proto | 1 + hailort/scripts/download_firmware_eth.cmd | 2 +- hailort/scripts/download_firmware_eth.sh | 2 +- hailort/scripts/download_hefs.cmd | 2 +- hailort/scripts/download_hefs.sh | 2 +- 314 files changed, 16018 insertions(+), 5386 deletions(-) delete mode 100644 hailort/drivers/win/include/hailo_pcie_version.h create mode 100644 hailort/hailo_firmware_eula create mode 100644 hailort/hailort_server/CMakeLists.txt create mode 100644 hailort/hailort_server/hailort_server.cpp create mode 100644 hailort/hailort_server/hailort_server.hpp create mode 100644 hailort/hailort_server/hailort_server.sh create mode 100644 hailort/hrpc/CMakeLists.txt create mode 100644 hailort/hrpc/client.cpp create mode 100644 hailort/hrpc/client.hpp create mode 100644 hailort/hrpc/os/pcie/raw_connection_internal.cpp create mode 100644 hailort/hrpc/os/pcie/raw_connection_internal.hpp create mode 100644 hailort/hrpc/os/posix/raw_connection_internal.cpp create mode 100644 hailort/hrpc/os/posix/raw_connection_internal.hpp create mode 100644 hailort/hrpc/os/windows/raw_connection_internal.cpp create mode 100644 hailort/hrpc/os/windows/raw_connection_internal.hpp create mode 100644 hailort/hrpc/raw_connection.cpp create mode 100644 hailort/hrpc/raw_connection.hpp create mode 100644 hailort/hrpc/rpc_connection.cpp create mode 100644 hailort/hrpc/rpc_connection.hpp create mode 100644 hailort/hrpc/server.cpp create mode 100644 hailort/hrpc/server.hpp create mode 100644 hailort/hrpc_protocol/CMakeLists.txt create mode 100644 hailort/hrpc_protocol/rpc.proto create mode 100644 hailort/hrpc_protocol/serializer.cpp create mode 100644 hailort/hrpc_protocol/serializer.hpp create mode 100644 hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_allocator.cpp create mode 100644 hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_allocator.hpp create mode 100644 hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_dmabuf_allocator.cpp create mode 100644 hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_dmabuf_allocator.hpp create mode 100644 hailort/libhailort/bindings/python/platform/hailo_tutorials/notebooks/HRT_3_Inference_Single_Model_Tutorial.ipynb create mode 100644 hailort/libhailort/bindings/python/platform/hailo_tutorials/notebooks/HRT_4_Async_Inference_Multiple_Models_Tutorial.ipynb create mode 100644 hailort/libhailort/bindings/python/src/infer_model_api.cpp create mode 100644 hailort/libhailort/bindings/python/src/infer_model_api.hpp create mode 100644 hailort/libhailort/bindings/python/src/vdevice_api.cpp create mode 100644 hailort/libhailort/include/hailo/hailort_dma-heap.h create mode 100644 hailort/libhailort/src/core_op/resource_manager/cache_buffer.cpp create mode 100644 hailort/libhailort/src/core_op/resource_manager/cache_buffer.hpp create mode 100644 hailort/libhailort/src/core_op/resource_manager/cache_manager.cpp create mode 100644 hailort/libhailort/src/core_op/resource_manager/cache_manager.hpp create mode 100644 hailort/libhailort/src/net_flow/ops/yolov8_bbox_only_post_process.cpp create mode 100644 hailort/libhailort/src/net_flow/ops/yolov8_bbox_only_post_process.hpp create mode 100644 hailort/libhailort/src/net_flow/ops_metadata/yolov8_bbox_only_op_metadata.hpp create mode 100644 hailort/libhailort/src/net_flow/pipeline/configured_infer_model_hrpc_client.cpp create mode 100644 hailort/libhailort/src/net_flow/pipeline/configured_infer_model_hrpc_client.hpp create mode 100644 hailort/libhailort/src/net_flow/pipeline/infer_model_hrpc_client.cpp create mode 100644 hailort/libhailort/src/net_flow/pipeline/infer_model_hrpc_client.hpp create mode 100644 hailort/libhailort/src/vdevice/vdevice_hrpc_client.cpp create mode 100644 hailort/libhailort/src/vdevice/vdevice_hrpc_client.hpp create mode 100644 hailort/libhailort/src/vdma/channel/channels_group.cpp create mode 100644 hailort/libhailort/src/vdma/channel/channels_group.hpp create mode 100644 hailort/libhailort/src/vdma/pcie_session.cpp create mode 100644 hailort/libhailort/src/vdma/pcie_session.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d02330..16ebff3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,16 @@ elseif(CLCACHE_PROGRAM) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CLCACHE_PROGRAM}") endif() +if(WIN32) + find_program(SCCACHE sccache) + if(SCCACHE) + set(CMAKE_C_COMPILER_LAUNCHER ${SCCACHE}) + set(CMAKE_CXX_COMPILER_LAUNCHER ${SCCACHE}) + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT Embedded) + cmake_policy(SET CMP0141 NEW) + endif() +endif() + project(HailoRT) # Prevent in-tree building diff --git a/common/include/context_switch_defs.h b/common/include/context_switch_defs.h index d3ce366..2f67163 100644 --- a/common/include/context_switch_defs.h +++ b/common/include/context_switch_defs.h @@ -122,6 +122,9 @@ typedef enum __attribute__((packed)) { CONTEXT_SWITCH_DEFS__ACTION_TYPE_CHANGE_BOUNDARY_INPUT_BATCH, CONTEXT_SWITCH_DEFS__ACTION_TYPE_PAUSE_VDMA_CHANNEL, CONTEXT_SWITCH_DEFS__ACTION_TYPE_RESUME_VDMA_CHANNEL, + CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_INPUT, + CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_OUTPUT, + CONTEXT_SWITCH_DEFS__ACTION_TYPE_WAIT_FOR_CACHE_UPDATED, /* Must be last */ CONTEXT_SWITCH_DEFS__ACTION_TYPE_COUNT @@ -343,6 +346,15 @@ typedef struct { uint8_t connected_d2h_packed_vdma_channel_id; } CONTEXT_SWITCH_DEFS__activate_ddr_buffer_input_data_t; +typedef struct { + uint8_t packed_vdma_channel_id; + uint8_t stream_index; + uint8_t network_index; + CONTEXT_SWITCH_DEFS__stream_reg_info_t stream_reg_info; + CONTROL_PROTOCOL__host_buffer_info_t host_buffer_info; + uint32_t initial_credit_size; +} CONTEXT_SWITCH_DEFS__activate_cache_input_data_t; + typedef struct { uint8_t packed_vdma_channel_id; uint8_t stream_index; @@ -367,6 +379,14 @@ typedef struct { uint32_t buffered_rows_count; } CONTEXT_SWITCH_DEFS__activate_ddr_buffer_output_data_t; +typedef struct { + uint8_t packed_vdma_channel_id; + uint8_t stream_index; + uint8_t network_index; + CONTEXT_SWITCH_DEFS__stream_reg_info_t stream_reg_info; + CONTROL_PROTOCOL__host_buffer_info_t host_buffer_info; +} CONTEXT_SWITCH_DEFS__activate_cache_output_data_t; + typedef struct { uint8_t packed_vdma_channel_id; CONTROL_PROTOCOL__host_buffer_info_t host_buffer_info; diff --git a/common/include/control_protocol.h b/common/include/control_protocol.h index 60ea8e3..1932eb7 100644 --- a/common/include/control_protocol.h +++ b/common/include/control_protocol.h @@ -81,7 +81,7 @@ extern "C" { /* Value to represent an operation should be performed on all streams. */ #define CONTROL_PROTOCOL__ALL_DATAFLOW_MANAGERS (0xFF) -#define CONTROL_PROTOCOL__MAX_CONTEXT_SIZE (3072) +#define CONTROL_PROTOCOL__MAX_CONTEXT_SIZE (4096) #define CONTROL_PROTOCOL__OPCODES_VARIABLES \ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_IDENTIFY, true, CPU_ID_APP_CPU)\ @@ -160,6 +160,10 @@ extern "C" { CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_SET_SLEEP_STATE, false, CPU_ID_APP_CPU)\ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CHANGE_HW_INFER_STATUS, false, CPU_ID_CORE_CPU)\ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_SIGNAL_DRIVER_DOWN, false, CPU_ID_CORE_CPU)\ + CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_INIT_CACHE_INFO, false, CPU_ID_CORE_CPU)\ + CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_GET_CACHE_INFO, false, CPU_ID_CORE_CPU)\ + CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_UPDATE_CACHE_READ_OFFSET, false, CPU_ID_CORE_CPU)\ + CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_SIGNAL_CACHE_UPDATED, false, CPU_ID_CORE_CPU)\ typedef enum { #define CONTROL_PROTOCOL__OPCODE_X(name, is_critical, cpu_id) name, @@ -971,6 +975,26 @@ typedef struct { #pragma warning(pop) #endif +typedef struct { + uint32_t cache_size; + uint32_t current_read_offset; + int32_t write_offset_delta; +} CONTROL_PROTOCOL__context_switch_cache_info_t; + +typedef struct { + uint32_t cache_info_length; + CONTROL_PROTOCOL__context_switch_cache_info_t cache_info; +} CONTROL_PROTOCOL__context_switch_init_cache_info_request_t; + +typedef struct { + uint32_t cache_info_length; + CONTROL_PROTOCOL__context_switch_cache_info_t cache_info; +} CONTROL_PROTOCOL__context_switch_get_cache_info_response_t; + +typedef struct { + uint32_t read_offset_delta_length; + int32_t read_offset_delta; +} CONTROL_PROTOCOL__context_switch_update_cache_read_offset_request_t; typedef CONTROL_PROTOCOL__read_memory_request_t CONTROL_PROTOCOL__read_user_config_request_t; typedef CONTROL_PROTOCOL__read_memory_response_t CONTROL_PROTOCOL__read_user_config_response_t; @@ -1357,6 +1381,7 @@ typedef union { CONTROL_PROTOCOL__get_overcurrent_state_response_t get_overcurrent_state_response; CONTROL_PROTOCOL__get_hw_consts_response_t get_hw_consts_response; CONTROL_PROTOCOL__change_hw_infer_status_response_t change_hw_infer_status_response; + CONTROL_PROTOCOL__context_switch_get_cache_info_response_t context_switch_get_cache_info_response; // Note: This array is larger than any legal request: // * Functions in this module won't write more than CONTROL_PROTOCOL__MAX_CONTROL_LENGTH bytes @@ -1392,6 +1417,8 @@ typedef union { CONTROL_PROTOCOL__sensor_set_generic_i2c_slave_request_t sensor_set_generic_i2c_slave_request; CONTROL_PROTOCOL__context_switch_set_network_group_header_request_t context_switch_set_network_group_header_request; CONTROL_PROTOCOL__context_switch_set_context_info_request_t context_switch_set_context_info_request; + CONTROL_PROTOCOL__context_switch_init_cache_info_request_t context_switch_init_cache_info_request; + CONTROL_PROTOCOL__context_switch_update_cache_read_offset_request_t context_switch_update_cache_read_offset_request; CONTROL_PROTOCOL__idle_time_set_measurement_request_t idle_time_set_measurement_request; CONTROL_PROTOCOL__download_context_action_list_request_t download_context_action_list_request; CONTROL_PROTOCOL__change_context_switch_status_request_t change_context_switch_status_request; diff --git a/common/include/d2h_events.h b/common/include/d2h_events.h index 5d26cd5..d064823 100644 --- a/common/include/d2h_events.h +++ b/common/include/d2h_events.h @@ -15,6 +15,8 @@ extern "C" { #include "status.h" #include "stdfloat.h" +#pragma pack(push, 1) + /** * @brief The d2h event manager structures relevant in the host */ @@ -59,6 +61,7 @@ typedef enum { HEALTH_MONITOR_CLOCK_CHANGED_EVENT_ID, HW_INFER_MANAGER_INFER_DONE, CONTEXT_SWITCH_RUN_TIME_ERROR, + START_UPDATE_CACHE_OFFSET_ID, D2H_EVENT_ID_COUNT /* Must be last*/ } D2H_EVENT_ID_t; @@ -157,6 +160,12 @@ typedef struct { #define D2H_EVENT_CONTEXT_SWITCH_RUN_TIME_ERROR_EVENT_PARAMETER_COUNT (5) +typedef struct { + uint64_t cache_id_bitmask; +} D2H_EVENT_start_update_cache_offset_message_t; + +#define D2H_EVENT_START_UPDATE_CACHE_OFFSET_PARAMETER_COUNT (1) + /* D2H_EVENT__message_parameters_t should be in the same order as hailo_notification_message_parameters_t */ typedef union { D2H_EVENT_rx_error_event_message_t rx_error_event; @@ -170,6 +179,7 @@ typedef union { D2H_EVENT_health_monitor_clock_changed_event_message_t health_monitor_clock_changed_event; D2H_EVENT_hw_infer_mamager_infer_done_message_t hw_infer_manager_infer_done_event; D2H_EVENT_context_switch_run_time_error_event_message_t context_switch_run_time_error_event; + D2H_EVENT_start_update_cache_offset_message_t start_update_cache_offset_event; } D2H_EVENT__message_parameters_t; typedef struct { @@ -187,6 +197,8 @@ typedef struct { uint8_t buffer[D2H_EVENT_MAX_SIZE]; } D2H_event_buffer_t; +#pragma pack(pop) + /********************************************************************** * Public Functions **********************************************************************/ diff --git a/common/include/firmware_header.h b/common/include/firmware_header.h index baed9a3..e0b2337 100644 --- a/common/include/firmware_header.h +++ b/common/include/firmware_header.h @@ -19,8 +19,7 @@ extern "C" { #define FIRMWARE_HEADER_MAGIC_HAILO8 (0x1DD89DE0) #define FIRMWARE_HEADER_MAGIC_HAILO15 (0xE905DAAB) -// TODO - HRT-11344 : change fw magic to pluto specific -#define FIRMWARE_HEADER_MAGIC_PLUTO (0xE905DAAB) +#define FIRMWARE_HEADER_MAGIC_PLUTO (0xF94739AB) typedef enum { FIRMWARE_HEADER_VERSION_INITIAL = 0, diff --git a/common/include/firmware_status.h b/common/include/firmware_status.h index 1aa6bf5..10fa142 100644 --- a/common/include/firmware_status.h +++ b/common/include/firmware_status.h @@ -413,6 +413,8 @@ Updating rules: FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_HW_INFER_STATE_LENGTH)\ FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_CHANNELS_INFO_LENGTH)\ FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_BATCH_COUNT_LENGTH)\ + FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_CACHE_INFO_LENGTH)\ + FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_READ_OFFSET_DELTA_LENGTH)\ \ FIRMWARE_MODULE__X(FIRMWARE_MODULE__POWER_MEASUREMENT)\ FIRMWARE_STATUS__X(HAILO_POWER_MEASUREMENT_STATUS_POWER_INIT_ERROR)\ @@ -765,6 +767,8 @@ Updating rules: FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_WRITE_DATA_BY_TYPE_ACTION_INVALID_MEMORY_SPACE)\ FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_REACHED_TIMEOUT_WHILE_WAITING_FOR_BATCH_SWITCH_CONTEXT_TO_END)\ FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_INVALID_EXTERNAL_ACTION_LIST_ADDRESS)\ + FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_INVALID_CACHE_SIZE)\ + FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_INVALID_READ_OFFSET_SIZE)\ \ FIRMWARE_MODULE__X(FIRMWARE_MODULE__D2H_EVENT_MANAGER)\ FIRMWARE_STATUS__X(HAILO_D2H_EVENT_MANAGER_STATUS_MESSAGE_HIGH_PRIORITY_QUEUE_CREATE_FAILED)\ diff --git a/hailort/.gitignore b/hailort/.gitignore index 80fa472..1f1d25b 100644 --- a/hailort/.gitignore +++ b/hailort/.gitignore @@ -1,4 +1,3 @@ build/ dist/ /external/ -cmake/external/*/ diff --git a/hailort/CMakeLists.txt b/hailort/CMakeLists.txt index 1f1cbb8..5dcaa06 100644 --- a/hailort/CMakeLists.txt +++ b/hailort/CMakeLists.txt @@ -3,7 +3,6 @@ cmake_minimum_required(VERSION 3.0.0) option(HAILO_BUILD_PYBIND "Build Python binding" OFF) option(HAILO_BUILD_EMULATOR "Build hailort for emulator" OFF) option(HAILO_BUILD_UT "Build Unit Tests" OFF) -option(HAILO_BUILD_DMABUF_TESTS "Build DMA buffer tests. Relevant only if HAILO_BUILD_UT is ON" OFF) option(HAILO_BUILD_HW_DEBUG_TOOL "Build hw debug tool" OFF) option(HAILO_BUILD_GSTREAMER "Compile gstreamer plugins" OFF) option(HAILO_BUILD_EXAMPLES "Build examples" OFF) @@ -12,6 +11,7 @@ option(HAILO_BUILD_SERVICE "Build hailort service" OFF) option(HAILO_BUILD_PROFILER "Build hailort profiler" ON) option(HAILO_COMPILE_WARNING_AS_ERROR "Add compilation flag for treating compilation warnings as errors" OFF) option(HAILO_SUPPORT_PACKAGING "Create HailoRT package (internal)" OFF) +option(HAILO_BUILD_DOC "Build doc" OFF) if (HAILO_COMPILE_WARNING_AS_ERROR) if(WIN32) @@ -31,8 +31,8 @@ endif() # Set firmware version add_definitions( -DFIRMWARE_VERSION_MAJOR=4 ) -add_definitions( -DFIRMWARE_VERSION_MINOR=17 ) -add_definitions( -DFIRMWARE_VERSION_REVISION=1 ) +add_definitions( -DFIRMWARE_VERSION_MINOR=18 ) +add_definitions( -DFIRMWARE_VERSION_REVISION=0 ) if(HAILO_BUILD_SERVICE) add_definitions( -DHAILO_SUPPORT_MULTI_PROCESS ) endif() @@ -57,6 +57,10 @@ set(HAILORT_COMMON_DIR ${PROJECT_SOURCE_DIR}/hailort/) set(COMMON_INC_DIR ${PROJECT_SOURCE_DIR}/common/include) set(DRIVER_INC_DIR ${PROJECT_SOURCE_DIR}/hailort/drivers/common) set(RPC_DIR ${PROJECT_SOURCE_DIR}/hailort/rpc) +set(HRPC_DIR ${PROJECT_SOURCE_DIR}/hailort/hrpc) +set(HRPC_PROTOCOL_DIR ${PROJECT_SOURCE_DIR}/hailort/hrpc_protocol) +set(HAILORT_SERVICE_DIR ${PROJECT_SOURCE_DIR}/hailort/hailort_service) +set(HAILORT_SERVER_DIR ${PROJECT_SOURCE_DIR}/hailort/hailort_server) if(CMAKE_SYSTEM_NAME STREQUAL QNX) include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/pevents.cmake) @@ -67,6 +71,8 @@ if(HAILO_BUILD_SERVICE) endif() add_subdirectory(common) +add_subdirectory(hrpc) +add_subdirectory(hrpc_protocol) add_subdirectory(libhailort) add_subdirectory(hailortcli) if(HAILO_BUILD_HW_DEBUG_TOOL) @@ -89,3 +95,5 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL QNX) add_subdirectory(drivers/qnx) endif() + +add_subdirectory(hailort_server) diff --git a/hailort/LICENSE-3RD-PARTY.md b/hailort/LICENSE-3RD-PARTY.md index 473d5a0..2d2b693 100644 --- a/hailort/LICENSE-3RD-PARTY.md +++ b/hailort/LICENSE-3RD-PARTY.md @@ -14,4 +14,4 @@ | pevents | Mahmoud Al-Qudsi | MIT License | master | Cloned entire package | https://github.com/neosmart/pevents.git | | grpc | Google Inc. | Apache License 2.0 | 1.46.3 | Cloned entire package | https://github.com/grpc/grpc | | stb | Sean Barrett | MIT License | 0.97 | Copied only the file `stb/stb_image_resize.h` | https://github.com/nothings/stb | -| eigen | | Mozilla Public License 2.0 | 3.4.0 | Cloned entire package | https://gitlab.com/libeigen/eigen | \ No newline at end of file +| eigen | | Mozilla Public License 2.0 | 3.4.0 | Cloned entire package | https://gitlab.com/libeigen/eigen | diff --git a/hailort/common/circular_buffer.hpp b/hailort/common/circular_buffer.hpp index e13c9cb..b3f381f 100644 --- a/hailort/common/circular_buffer.hpp +++ b/hailort/common/circular_buffer.hpp @@ -20,18 +20,6 @@ namespace hailort { -typedef struct { - volatile int head; - volatile int tail; - int size; - int size_mask; -} circbuf_t; - -//TODO: Do not change the behavior of this module. see PLDA descs impl.. -//TODO: optimize macros -#ifndef MIN -#define MIN(x,y) (((x) < (y)) ? (x) : (y)) -#endif #ifdef _WIN32 #define _CB_FETCH(x) (InterlockedOr((LONG volatile*)(&x), (LONG)0)) #define _CB_SET(x, value) (InterlockedExchange((LONG volatile*)(&x), (LONG)(value))) @@ -40,25 +28,114 @@ typedef struct { #define _CB_SET(x, value) ((void)__sync_lock_test_and_set(&(x), value)) #endif -#define CB_INIT(circbuf, s) \ - (circbuf).head = 0; \ - (circbuf).tail = 0; \ - (circbuf).size = static_cast(s); \ - (circbuf).size_mask = static_cast((s) - 1) -#define CB_RESET(circbuf) \ - (circbuf).head = 0; \ - (circbuf).tail = 0 -#define CB_HEAD(x) _CB_FETCH((x).head) -#define CB_TAIL(x) _CB_FETCH((x).tail) -#define CB_SIZE(x) _CB_FETCH((x).size) -#define CB_ENQUEUE(circbuf, value) _CB_SET((circbuf).head, ((circbuf).head + (value)) & ((circbuf).size_mask)) -#define CB_DEQUEUE(circbuf, value) _CB_SET((circbuf).tail, ((circbuf).tail + (value)) & ((circbuf).size_mask)) -#define CB_AVAIL(circbuf, head, tail) ((((circbuf).size)-1+(tail)-(head)) & ((circbuf).size_mask)) -#define CB_AVAIL_CONT(circbuf, head, tail) \ - MIN(CB_AVAIL((circbuf), (head), (tail)), (circbuf).size - (head)) -#define CB_PROG(circbuf, head, tail) ((((circbuf).size)+(head)-(tail)) & ((circbuf).size_mask)) -#define CB_PROG_CONT(circbuf, head, tail) \ - MIN(CB_PROG((circbuf), (head), (tail)), (circbuf).size - (tail)) +// Note: We use tag dispatching to select the right implementation for power of 2 size +// There's a minor performance gain for power of 2 size, as we can use a mask instead of modulo +// * If a CircularBuffer/Array with the IsPow2Tag, then the size must be a power of 2. +// * If a CircularBuffer/Array with the IsNotPow2Tag, then the size may be any positive integer (we simply won't +// use the mask optimization for modulo operation, even if the size is a power of 2). +struct IsPow2Tag {}; +struct IsNotPow2Tag {}; +template +struct CircularBuffer +{ +public: + CircularBuffer(int s) : + m_head(0), + m_tail(0), + m_size(s), + m_size_mask(s - 1) + { + check_size(s, Pow2Tag()); + } + + void reset() + { + m_head = 0; + m_tail = 0; + } + + void enqueue(int value) + { + _CB_SET(m_head, modulo(m_head + value, Pow2Tag())); + } + + void set_head(int value) + { + _CB_SET(m_head, value); + } + + void dequeue(int value) + { + _CB_SET(m_tail, modulo(m_tail + value, Pow2Tag())); + } + + void set_tail(int value) + { + _CB_SET(m_tail, value); + } + + int avail(int head, int tail) const + { + return modulo(m_size - 1 + tail - head, Pow2Tag()); + } + + int prog(int head, int tail) const + { + return modulo(m_size + head - tail, Pow2Tag()); + } + + int head() const + { + return _CB_FETCH(m_head); + } + + int tail() const + { + return _CB_FETCH(m_tail); + } + + int size() const + { + return m_size; + } + + int size_mask() const + { + return m_size_mask; + } + +private: + int modulo(int val, IsPow2Tag) const + { + return val & m_size_mask; + } + + int modulo(int val, IsNotPow2Tag) const + { + return val % m_size; + } + + void check_size(size_t size, IsPow2Tag) + { + assert(0 != size); + assert(is_powerof2(size)); + + (void)size; // For release + } + + void check_size(size_t size, IsNotPow2Tag) + { + assert(0 != size); + + (void)size; // For release + } + + volatile int m_head; + volatile int m_tail; + const int m_size; + // For power of 2 size, we can use a mask instead of modulo + const int m_size_mask; +}; template @@ -69,11 +146,10 @@ struct is_std_array> : public std::true_type {}; // TODO: implement more functionalities, better move semantic handle // TODO: support consts methods (front(), empty()), right now CB_* macros requires non const pointer to head+tail -template> +template> class CircularArray final { public: - static_assert(std::is_default_constructible::value, "CircularArray object must be default constructible"); // Based on https://en.cppreference.com/w/cpp/iterator/iterator @@ -85,12 +161,16 @@ public: { public: explicit iterator(int index, CircularArray &array) : m_array(array), m_index(index) {} - iterator& operator++() { m_index = ((m_index + 1) & m_array.m_circ.size_mask); return *this; } + iterator& operator++() { increment(Pow2Tag()); return *this; } iterator operator++(int) { iterator retval = *this; ++(*this); return retval; } bool operator==(iterator other) const { return m_index == other.m_index; } bool operator!=(iterator other) const { return !(*this == other); } T &operator*() const { return m_array.m_array[m_index]; } + private: + void increment(IsPow2Tag) { m_index = (m_index + 1) & m_array.m_circ.size_mask(); } + void increment(IsNotPow2Tag) { m_index = (m_index + 1) % m_array.m_circ.size(); } + CircularArray &m_array; int m_index; }; @@ -98,51 +178,47 @@ public: // Ctor for Container=std::vector template >::value>> - CircularArray(size_t storage_size) + CircularArray(size_t storage_size) : + m_circ(static_cast(storage_size)) { - // storage size must be a power of 2 - assert(is_powerof2(storage_size)); - CB_INIT(m_circ, storage_size); m_array.resize(storage_size); } // Ctor for Container=std::array template ::value>> - CircularArray(size_t storage_size, int = 0) + CircularArray(size_t storage_size, int = 0) : + m_circ(static_cast(storage_size)) { - // storage size must be a power of 2 - assert(is_powerof2(storage_size)); assert(storage_size <= std::tuple_size::value); - CB_INIT(m_circ, storage_size); } void push_back(T &&element) { assert(!full()); - m_array[CB_HEAD(m_circ)] = std::move(element); - CB_ENQUEUE(m_circ, 1); + m_array[m_circ.head()] = std::move(element); + m_circ.enqueue(1); } void push_back(const T& element) { assert(!full()); - m_array[CB_HEAD(m_circ)] = element; - CB_ENQUEUE(m_circ, 1); + m_array[m_circ.head()] = element; + m_circ.enqueue(1); } void pop_front() { assert(!empty()); // Clear previous front - m_array[CB_TAIL(m_circ)] = T(); - CB_DEQUEUE(m_circ, 1); + m_array[m_circ.tail()] = T(); + m_circ.dequeue(1); } T &front() { assert(!empty()); - return m_array[CB_TAIL(m_circ)]; + return m_array[m_circ.tail()]; } void reset() @@ -157,36 +233,36 @@ public: bool empty() const { - return CB_HEAD(m_circ) == CB_TAIL(m_circ); + return m_circ.head() == m_circ.tail(); } bool full() const { - return 0 == CB_AVAIL(m_circ, CB_HEAD(m_circ), CB_TAIL(m_circ)); + return 0 == m_circ.avail(m_circ.head(), m_circ.tail()); } size_t size() const { - return CB_PROG(m_circ, CB_HEAD(m_circ), CB_TAIL(m_circ)); + return m_circ.prog(m_circ.head(), m_circ.tail()); } size_t capacity() const { - return CB_SIZE(m_circ) - 1; + return m_circ.size() - 1; } iterator begin() { - return iterator(CB_TAIL(m_circ), *this); + return iterator(m_circ.tail(), *this); } iterator end() { - return iterator(CB_HEAD(m_circ), *this); + return iterator(m_circ.head(), *this); } private: - circbuf_t m_circ; + CircularBuffer m_circ; Container m_array; }; diff --git a/hailort/common/event_internal.cpp b/hailort/common/event_internal.cpp index 50d0259..3200505 100644 --- a/hailort/common/event_internal.cpp +++ b/hailort/common/event_internal.cpp @@ -46,18 +46,15 @@ WaitOrShutdown::WaitOrShutdown(WaitablePtr waitable, EventPtr shutdown_event) : hailo_status WaitOrShutdown::wait(std::chrono::milliseconds timeout) { - auto index = m_waitable_group.wait_any(timeout); - if (index.status() == HAILO_TIMEOUT) { - return index.status(); - } - CHECK_EXPECTED_AS_STATUS(index); - - assert(index.value() <= WAITABLE_INDEX); - return (index.value() == SHUTDOWN_INDEX) ? HAILO_SHUTDOWN_EVENT_SIGNALED : HAILO_SUCCESS; + TRY_WITH_ACCEPTABLE_STATUS(HAILO_TIMEOUT, const auto index, m_waitable_group.wait_any(timeout)); + assert(index <= WAITABLE_INDEX); + return (index == SHUTDOWN_INDEX) ? HAILO_SHUTDOWN_EVENT_SIGNALED : HAILO_SUCCESS; } hailo_status WaitOrShutdown::signal() { + // Cannot signal a WaitOrShutdown which has only shutdown event + CHECK_NOT_NULL(m_waitable, HAILO_INVALID_OPERATION); return m_waitable->signal(); } @@ -71,7 +68,11 @@ WaitableGroup WaitOrShutdown::create_waitable_group(WaitablePtr waitable, EventP // Note the order - consistent with SHUTDOWN_INDEX, WAITABLE_INDEX. std::vector> waitables; waitables.emplace_back(std::ref(*shutdown_event)); - waitables.emplace_back(std::ref(*waitable)); + + if (nullptr != waitable) { + waitables.emplace_back(std::ref(*waitable)); + } + return waitables; } diff --git a/hailort/common/file_utils.cpp b/hailort/common/file_utils.cpp index 5360484..1b95b32 100644 --- a/hailort/common/file_utils.cpp +++ b/hailort/common/file_utils.cpp @@ -28,7 +28,7 @@ Expected get_istream_size(std::ifstream &s) s.seekg(beg_pos, s.beg); CHECK_AS_EXPECTED(s.good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed"); - auto total_size = static_cast(size - beg_pos); + auto total_size = static_cast(size - beg_pos); CHECK_AS_EXPECTED(total_size <= std::numeric_limits::max(), HAILO_FILE_OPERATION_FAILURE, "File size {} is too big", total_size); return Expected(static_cast(total_size)); @@ -39,16 +39,193 @@ Expected read_binary_file(const std::string &file_path, const BufferStor std::ifstream file(file_path, std::ios::in | std::ios::binary); CHECK_AS_EXPECTED(file.good(), HAILO_OPEN_FILE_FAILURE, "Error opening file {}", file_path); - auto file_size = get_istream_size(file); - CHECK_EXPECTED(file_size, "Failed to get file size"); - - auto buffer = Buffer::create(file_size.value(), output_buffer_params); - CHECK_EXPECTED(buffer, "Failed to allocate file buffer ({} bytes}", file_size.value()); + TRY(const auto file_size, get_istream_size(file), "Failed to get file size"); + TRY(auto buffer, Buffer::create(file_size, output_buffer_params), + "Failed to allocate file buffer ({} bytes}", file_size); // Read the data - file.read(reinterpret_cast(buffer->data()), buffer->size()); + file.read(reinterpret_cast(buffer.data()), buffer.size()); CHECK_AS_EXPECTED(file.good(), HAILO_FILE_OPERATION_FAILURE, "Failed reading file {}", file_path); - return buffer.release(); + return buffer; +} + +Expected> SeekableBytesReader::create_reader(const std::string &file_path) +{ + auto ptr = make_shared_nothrow(file_path); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + return ptr; +} + +Expected> SeekableBytesReader::create_reader(const MemoryView &memview) +{ + auto ptr = make_shared_nothrow(memview); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + return ptr; +} + +FileReader::FileReader(const std::string &file_path) : m_file_path(file_path) {} + +hailo_status FileReader::read(uint8_t *buffer, size_t n) +{ + assert(nullptr != m_fstream); + (void)m_fstream->read(reinterpret_cast(buffer), n); + return m_fstream->good() ? HAILO_SUCCESS : HAILO_FILE_OPERATION_FAILURE; +} + +hailo_status FileReader::read_from_offset(size_t offset, MemoryView &dst, size_t size) +{ + assert(nullptr != m_fstream); + + auto beg_pos = m_fstream->tellg(); + (void)m_fstream->seekg(offset); + CHECK(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed"); + + (void)m_fstream->read(reinterpret_cast(dst.data()), size); + CHECK(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::read() failed"); + + (void)m_fstream->seekg(beg_pos); + CHECK(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed"); + + return HAILO_SUCCESS; +} + +hailo_status FileReader::open() +{ + if (nullptr == m_fstream) { // The first call to open creates the ifstream object + m_fstream = std::make_shared(m_file_path, std::ios::in | std::ios::binary); + return m_fstream->good() ? HAILO_SUCCESS : HAILO_OPEN_FILE_FAILURE; + } + m_fstream->open(m_file_path, std::ios::in | std::ios::binary); + return m_fstream->good() ? HAILO_SUCCESS : HAILO_OPEN_FILE_FAILURE; +} + +bool FileReader::is_open() const +{ + return m_fstream->is_open(); +} + +hailo_status FileReader::seek(size_t position) +{ + assert(nullptr != m_fstream); + (void)m_fstream->seekg(position, m_fstream->beg); + return m_fstream->good() ? HAILO_SUCCESS : HAILO_FILE_OPERATION_FAILURE; +} + +Expected FileReader::tell() +{ + assert(nullptr != m_fstream); + auto offset = m_fstream->tellg(); + return m_fstream->good() ? Expected(static_cast(offset)) : make_unexpected(HAILO_FILE_OPERATION_FAILURE); +} + +hailo_status FileReader::close() +{ + assert(nullptr != m_fstream); + m_fstream->close(); + return m_fstream->good() ? HAILO_SUCCESS : HAILO_CLOSE_FAILURE; +} + +Expected FileReader::get_size() +{ + assert(nullptr != m_fstream); + + auto beg_pos = m_fstream->tellg(); + CHECK_AS_EXPECTED(-1 != beg_pos, HAILO_FILE_OPERATION_FAILURE, "ifstream::tellg() failed"); + + (void)m_fstream->seekg(0, m_fstream->end); + CHECK_AS_EXPECTED(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed"); + + auto file_size = m_fstream->tellg(); + CHECK_AS_EXPECTED(-1 != file_size, HAILO_FILE_OPERATION_FAILURE, "ifstream::tellg() failed"); + + (void)m_fstream->seekg(beg_pos, m_fstream->beg); + CHECK_AS_EXPECTED(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "ifstream::seekg() failed"); + + return static_cast(file_size); +} + +std::shared_ptr FileReader::get_fstream() const +{ + return m_fstream; +} + +Expected FileReader::calculate_remaining_size() +{ + assert(nullptr != m_fstream); + auto remaining_size = get_istream_size(*m_fstream); + CHECK_AS_EXPECTED(m_fstream->good(), HAILO_FILE_OPERATION_FAILURE, "FileReader::calculate_remaining_size() failed"); + return remaining_size; +} + +Expected FileReader::good() const +{ + assert(nullptr != m_fstream); + return m_fstream->good(); +} + +BufferReader::BufferReader(const MemoryView &memview) : m_memview(memview) {} + +hailo_status BufferReader::read(uint8_t *buffer, size_t n) +{ + assert(m_seek_offset + n <= m_memview.size()); + memcpy(buffer, m_memview.data() + m_seek_offset, n); + m_seek_offset += n; + return HAILO_SUCCESS; +} + +hailo_status BufferReader::read_from_offset(size_t offset, MemoryView &dst, size_t size) +{ + memcpy(dst.data(), m_memview.data() + offset, size); + return HAILO_SUCCESS; +} + +hailo_status BufferReader::open() +{ + // In case we use the buffer, we don't need to check if the file is open + return HAILO_SUCCESS; +} + +bool BufferReader::is_open() const +{ + // In case we use the buffer, we don't need to check if the file is open + return true; +} + +hailo_status BufferReader::seek(size_t position) +{ + assert(position < m_memview.size()); + m_seek_offset = position; + return HAILO_SUCCESS; +} + +Expected BufferReader::tell() +{ + return Expected(m_seek_offset); +} + +hailo_status BufferReader::close() +{ + return HAILO_SUCCESS; +} + +Expected BufferReader::get_size() +{ + return Expected(m_memview.size()); +} + +Expected BufferReader::calculate_remaining_size() +{ + return m_memview.size() - m_seek_offset; +} + +Expected BufferReader::good() const +{ + return true; +} + +const MemoryView BufferReader::get_memview() const +{ + return m_memview; } } /* namespace hailort */ diff --git a/hailort/common/file_utils.hpp b/hailort/common/file_utils.hpp index 028c888..fbcbe31 100644 --- a/hailort/common/file_utils.hpp +++ b/hailort/common/file_utils.hpp @@ -27,6 +27,74 @@ Expected get_istream_size(std::ifstream &s); Expected read_binary_file(const std::string &file_path, const BufferStorageParams &output_buffer_params = {}); +class FileReader; +class BufferReader; + +class SeekableBytesReader +{ +public: + virtual ~SeekableBytesReader() = default; + virtual hailo_status read(uint8_t *buffer, size_t n) = 0; + virtual hailo_status read_from_offset(size_t offset, MemoryView &dst, size_t n) = 0; + virtual hailo_status open() = 0; + virtual bool is_open() const = 0; + virtual hailo_status seek(size_t position) = 0; + virtual Expected tell() = 0; + virtual hailo_status close() = 0; + virtual Expected get_size() = 0; + virtual Expected good() const = 0; + virtual Expected calculate_remaining_size() = 0; + static Expected> create_reader(const std::string &file_path); + static Expected> create_reader(const MemoryView &memview); +}; + +class FileReader : public SeekableBytesReader +{ +public: + FileReader(const std::string &file_path); + + virtual hailo_status read(uint8_t *buffer, size_t n); + virtual hailo_status read_from_offset(size_t offset, MemoryView &dst, size_t n); + virtual hailo_status open(); + virtual bool is_open() const; + virtual hailo_status seek(size_t position); + virtual Expected tell(); + virtual hailo_status close(); + virtual Expected get_size(); + virtual Expected good() const; + virtual Expected calculate_remaining_size(); + + std::shared_ptr get_fstream() const; + +private: + std::shared_ptr m_fstream = nullptr; + std::string m_file_path; + +}; + +class BufferReader : public SeekableBytesReader +{ +public: + BufferReader(const MemoryView &memview); + + virtual hailo_status read(uint8_t *buffer, size_t n); + virtual hailo_status read_from_offset(size_t offset, MemoryView &dst, size_t n); + virtual hailo_status open(); + virtual bool is_open() const; + virtual hailo_status seek(size_t position); + virtual Expected tell(); + virtual hailo_status close(); + virtual Expected get_size(); + virtual Expected good() const; + virtual Expected calculate_remaining_size(); + + const MemoryView get_memview() const; + +private: + MemoryView m_memview; + size_t m_seek_offset = 0; +}; + } /* namespace hailort */ #endif /* _HAILO_FILE_UTILS_HPP_ */ diff --git a/hailort/common/filesystem.hpp b/hailort/common/filesystem.hpp index b650b46..360cdfd 100644 --- a/hailort/common/filesystem.hpp +++ b/hailort/common/filesystem.hpp @@ -88,7 +88,7 @@ private: FindFile &operator=(FindFile &&other) = delete; FindFile(FindFile &&other); - Filesystem::FileInfo get_cur_file_info(); + Filesystem::FileInfo get_cur_file_info() const; // Will return HAILO_INVALID_OPERATION when the iteration is complete or HAILO_FILE_OPERATION_FAILURE upon failure hailo_status next_file(); diff --git a/hailort/common/logger_macros.hpp b/hailort/common/logger_macros.hpp index bd31151..544c619 100644 --- a/hailort/common/logger_macros.hpp +++ b/hailort/common/logger_macros.hpp @@ -24,8 +24,15 @@ #endif #endif +#if defined(__linux__) && !defined(__ANDROID__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overflow" +#endif #include #include +#if defined(__linux__) && !defined(__ANDROID__) +#pragma GCC diagnostic pop +#endif inline std::ostream& operator<<(std::ostream& os, const hailo_status& status) { diff --git a/hailort/common/os/posix/ethernet_utils.cpp b/hailort/common/os/posix/ethernet_utils.cpp index f675ec5..556ead1 100644 --- a/hailort/common/os/posix/ethernet_utils.cpp +++ b/hailort/common/os/posix/ethernet_utils.cpp @@ -75,13 +75,12 @@ Expected EthernetUtils::get_ip_from_interface(const std::string &in struct ifreq ifr = {}; /* Create socket */ - auto socket = Socket::create(AF_INET, SOCK_DGRAM, 0); - CHECK_EXPECTED(socket); + TRY(const auto socket, Socket::create(AF_INET, SOCK_DGRAM, 0)); /* Convert interface name to ip address */ ifr.ifr_addr.sa_family = AF_INET; (void)strncpy(ifr.ifr_name, interface_name.c_str(), IFNAMSIZ-1); - auto posix_rc = ioctl(socket->get_fd(), SIOCGIFADDR, &ifr); + auto posix_rc = ioctl(socket.get_fd(), SIOCGIFADDR, &ifr); CHECK_AS_EXPECTED(posix_rc >= 0, HAILO_ETH_INTERFACE_NOT_FOUND, "Interface was not found. ioctl with SIOCGIFADDR has failed. errno: {:#x}", errno); diff --git a/hailort/common/os/posix/filesystem.cpp b/hailort/common/os/posix/filesystem.cpp index 57a12ac..51d2342 100644 --- a/hailort/common/os/posix/filesystem.cpp +++ b/hailort/common/os/posix/filesystem.cpp @@ -62,12 +62,11 @@ Expected> Filesystem::get_files_in_dir_flat(const std:: { const std::string dir_path_with_sep = has_suffix(dir_path, SEPARATOR) ? dir_path : dir_path + SEPARATOR; - auto dir = DirWalker::create(dir_path_with_sep); - CHECK_EXPECTED(dir); + TRY(auto dir, DirWalker::create(dir_path_with_sep)); std::vector files; struct dirent *entry = nullptr; - while ((entry = dir->next_file()) != nullptr) { + while ((entry = dir.next_file()) != nullptr) { if (entry->d_type != DT_REG) { continue; } @@ -106,21 +105,19 @@ Expected> Filesystem::get_latest_files_in_dir_flat(cons std::time_t curr_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); const std::string dir_path_with_sep = has_suffix(dir_path, SEPARATOR) ? dir_path : dir_path + SEPARATOR; - auto dir = DirWalker::create(dir_path_with_sep); - CHECK_EXPECTED(dir); + TRY(auto dir, DirWalker::create(dir_path_with_sep)); std::vector files; struct dirent *entry = nullptr; - while ((entry = dir->next_file()) != nullptr) { + while ((entry = dir.next_file()) != nullptr) { if (entry->d_type != DT_REG) { continue; } const std::string file_path = dir_path_with_sep + std::string(entry->d_name); - auto file_modified_time = get_file_modified_time(file_path); - CHECK_EXPECTED(file_modified_time); + TRY(const auto file_modified_time, get_file_modified_time(file_path)); - auto time_diff_sec = std::difftime(curr_time, file_modified_time.value()); + auto time_diff_sec = std::difftime(curr_time, file_modified_time); auto time_diff_millisec = time_diff_sec * 1000; if (time_diff_millisec <= static_cast(time_interval.count())) { files.emplace_back(file_path); diff --git a/hailort/common/os/posix/process.cpp b/hailort/common/os/posix/process.cpp index 808a2e1..ca10204 100644 --- a/hailort/common/os/posix/process.cpp +++ b/hailort/common/os/posix/process.cpp @@ -17,12 +17,10 @@ namespace hailort Expected> Process::create_and_wait_for_output(const std::string &command_line, uint32_t max_output_size) { - auto popen_expected = PopenWrapper::create(command_line); - CHECK_EXPECTED(popen_expected); - const auto output_expected = popen_expected->read_stdout(max_output_size); - CHECK_EXPECTED(output_expected); - const auto process_exit_code = popen_expected->close(); - return std::make_pair(process_exit_code, output_expected.value()); + TRY(auto popen, PopenWrapper::create(command_line)); + TRY(const auto output, popen.read_stdout(max_output_size)); + const auto process_exit_code = popen.close(); + return std::make_pair(process_exit_code, output); } Expected Process::PopenWrapper::create(const std::string &command_line) @@ -61,15 +59,14 @@ Expected Process::PopenWrapper::read_stdout(uint32_t max_output_siz { assert (nullptr != m_pipe); - // We zero out the bufer so that output won't contain junk from the heap - auto output = Buffer::create(max_output_size, 0); - CHECK_EXPECTED(output); + // We zero out the buffer so that output won't contain junk from the heap + TRY(auto output, Buffer::create(max_output_size, 0)); - const auto num_read = fread(reinterpret_cast(output->data()), sizeof(uint8_t), output->size(), m_pipe); - if (num_read != output->size()) { + const auto num_read = fread(reinterpret_cast(output.data()), sizeof(uint8_t), output.size(), m_pipe); + if (num_read != output.size()) { if (feof(m_pipe)) { // We remove the trailing newline we get from fread - const auto output_as_str = output->to_string(); + const auto output_as_str = output.to_string(); if (output_as_str[output_as_str.length() - 1] == '\n') { return output_as_str.substr(0, num_read - 1); } @@ -81,7 +78,7 @@ Expected Process::PopenWrapper::read_stdout(uint32_t max_output_siz } else { // Truncate output LOGGER__TRACE("Truncating output to {} chars long", max_output_size); - return output->to_string(); + return output.to_string(); } } diff --git a/hailort/common/os/posix/socket.cpp b/hailort/common/os/posix/socket.cpp index 0260e9b..6f2fb61 100644 --- a/hailort/common/os/posix/socket.cpp +++ b/hailort/common/os/posix/socket.cpp @@ -34,13 +34,10 @@ hailo_status Socket::SocketModuleWrapper::free_module() Expected Socket::create(int af, int type, int protocol) { - auto module_wrapper = SocketModuleWrapper::create(); - CHECK_EXPECTED(module_wrapper); - - auto socket_fd = create_socket_fd(af, type, protocol); - CHECK_EXPECTED(socket_fd); + TRY(auto module_wrapper, SocketModuleWrapper::create()); + TRY(const auto socket_fd, create_socket_fd(af, type, protocol)); - auto obj = Socket(module_wrapper.release(), socket_fd.release()); + auto obj = Socket(std::move(module_wrapper), socket_fd); return obj; } diff --git a/hailort/common/os/posix/traffic_control.cpp b/hailort/common/os/posix/traffic_control.cpp index 71aa3f4..e80381f 100644 --- a/hailort/common/os/posix/traffic_control.cpp +++ b/hailort/common/os/posix/traffic_control.cpp @@ -22,17 +22,12 @@ namespace hailort Expected TrafficControlUtil::create(const std::string &ip, uint16_t port, uint32_t rate_bytes_per_sec) { - auto interface_name = EthernetUtils::get_interface_from_board_ip(ip); - CHECK_EXPECTED(interface_name, "get_interface_name failed with status {}", interface_name.status()); + TRY(const auto interface_name, EthernetUtils::get_interface_from_board_ip(ip), "get_interface_name failed"); + TRY(const auto board_id, ip_to_board_id(ip), "ip_to_board_id failed"); + TRY(const auto is_sudo_needed, check_is_sudo_needed(), "check_is_sudo_needed failed"); - auto board_id = ip_to_board_id(ip); - CHECK_EXPECTED(board_id, "ip_to_board_id failed with status {}", board_id.status()); - - auto is_sudo_needed = check_is_sudo_needed(); - CHECK_EXPECTED(is_sudo_needed, "check_is_sudo_needed failed with status {}", is_sudo_needed.status()); - - return TrafficControlUtil(ip, interface_name.release(), board_id.release(), port, port_to_port_id(port), - rate_bytes_per_sec, is_sudo_needed.release()); + return TrafficControlUtil(ip, interface_name, board_id, port, port_to_port_id(port), + rate_bytes_per_sec, is_sudo_needed); } TrafficControlUtil::TrafficControlUtil(const std::string& board_address, const std::string& interface_name, @@ -185,28 +180,26 @@ uint16_t TrafficControlUtil::port_to_port_id(uint16_t port) Expected TrafficControlUtil::check_is_sudo_needed() { - const auto result = Process::create_and_wait_for_output("id -u", MAX_COMMAND_OUTPUT_LENGTH); - CHECK_EXPECTED(result); + TRY(const auto result, Process::create_and_wait_for_output("id -u", MAX_COMMAND_OUTPUT_LENGTH)); // If the user id is zero then we don't need to add `sudo` to our commands - return std::move(result->second != "0"); + return std::move(result.second != "0"); } hailo_status TrafficControlUtil::run_command(const std::string &commnad, bool add_sudo, const std::vector &allowed_errors, bool ignore_fails) { // Note: we redirect stderr to stdout - const auto result = Process::create_and_wait_for_output( + TRY(const auto result, Process::create_and_wait_for_output( add_sudo ? "sudo " + commnad + " 2>&1" : commnad + " 2>&1", - MAX_COMMAND_OUTPUT_LENGTH); - CHECK_EXPECTED_AS_STATUS(result); + MAX_COMMAND_OUTPUT_LENGTH)); - const uint32_t exit_code = result->first; + const uint32_t exit_code = result.first; if (0 == exit_code) { return HAILO_SUCCESS; } - std::string cmd_output = result->second; + std::string cmd_output = result.second; // No output = everything was OK bool is_output_valid = cmd_output.empty(); if ((!is_output_valid) && (!allowed_errors.empty())) { @@ -225,11 +218,10 @@ hailo_status TrafficControlUtil::run_command(const std::string &commnad, bool ad Expected TrafficControl::create(const std::string &ip, uint16_t port, uint32_t rate_bytes_per_sec) { - auto tc_util = TrafficControlUtil::create(ip, port, rate_bytes_per_sec); - CHECK_EXPECTED(tc_util); + TRY(auto tc_util, TrafficControlUtil::create(ip, port, rate_bytes_per_sec)); hailo_status rate_set_status = HAILO_UNINITIALIZED; - TrafficControl tc(tc_util.release(), rate_set_status); + TrafficControl tc(std::move(tc_util), rate_set_status); CHECK_SUCCESS_AS_EXPECTED(rate_set_status, "Failed setting rate limit with status {}", rate_set_status); return tc; diff --git a/hailort/common/os/windows/ethernet_utils.cpp b/hailort/common/os/windows/ethernet_utils.cpp index 4dcfd3f..9f4fa86 100644 --- a/hailort/common/os/windows/ethernet_utils.cpp +++ b/hailort/common/os/windows/ethernet_utils.cpp @@ -54,10 +54,9 @@ Expected NetworkInterface::get_all_interfaces() return make_unexpected(HAILO_UNEXPECTED_INTERFACE_INFO_FAILURE); } - auto interface_info_buffer = Buffer::create(required_size, 0); - CHECK_EXPECTED(interface_info_buffer); + TRY(auto interface_info_buffer, Buffer::create(required_size, 0)); ret_value = GetAdaptersAddresses(IPV4, UNICAST_ONLY, RESERVED, - interface_info_buffer->as_pointer(), &required_size); + interface_info_buffer.as_pointer(), &required_size); if (ret_value == ERROR_NO_DATA) { LOGGER__ERROR("No IPv4 interfaces found"); return make_unexpected(HAILO_NO_IPV4_INTERFACES_FOUND); @@ -67,7 +66,7 @@ Expected NetworkInterface::get_all_interfaces() } NetworkInterfaces interfaces; - PIP_ADAPTER_ADDRESSES interface_info = interface_info_buffer->as_pointer(); + PIP_ADAPTER_ADDRESSES interface_info = interface_info_buffer.as_pointer(); while (interface_info != nullptr) { PIP_ADAPTER_UNICAST_ADDRESS first_unicast_address = interface_info->FirstUnicastAddress; @@ -83,10 +82,9 @@ Expected NetworkInterface::get_all_interfaces() continue; } - auto ip = Buffer::create(IPV4_STRING_MAX_LENGTH); - CHECK_EXPECTED(ip); + TRY(auto ip, Buffer::create(IPV4_STRING_MAX_LENGTH)); const auto result = Socket::ntop(AF_INET, &(reinterpret_cast(address_struct)->sin_addr), - ip->as_pointer(), EthernetUtils::MAX_INTERFACE_SIZE); + ip.as_pointer(), EthernetUtils::MAX_INTERFACE_SIZE); if (result != HAILO_SUCCESS) { LOGGER__DEBUG("Failed converting unicast address to string (result={}). Skipping.", result); continue; @@ -98,7 +96,7 @@ Expected NetworkInterface::get_all_interfaces() continue; } interfaces.emplace_back(interface_info->IfIndex, interface_info->AdapterName, - friendly_name_ansi.value(), ip->to_string()); + friendly_name_ansi.value(), ip.to_string()); interface_info = interface_info->Next; } @@ -130,9 +128,8 @@ Expected ArpTable::create(uint32_t interface_index) return make_unexpected(HAILO_UNEXPECTED_ARP_TABLE_FAILURE); } - auto ip_net_table_buffer = Buffer::create(required_size, 0); - CHECK_EXPECTED(ip_net_table_buffer); - ret_value = GetIpNetTable(ip_net_table_buffer->as_pointer(), &required_size, SORTED); + TRY(auto ip_net_table_buffer, Buffer::create(required_size, 0)); + ret_value = GetIpNetTable(ip_net_table_buffer.as_pointer(), &required_size, SORTED); if (ret_value == ERROR_NO_DATA) { LOGGER__ERROR("No IPv4 interfaces found"); return make_unexpected(HAILO_NO_IPV4_INTERFACES_FOUND); @@ -142,7 +139,7 @@ Expected ArpTable::create(uint32_t interface_index) } std::unordered_map result; - const PMIB_IPNETTABLE ip_net_table = ip_net_table_buffer->as_pointer(); + const PMIB_IPNETTABLE ip_net_table = ip_net_table_buffer.as_pointer(); for (uint32_t i = 0; i < ip_net_table->dwNumEntries; i++) { if (ip_net_table->table[i].dwIndex != interface_index) { continue; @@ -162,18 +159,15 @@ Expected ArpTable::create(uint32_t interface_index) Expected EthernetUtils::get_interface_from_board_ip(const std::string &board_ip) { - auto network_interfaces = NetworkInterface::get_all_interfaces(); - CHECK_EXPECTED(network_interfaces); + TRY(const auto network_interfaces, NetworkInterface::get_all_interfaces()); struct in_addr board_ip_struct{}; auto status = Socket::pton(AF_INET, board_ip.c_str(), &board_ip_struct); CHECK_SUCCESS_AS_EXPECTED(status, "Invalid board ip address {}", board_ip); - for (const auto& network_interface : network_interfaces.value()) { - auto arp_table = ArpTable::create(network_interface.index()); - CHECK_EXPECTED(arp_table); - - const auto mac_address = arp_table->get_mac_address(static_cast(board_ip_struct.S_un.S_addr)); + for (const auto &network_interface : network_interfaces) { + TRY(const auto arp_table, ArpTable::create(network_interface.index())); + const auto mac_address = arp_table.get_mac_address(static_cast(board_ip_struct.S_un.S_addr)); if (mac_address) { return network_interface.friendly_name(); } @@ -184,10 +178,9 @@ Expected EthernetUtils::get_interface_from_board_ip(const std::stri Expected EthernetUtils::get_ip_from_interface(const std::string &interface_name) { - auto network_interfaces = NetworkInterface::get_all_interfaces(); - CHECK_EXPECTED(network_interfaces); + TRY(const auto network_interfaces, NetworkInterface::get_all_interfaces()); - for (const auto& network_interface : network_interfaces.value()) { + for (const auto &network_interface : network_interfaces) { if (network_interface.friendly_name() == interface_name) { return network_interface.ip(); } diff --git a/hailort/common/os/windows/filesystem.cpp b/hailort/common/os/windows/filesystem.cpp index 6dbcc25..73b9840 100644 --- a/hailort/common/os/windows/filesystem.cpp +++ b/hailort/common/os/windows/filesystem.cpp @@ -63,7 +63,7 @@ Filesystem::FindFile::FindFile(FindFile &&other) : m_find_data(other.m_find_data) {} -Filesystem::FileInfo Filesystem::FindFile::get_cur_file_info() +Filesystem::FileInfo Filesystem::FindFile::get_cur_file_info() const { return {m_find_data.cFileName, m_find_data.dwFileAttributes}; } @@ -93,18 +93,17 @@ Expected> Filesystem::get_files_in_dir_flat(const std:: { const std::string dir_path_with_sep = has_suffix(dir_path, SEPARATOR) ? dir_path : dir_path + SEPARATOR; - auto dir = FindFile::create(dir_path_with_sep); - CHECK_EXPECTED(dir); + TRY(auto dir, FindFile::create(dir_path_with_sep)); std::vector files; - auto file_info = dir->get_cur_file_info(); + auto file_info = dir.get_cur_file_info(); if (is_regular_or_readonly_file(file_info.attrs)) { files.emplace_back(file_info.path); } hailo_status status = HAILO_UNINITIALIZED; while (true) { - status = dir->next_file(); + status = dir.next_file(); if (HAILO_INVALID_OPERATION == status) { // We're done break; @@ -115,7 +114,7 @@ Expected> Filesystem::get_files_in_dir_flat(const std:: continue; } - file_info = dir->get_cur_file_info(); + file_info = dir.get_cur_file_info(); if (is_regular_or_readonly_file(file_info.attrs)) { files.emplace_back(dir_path_with_sep + file_info.path); } diff --git a/hailort/common/os/windows/socket.cpp b/hailort/common/os/windows/socket.cpp index 1c7789f..c21160e 100644 --- a/hailort/common/os/windows/socket.cpp +++ b/hailort/common/os/windows/socket.cpp @@ -40,13 +40,10 @@ hailo_status Socket::SocketModuleWrapper::free_module() Expected Socket::create(int af, int type, int protocol) { - auto module_wrapper = SocketModuleWrapper::create(); - CHECK_EXPECTED(module_wrapper); - - auto socket_fd = create_socket_fd(af, type, protocol); - CHECK_EXPECTED(socket_fd); + TRY(auto module_wrapper, SocketModuleWrapper::create()); + TRY(const auto socket_fd, create_socket_fd(af, type, protocol)); - auto obj = Socket(module_wrapper.release(), socket_fd.release()); + auto obj = Socket(std::move(module_wrapper), socket_fd); return std::move(obj); } @@ -118,8 +115,7 @@ hailo_status Socket::ntop(int af, const void *src, char *dst, socklen_t size) CHECK_ARG_NOT_NULL(src); CHECK_ARG_NOT_NULL(dst); - auto module_wrapper = SocketModuleWrapper::create(); - CHECK_EXPECTED_AS_STATUS(module_wrapper); + TRY(const auto module_wrapper, SocketModuleWrapper::create()); const char *inet_result = inet_ntop(af, src, dst, size); CHECK(nullptr != inet_result, HAILO_ETH_FAILURE, "Failed inet_ntop. WSALE={}", WSAGetLastError()); @@ -134,8 +130,7 @@ hailo_status Socket::pton(int af, const char *src, void *dst) CHECK_ARG_NOT_NULL(src); CHECK_ARG_NOT_NULL(dst); - auto module_wrapper = SocketModuleWrapper::create(); - CHECK_EXPECTED_AS_STATUS(module_wrapper); + TRY(const auto module_wrapper, SocketModuleWrapper::create()); inet_result = inet_pton(af, src, dst); if (1 != inet_result) { diff --git a/hailort/common/socket.hpp b/hailort/common/socket.hpp index afe0afd..c70df32 100644 --- a/hailort/common/socket.hpp +++ b/hailort/common/socket.hpp @@ -42,7 +42,7 @@ public: m_module_wrapper(std::move(other.m_module_wrapper)), m_socket_fd(std::exchange(other.m_socket_fd, INVALID_SOCKET)) {}; - socket_t get_fd() { return m_socket_fd; } + socket_t get_fd() const { return m_socket_fd; } static hailo_status ntop(int af, const void *src, char *dst, socklen_t size); static hailo_status pton(int af, const char *src, void *dst); diff --git a/hailort/common/string_utils.cpp b/hailort/common/string_utils.cpp index ee4ff47..d54031d 100644 --- a/hailort/common/string_utils.cpp +++ b/hailort/common/string_utils.cpp @@ -63,13 +63,12 @@ Expected StringUtils::to_int32(const std::string &str, int base) Expected StringUtils::to_uint8(const std::string &str, int base) { - auto number = to_uint32(str, base); - CHECK_EXPECTED(number); + TRY(const auto number, to_uint32(str, base)); - CHECK_AS_EXPECTED(((number.value() >= std::numeric_limits::min()) && (number.value() <= std::numeric_limits::max())), + CHECK_AS_EXPECTED(((number >= std::numeric_limits::min()) && (number <= std::numeric_limits::max())), HAILO_INVALID_ARGUMENT, "Failed to convert string {} to uint8_t.", str); - return static_cast(number.value()); + return static_cast(number); } std::string StringUtils::to_hex_string(const uint8_t *array, size_t size, bool uppercase, const std::string &delimiter) diff --git a/hailort/common/utils.hpp b/hailort/common/utils.hpp index 8ffc09d..3eb719e 100644 --- a/hailort/common/utils.hpp +++ b/hailort/common/utils.hpp @@ -14,6 +14,7 @@ #include "hailo/hailort.h" #include "hailo/expected.hpp" +#include "hailo/buffer.hpp" #include "common/logger_macros.hpp" #include @@ -22,6 +23,9 @@ #include #include #include +#include +#include +#include namespace hailort @@ -31,6 +35,10 @@ namespace hailort #define IS_FIT_IN_UINT16(number) ((std::numeric_limits::max() >= ((int32_t)(number))) && (std::numeric_limits::min() <= ((int32_t)(number)))) #define IS_FIT_IN_UINT32(number) ((std::numeric_limits::max() >= ((int64_t)(number))) && (std::numeric_limits::min() <= ((int64_t)(number)))) +static const uint32_t POLYNOMIAL = 0xEDB88320; + +static const size_t MB = 1024 * 1024; + template static inline bool contains(const std::vector &container, const T &value) { @@ -264,6 +272,10 @@ inline hailo_status get_status(const Expected &exp) #define CHECK_GRPC_STATUS_AS_EXPECTED(status) _CHECK_GRPC_STATUS(status, make_unexpected(HAILO_RPC_FAILED), SERVICE_WARNING_MSG) #endif +// Macros that check status. If status is 'valid_error', return without printing error to the prompt. +#define CHECK_EXPECTED_WITH_ACCEPTABLE_STATUS(valid_error, exp, ...) if (valid_error == (exp).status()) {return make_unexpected(valid_error);} CHECK_SUCCESS(exp, __VA_ARGS__); + + #define __HAILO_CONCAT(x, y) x ## y #define _HAILO_CONCAT(x, y) __HAILO_CONCAT(x, y) @@ -272,6 +284,11 @@ inline hailo_status get_status(const Expected &exp) CHECK_EXPECTED(expected_var_name, __VA_ARGS__); \ var_decl = expected_var_name.release() +#define _TRY_V(expected_var_name, var_decl, expr, ...) \ + auto expected_var_name = (expr); \ + CHECK_EXPECTED(expected_var_name, __VA_ARGS__); \ + var_decl = expected_var_name.value() + /** * The TRY macro is used to allow easier validation and access for variables returned as Expected. * If the expression returns an Expected with status HAILO_SUCCESS, the macro will release the expected and assign @@ -289,6 +306,19 @@ inline hailo_status get_status(const Expected &exp) */ #define TRY(var_decl, expr, ...) _TRY(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__) +/** + * Same us TRY macro but instead of returning released value, it will return the value itself. +*/ +// TODO: HRT-13624: Remove after 'expected' implementation is fixed +#define TRY_V(var_decl, expr, ...) _TRY_V(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__) + +#define _TRY_WITH_ACCEPTABLE_STATUS(valid_error, expected_var_name, var_decl, expr, ...) \ + auto expected_var_name = (expr); \ + CHECK_EXPECTED_WITH_ACCEPTABLE_STATUS(valid_error, expected_var_name, __VA_ARGS__); \ + var_decl = expected_var_name.release() + +#define TRY_WITH_ACCEPTABLE_STATUS(valid_error, var_decl, expr, ...) _TRY_WITH_ACCEPTABLE_STATUS(valid_error, _HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__) + #ifndef _MSC_VER #define IGNORE_DEPRECATION_WARNINGS_BEGIN _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") @@ -343,6 +373,124 @@ static inline bool is_env_variable_on(const char* env_var_name, const std::strin return ((nullptr != env_var) && (strncmp(env_var, required_value.c_str(), required_value.size()) == 0)); } +static inline Expected get_env_variable(const std::string &env_var_name) +{ + const auto env_var = std::getenv(env_var_name.c_str()); + // Using ifs instead of CHECKs to avoid printing the error message + if (nullptr == env_var) { + return make_unexpected(HAILO_NOT_FOUND); + } + + const auto result = std::string(env_var); + if (result.empty()) { + return make_unexpected(HAILO_NOT_FOUND); + } + + return Expected(result); +} + +class CRC32 { +public: + CRC32() { + generate_table(); + } + + uint32_t calculate(std::ifstream &s, size_t buffer_size) const { + auto beg_pos = s.tellg(); // Saves current position + uint32_t crc = 0xFFFFFFFF; + std::vector buffer(MB); + + size_t total_bytes_read = 0; + + while (total_bytes_read < buffer_size) { + size_t bytes_to_read = std::min(buffer_size - total_bytes_read, MB); + s.read(buffer.data(), bytes_to_read); + + size_t bytes_read = s.gcount(); + total_bytes_read += bytes_read; + for (size_t i = 0; i < bytes_read; ++i) { + crc = (crc >> 8) ^ table[(crc ^ static_cast(buffer[i])) & 0xFF]; + } + } + + s.seekg(beg_pos, std::ios::beg); // Return to the original position + return crc ^ 0xFFFFFFFF; + } + + uint32_t calculate(const MemoryView &buffer) const { + uint32_t crc = 0xFFFFFFFF; + auto data = buffer.data(); + + for (size_t i = 0; i < buffer.size(); ++i) { + crc = (crc >> 8) ^ table[(crc ^ data[i]) & 0xFF]; + } + + return crc ^ 0xFFFFFFFF; + } + + static Expected calc_crc_on_buffer(const MemoryView &buffer) + { + CRC32 crcCalculator; + return crcCalculator.calculate(buffer); + } + + static Expected calc_crc_on_stream(std::ifstream &s, size_t size) + { + CRC32 crcCalculator; + return crcCalculator.calculate(s, size); + } + +private: + uint32_t table[256]; + + void generate_table() { + for (uint32_t i = 0; i < 256; ++i) { + uint32_t crc = i; + for (uint32_t j = 0; j < 8; ++j) { + crc = (crc & 1) ? (crc >> 1) ^ POLYNOMIAL : (crc >> 1); + } + table[i] = crc; + } + } +}; + +class BufferUtils final +{ +public: + BufferUtils() = delete; + + static void summarize_buffer(const Buffer& buffer, std::ostream& os) + { + os << "Buffer addr = " << static_cast(buffer.data()) << ", size = " << buffer.size() << std::endl; + + if (buffer.size() == 0) { + os << "Buffer is empty" << std::endl; + return; + } + + size_t range_start = 0; + uint8_t current_value = buffer[0]; + for (size_t i = 1; i < buffer.size(); ++i) { + if (buffer[i] != current_value) { + print_range(range_start, i, current_value, os); + current_value = buffer[i]; + range_start = i; + } + } + + // Print the last range + print_range(range_start, buffer.size(), current_value, os); + } + +private: + static void print_range(size_t range_start, size_t range_end_exclusive, uint8_t value, std::ostream& os) + { + const auto message = fmt::format("[0x{:08X}:0x{:08X}] - 0x{:02X} ({} bytes)", + range_start, range_end_exclusive - 1, static_cast(value), range_end_exclusive - range_start); + os << message << std::endl; + } +}; + } /* namespace hailort */ #endif /* HAILO_UTILS_H_ */ \ No newline at end of file diff --git a/hailort/drivers/common/hailo_ioctl_common.h b/hailort/drivers/common/hailo_ioctl_common.h index 0911f42..5f6cddf 100644 --- a/hailort/drivers/common/hailo_ioctl_common.h +++ b/hailort/drivers/common/hailo_ioctl_common.h @@ -6,6 +6,14 @@ #ifndef _HAILO_IOCTL_COMMON_H_ #define _HAILO_IOCTL_COMMON_H_ +#define HAILO_DRV_VER_MAJOR 4 +#define HAILO_DRV_VER_MINOR 18 +#define HAILO_DRV_VER_REVISION 0 + +#define _STRINGIFY_EXPANDED( x ) #x +#define _STRINGIFY_NUMBER( x ) _STRINGIFY_EXPANDED(x) +#define HAILO_DRV_VER _STRINGIFY_NUMBER(HAILO_DRV_VER_MAJOR) "." _STRINGIFY_NUMBER(HAILO_DRV_VER_MINOR) "." _STRINGIFY_NUMBER(HAILO_DRV_VER_REVISION) + // This value is not easily changeable. // For example: the channel interrupts ioctls assume we have up to 32 channels @@ -23,14 +31,17 @@ #define INVALID_DRIVER_HANDLE_VALUE ((uintptr_t)-1) // Used by windows and unix driver to raise the right CPU control handle to the FW. The same as in pcie_service FW -#define FW_ACCESS_CORE_CPU_CONTROL_SHIFT (1) -#define FW_ACCESS_CORE_CPU_CONTROL_MASK (1 << FW_ACCESS_CORE_CPU_CONTROL_SHIFT) -#define FW_ACCESS_CONTROL_INTERRUPT_SHIFT (0) -#define FW_ACCESS_APP_CPU_CONTROL_MASK (1 << FW_ACCESS_CONTROL_INTERRUPT_SHIFT) -#define FW_ACCESS_DRIVER_SHUTDOWN_SHIFT (2) -#define FW_ACCESS_DRIVER_SHUTDOWN_MASK (1 << FW_ACCESS_DRIVER_SHUTDOWN_SHIFT) +#define FW_ACCESS_CORE_CPU_CONTROL_SHIFT (1) +#define FW_ACCESS_CORE_CPU_CONTROL_MASK (1 << FW_ACCESS_CORE_CPU_CONTROL_SHIFT) +#define FW_ACCESS_CONTROL_INTERRUPT_SHIFT (0) +#define FW_ACCESS_APP_CPU_CONTROL_MASK (1 << FW_ACCESS_CONTROL_INTERRUPT_SHIFT) +#define FW_ACCESS_DRIVER_SHUTDOWN_SHIFT (2) +#define FW_ACCESS_DRIVER_SHUTDOWN_MASK (1 << FW_ACCESS_DRIVER_SHUTDOWN_SHIFT) +#define FW_ACCESS_SOC_CONNECT_SHIFT (3) +#define FW_ACCESS_SOC_CONNECT_MASK (1 << FW_ACCESS_SOC_CONNECT_SHIFT) + +#define INVALID_VDMA_CHANNEL (0xff) -#define INVALID_VDMA_CHANNEL (0xff) #if !defined(__cplusplus) && defined(NTDDI_VERSION) #include @@ -53,14 +64,23 @@ typedef uint8_t bool; #define INT_MAX 0x7FFFFFFF #endif // !defined(INT_MAX) +#if !defined(ECONNRESET) +#define ECONNRESET 104 /* Connection reset by peer */ +#endif // !defined(ECONNRESET) // {d88d31f1-fede-4e71-ac2a-6ce0018c1501} -DEFINE_GUID (GUID_DEVINTERFACE_HailoKM, +DEFINE_GUID (GUID_DEVINTERFACE_HailoKM_NNC, 0xd88d31f1,0xfede,0x4e71,0xac,0x2a,0x6c,0xe0,0x01,0x8c,0x15,0x01); -#define HAILO_GENERAL_IOCTL_MAGIC 0 -#define HAILO_VDMA_IOCTL_MAGIC 1 -#define HAILO_NON_LINUX_IOCTL_MAGIC 2 +// {7f16047d-64b8-207a-0092-e970893970a2} +DEFINE_GUID (GUID_DEVINTERFACE_HailoKM_SOC, + 0x7f16047d,0x64b8,0x207a,0x00,0x92,0xe9,0x70,0x89,0x39,0x70,0xa2); + +#define HAILO_GENERAL_IOCTL_MAGIC 0 +#define HAILO_VDMA_IOCTL_MAGIC 1 +#define HAILO_SOC_IOCTL_MAGIC 2 +#define HAILO_PCI_EP_IOCTL_MAGIC 3 +#define HAILO_NNC_IOCTL_MAGIC 4 #define HAILO_IOCTL_COMPATIBLE CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_ANY_ACCESS) @@ -114,9 +134,11 @@ static ULONG FORCEINLINE _IOC_(ULONG nr, ULONG type, ULONG size, bool read, bool #define _IOWR_ _IOWR #define _IO_ _IO -#define HAILO_GENERAL_IOCTL_MAGIC 'g' -#define HAILO_VDMA_IOCTL_MAGIC 'v' -#define HAILO_NON_LINUX_IOCTL_MAGIC 'w' +#define HAILO_GENERAL_IOCTL_MAGIC 'g' +#define HAILO_VDMA_IOCTL_MAGIC 'v' +#define HAILO_SOC_IOCTL_MAGIC 's' +#define HAILO_NNC_IOCTL_MAGIC 'n' +#define HAILO_PCI_EP_IOCTL_MAGIC 'p' #elif defined(__QNX__) // #ifdef _MSC_VER #include @@ -132,7 +154,6 @@ static ULONG FORCEINLINE _IOC_(ULONG nr, ULONG type, ULONG size, bool read, bool #define _IO_ __DION #define HAILO_GENERAL_IOCTL_MAGIC _DCMD_ALL #define HAILO_VDMA_IOCTL_MAGIC _DCMD_MISC -#define HAILO_NON_LINUX_IOCTL_MAGIC _DCMD_PROC #else // #ifdef _MSC_VER #error "unsupported platform!" @@ -161,6 +182,16 @@ enum hailo_dma_data_direction { HAILO_DMA_MAX_ENUM = INT_MAX, }; +// Enum that states what type of buffer we are working with in the driver +// TODO: HRT-13580 - Add specific type for user allocated and for driver allocated +enum hailo_dma_buffer_type { + HAILO_DMA_USER_PTR_BUFFER = 0, + HAILO_DMA_DMABUF_BUFFER = 1, + + /** Max enum value to maintain ABI Integrity */ + HAILO_DMA_BUFFER_MAX_ENUM = INT_MAX, +}; + // Enum that determines if buffer should be allocated from user space or from driver enum hailo_allocation_mode { HAILO_ALLOCATION_MODE_USERSPACE = 0, @@ -170,10 +201,19 @@ enum hailo_allocation_mode { HAILO_ALLOCATION_MODE_MAX_ENUM = INT_MAX, }; +enum hailo_vdma_interrupts_domain { + HAILO_VDMA_INTERRUPTS_DOMAIN_NONE = 0, + HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE = (1 << 0), + HAILO_VDMA_INTERRUPTS_DOMAIN_HOST = (1 << 1), + + /** Max enum value to maintain ABI Integrity */ + HAILO_VDMA_INTERRUPTS_DOMAIN_MAX_ENUM = INT_MAX, +}; + /* structure used in ioctl HAILO_VDMA_BUFFER_MAP */ struct hailo_vdma_buffer_map_params { #if defined(__linux__) || defined(_MSC_VER) - void* user_address; // in + uintptr_t user_address; // in #elif defined(__QNX__) shm_handle_t shared_memory_handle; // in #else @@ -181,6 +221,7 @@ struct hailo_vdma_buffer_map_params { #endif // __linux__ size_t size; // in enum hailo_dma_data_direction data_direction; // in + enum hailo_dma_buffer_type buffer_type; // in uintptr_t allocated_buffer_handle; // in size_t mapped_handle; // out }; @@ -204,31 +245,27 @@ struct hailo_desc_list_release_params { uintptr_t desc_handle; // in }; -/* structure used in ioctl HAILO_NON_LINUX_DESC_LIST_MMAP */ -struct hailo_non_linux_desc_list_mmap_params { - uintptr_t desc_handle; // in - size_t size; // in - void* user_address; // out -}; - /* structure used in ioctl HAILO_DESC_LIST_BIND_VDMA_BUFFER */ -struct hailo_desc_list_bind_vdma_buffer_params { +struct hailo_desc_list_program_params { size_t buffer_handle; // in size_t buffer_size; // in size_t buffer_offset; // in uintptr_t desc_handle; // in uint8_t channel_index; // in uint32_t starting_desc; // in + bool should_bind; // in + enum hailo_vdma_interrupts_domain last_interrupts_domain; // in + bool is_debug; // in }; -/* structure used in ioctl HAILO_VDMA_INTERRUPTS_ENABLE */ -struct hailo_vdma_interrupts_enable_params { +/* structure used in ioctl HAILO_VDMA_ENABLE_CHANNELS */ +struct hailo_vdma_enable_channels_params { uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in bool enable_timestamps_measure; // in }; -/* structure used in ioctl HAILO_VDMA_INTERRUPTS_DISABLE */ -struct hailo_vdma_interrupts_disable_params { +/* structure used in ioctl HAILO_VDMA_DISABLE_CHANNELS */ +struct hailo_vdma_disable_channels_params { uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in }; @@ -237,7 +274,7 @@ struct hailo_vdma_interrupts_channel_data { uint8_t engine_index; uint8_t channel_index; bool is_active; // If not activate, num_processed is ignored. - uint16_t host_num_processed; + uint8_t transfers_completed; // Number of transfers completed. uint8_t host_error; // Channel errors bits on source side uint8_t device_error; // Channel errors bits on dest side bool validation_success; // If the validation of the channel was successful @@ -312,6 +349,10 @@ enum hailo_transfer_memory_type { HAILO_TRANSFER_MEMORY_DMA_ENGINE1, HAILO_TRANSFER_MEMORY_DMA_ENGINE2, + // PCIe EP driver memories + HAILO_TRANSFER_MEMORY_PCIE_EP_CONFIG = 0x400, + HAILO_TRANSFER_MEMORY_PCIE_EP_BRIDGE, + /** Max enum value to maintain ABI Integrity */ HAILO_TRANSFER_MEMORY_MAX_ENUM = INT_MAX, }; @@ -352,15 +393,26 @@ enum hailo_board_type { HAILO_BOARD_TYPE_HAILO8 = 0, HAILO_BOARD_TYPE_HAILO15, HAILO_BOARD_TYPE_PLUTO, + HAILO_BOARD_TYPE_HAILO10H, + HAILO_BOARD_TYPE_HAILO10H_LEGACY, HAILO_BOARD_TYPE_COUNT, /** Max enum value to maintain ABI Integrity */ HAILO_BOARD_TYPE_MAX_ENUM = INT_MAX }; +enum hailo_accelerator_type { + HAILO_ACCELERATOR_TYPE_NNC, + HAILO_ACCELERATOR_TYPE_SOC, + + /** Max enum value to maintain ABI Integrity */ + HAILO_ACCELERATOR_TYPE_MAX_ENUM = INT_MAX +}; + enum hailo_dma_type { HAILO_DMA_TYPE_PCIE, HAILO_DMA_TYPE_DRAM, + HAILO_DMA_TYPE_PCI_EP, /** Max enum value to maintain ABI Integrity */ HAILO_DMA_TYPE_MAX_ENUM = INT_MAX, @@ -428,15 +480,6 @@ struct hailo_vdma_transfer_buffer { uint32_t size; // in }; -enum hailo_vdma_interrupts_domain { - HAILO_VDMA_INTERRUPTS_DOMAIN_NONE = 0, - HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE = (1 << 0), - HAILO_VDMA_INTERRUPTS_DOMAIN_HOST = (1 << 1), - - /** Max enum value to maintain ABI Integrity */ - HAILO_VDMA_INTERRUPTS_DOMAIN_MAX_ENUM = INT_MAX, -}; - // We allow maximum 2 buffers per transfer since we may have an extra buffer // to make sure each buffer is aligned to page size. #define HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER (2) @@ -460,6 +503,35 @@ struct hailo_vdma_launch_transfer_params { // more info (e.g desc complete status) uint32_t descs_programed; // out, amount of descriptors programed. + int launch_transfer_status; // out, status of the launch transfer call. (only used in case of error) +}; + +/* structure used in ioctl HAILO_SOC_CONNECT */ +struct hailo_soc_connect_params { + uint8_t input_channel_index; // out + uint8_t output_channel_index; // out + uintptr_t input_desc_handle; // in + uintptr_t output_desc_handle; // in +}; + +/* structure used in ioctl HAILO_SOC_CLOSE */ +struct hailo_soc_close_params { + uint8_t input_channel_index; // in + uint8_t output_channel_index; // in +}; + +/* structure used in ioctl HAILO_PCI_EP_ACCEPT */ +struct hailo_pci_ep_accept_params { + uint8_t input_channel_index; // out + uint8_t output_channel_index; // out + uintptr_t input_desc_handle; // in + uintptr_t output_desc_handle; // in +}; + +/* structure used in ioctl HAILO_PCI_EP_CLOSE */ +struct hailo_pci_ep_close_params { + uint8_t input_channel_index; // in + uint8_t output_channel_index; // in }; #ifdef _MSC_VER @@ -469,8 +541,8 @@ struct tCompatibleHailoIoctlData ULONG_PTR Value; union { struct hailo_memory_transfer_params MemoryTransfer; - struct hailo_vdma_interrupts_enable_params VdmaInterruptsEnable; - struct hailo_vdma_interrupts_disable_params VdmaInterruptsDisable; + struct hailo_vdma_enable_channels_params VdmaEnableChannels; + struct hailo_vdma_disable_channels_params VdmaDisableChannels; struct hailo_vdma_interrupts_read_timestamp_params VdmaInterruptsReadTimestamps; struct hailo_vdma_interrupts_wait_params VdmaInterruptsWait; struct hailo_vdma_buffer_sync_params VdmaBufferSync; @@ -479,14 +551,17 @@ struct tCompatibleHailoIoctlData struct hailo_vdma_buffer_unmap_params VdmaBufferUnmap; struct hailo_desc_list_create_params DescListCreate; struct hailo_desc_list_release_params DescListReleaseParam; - struct hailo_desc_list_bind_vdma_buffer_params DescListBind; + struct hailo_desc_list_program_params DescListProgram; struct hailo_d2h_notification D2HNotification; struct hailo_device_properties DeviceProperties; struct hailo_driver_info DriverInfo; - struct hailo_non_linux_desc_list_mmap_params DescListMmap; struct hailo_read_log_params ReadLog; struct hailo_mark_as_in_use_params MarkAsInUse; struct hailo_vdma_launch_transfer_params LaunchTransfer; + struct hailo_soc_connect_params ConnectParams; + struct hailo_soc_close_params SocCloseParams; + struct hailo_pci_ep_accept_params AcceptParams; + struct hailo_pci_ep_close_params PciEpCloseParams; } Buffer; }; #endif // _MSC_VER @@ -495,30 +570,20 @@ struct tCompatibleHailoIoctlData enum hailo_general_ioctl_code { HAILO_MEMORY_TRANSFER_CODE, - HAILO_FW_CONTROL_CODE, - HAILO_READ_NOTIFICATION_CODE, - HAILO_DISABLE_NOTIFICATION_CODE, HAILO_QUERY_DEVICE_PROPERTIES_CODE, HAILO_QUERY_DRIVER_INFO_CODE, - HAILO_READ_LOG_CODE, - HAILO_RESET_NN_CORE_CODE, // Must be last HAILO_GENERAL_IOCTL_MAX_NR, }; #define HAILO_MEMORY_TRANSFER _IOWR_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_MEMORY_TRANSFER_CODE, struct hailo_memory_transfer_params) -#define HAILO_FW_CONTROL _IOWR_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_FW_CONTROL_CODE, struct hailo_fw_control) -#define HAILO_READ_NOTIFICATION _IOW_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_READ_NOTIFICATION_CODE, struct hailo_d2h_notification) -#define HAILO_DISABLE_NOTIFICATION _IO_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_DISABLE_NOTIFICATION_CODE) #define HAILO_QUERY_DEVICE_PROPERTIES _IOW_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_QUERY_DEVICE_PROPERTIES_CODE, struct hailo_device_properties) #define HAILO_QUERY_DRIVER_INFO _IOW_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_QUERY_DRIVER_INFO_CODE, struct hailo_driver_info) -#define HAILO_READ_LOG _IOWR_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_READ_LOG_CODE, struct hailo_read_log_params) -#define HAILO_RESET_NN_CORE _IO_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_RESET_NN_CORE_CODE) enum hailo_vdma_ioctl_code { - HAILO_VDMA_INTERRUPTS_ENABLE_CODE, - HAILO_VDMA_INTERRUPTS_DISABLE_CODE, + HAILO_VDMA_ENABLE_CHANNELS_CODE, + HAILO_VDMA_DISABLE_CHANNELS_CODE, HAILO_VDMA_INTERRUPTS_WAIT_CODE, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE, HAILO_VDMA_BUFFER_MAP_CODE, @@ -526,7 +591,7 @@ enum hailo_vdma_ioctl_code { HAILO_VDMA_BUFFER_SYNC_CODE, HAILO_DESC_LIST_CREATE_CODE, HAILO_DESC_LIST_RELEASE_CODE, - HAILO_DESC_LIST_BIND_VDMA_BUFFER_CODE, + HAILO_DESC_LIST_PROGRAM_CODE, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE, HAILO_MARK_AS_IN_USE_CODE, @@ -538,38 +603,67 @@ enum hailo_vdma_ioctl_code { HAILO_VDMA_IOCTL_MAX_NR, }; -#define HAILO_VDMA_INTERRUPTS_ENABLE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_ENABLE_CODE, struct hailo_vdma_interrupts_enable_params) -#define HAILO_VDMA_INTERRUPTS_DISABLE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_DISABLE_CODE, struct hailo_vdma_interrupts_disable_params) +#define HAILO_VDMA_ENABLE_CHANNELS _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_ENABLE_CHANNELS_CODE, struct hailo_vdma_enable_channels_params) +#define HAILO_VDMA_DISABLE_CHANNELS _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_DISABLE_CHANNELS_CODE, struct hailo_vdma_disable_channels_params) #define HAILO_VDMA_INTERRUPTS_WAIT _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_WAIT_CODE, struct hailo_vdma_interrupts_wait_params) #define HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE, struct hailo_vdma_interrupts_read_timestamp_params) -#define HAILO_VDMA_BUFFER_MAP _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_MAP_CODE, struct hailo_vdma_buffer_map_params) -#define HAILO_VDMA_BUFFER_UNMAP _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_UNMAP_CODE, struct hailo_vdma_buffer_unmap_params) -#define HAILO_VDMA_BUFFER_SYNC _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_SYNC_CODE, struct hailo_vdma_buffer_sync_params) +#define HAILO_VDMA_BUFFER_MAP _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_MAP_CODE, struct hailo_vdma_buffer_map_params) +#define HAILO_VDMA_BUFFER_UNMAP _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_UNMAP_CODE, struct hailo_vdma_buffer_unmap_params) +#define HAILO_VDMA_BUFFER_SYNC _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_SYNC_CODE, struct hailo_vdma_buffer_sync_params) -#define HAILO_DESC_LIST_CREATE _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_CREATE_CODE, struct hailo_desc_list_create_params) -#define HAILO_DESC_LIST_RELEASE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_RELEASE_CODE, struct hailo_desc_list_release_params) -#define HAILO_DESC_LIST_BIND_VDMA_BUFFER _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_BIND_VDMA_BUFFER_CODE, struct hailo_desc_list_bind_vdma_buffer_params) +#define HAILO_DESC_LIST_CREATE _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_CREATE_CODE, struct hailo_desc_list_create_params) +#define HAILO_DESC_LIST_RELEASE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_RELEASE_CODE, struct hailo_desc_list_release_params) +#define HAILO_DESC_LIST_PROGRAM _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_PROGRAM_CODE, struct hailo_desc_list_program_params) -#define HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE, struct hailo_allocate_low_memory_buffer_params) -#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE, struct hailo_free_low_memory_buffer_params) +#define HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE, struct hailo_allocate_low_memory_buffer_params) +#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE, struct hailo_free_low_memory_buffer_params) -#define HAILO_MARK_AS_IN_USE _IOW_(HAILO_VDMA_IOCTL_MAGIC, HAILO_MARK_AS_IN_USE_CODE, struct hailo_mark_as_in_use_params) +#define HAILO_MARK_AS_IN_USE _IOW_(HAILO_VDMA_IOCTL_MAGIC, HAILO_MARK_AS_IN_USE_CODE, struct hailo_mark_as_in_use_params) -#define HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE, struct hailo_allocate_continuous_buffer_params) -#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE, struct hailo_free_continuous_buffer_params) +#define HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE, struct hailo_allocate_continuous_buffer_params) +#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE, struct hailo_free_continuous_buffer_params) -#define HAILO_VDMA_LAUNCH_TRANSFER _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LAUNCH_TRANSFER_CODE, struct hailo_vdma_launch_transfer_params) +#define HAILO_VDMA_LAUNCH_TRANSFER _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LAUNCH_TRANSFER_CODE, struct hailo_vdma_launch_transfer_params) +enum hailo_nnc_ioctl_code { + HAILO_FW_CONTROL_CODE, + HAILO_READ_NOTIFICATION_CODE, + HAILO_DISABLE_NOTIFICATION_CODE, + HAILO_READ_LOG_CODE, + HAILO_RESET_NN_CORE_CODE, -enum hailo_non_linux_ioctl_code { - HAILO_NON_LINUX_DESC_LIST_MMAP_CODE, + // Must be last + HAILO_NNC_IOCTL_MAX_NR +}; + +#define HAILO_FW_CONTROL _IOWR_(HAILO_NNC_IOCTL_MAGIC, HAILO_FW_CONTROL_CODE, struct hailo_fw_control) +#define HAILO_READ_NOTIFICATION _IOW_(HAILO_NNC_IOCTL_MAGIC, HAILO_READ_NOTIFICATION_CODE, struct hailo_d2h_notification) +#define HAILO_DISABLE_NOTIFICATION _IO_(HAILO_NNC_IOCTL_MAGIC, HAILO_DISABLE_NOTIFICATION_CODE) +#define HAILO_READ_LOG _IOWR_(HAILO_NNC_IOCTL_MAGIC, HAILO_READ_LOG_CODE, struct hailo_read_log_params) +#define HAILO_RESET_NN_CORE _IO_(HAILO_NNC_IOCTL_MAGIC, HAILO_RESET_NN_CORE_CODE) + +enum hailo_soc_ioctl_code { + HAILO_SOC_IOCTL_CONNECT_CODE, + HAILO_SOC_IOCTL_CLOSE_CODE, // Must be last - HAILO_NON_LINUX_IOCTL_MAX_NR, + HAILO_SOC_IOCTL_MAX_NR, }; -#define HAILO_NON_LINUX_DESC_LIST_MMAP _IOWR_(HAILO_NON_LINUX_IOCTL_MAGIC, HAILO_NON_LINUX_DESC_LIST_MMAP_CODE, struct hailo_non_linux_desc_list_mmap_params) +#define HAILO_SOC_CONNECT _IOWR_(HAILO_SOC_IOCTL_MAGIC, HAILO_SOC_IOCTL_CONNECT_CODE, struct hailo_soc_connect_params) +#define HAILO_SOC_CLOSE _IOR_(HAILO_SOC_IOCTL_MAGIC, HAILO_SOC_IOCTL_CLOSE_CODE, struct hailo_soc_close_params) + + +enum hailo_pci_ep_ioctl_code { + HAILO_PCI_EP_ACCEPT_CODE, + HAILO_PCI_EP_CLOSE_CODE, + + // Must be last + HAILO_PCI_EP_IOCTL_MAX_NR, +}; +#define HAILO_PCI_EP_ACCEPT _IOWR_(HAILO_PCI_EP_IOCTL_MAGIC, HAILO_PCI_EP_ACCEPT_CODE, struct hailo_pci_ep_accept_params) +#define HAILO_PCI_EP_CLOSE _IOR_(HAILO_PCI_EP_IOCTL_MAGIC, HAILO_PCI_EP_CLOSE_CODE, struct hailo_pci_ep_close_params) #endif /* _HAILO_IOCTL_COMMON_H_ */ diff --git a/hailort/drivers/win/include/hailo_pcie_version.h b/hailort/drivers/win/include/hailo_pcie_version.h deleted file mode 100644 index 2dfaa33..0000000 --- a/hailort/drivers/win/include/hailo_pcie_version.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef _HAILO_PCIE_VERSION_H_ -#define _HAILO_PCIE_VERSION_H_ - -#include "..\..\common\hailo_pcie_version.h" - -#define STRINGIFY_EXPANDED( x ) #x -#define STRINGIFY_NUMBER( x ) STRINGIFY_EXPANDED(x) -#define HAILO_DRV_VER STRINGIFY_NUMBER(HAILO_DRV_VER_MAJOR) "." STRINGIFY_NUMBER(HAILO_DRV_VER_MINOR) "." STRINGIFY_NUMBER(HAILO_DRV_VER_REVISION) - -#endif /* _HAILO_PCIE_VERSION_H_ */ diff --git a/hailort/hailo_firmware_eula b/hailort/hailo_firmware_eula new file mode 100644 index 0000000..c59b7aa --- /dev/null +++ b/hailort/hailo_firmware_eula @@ -0,0 +1,12 @@ +Copyright 2024 (C) Hailo Technologies Ltd. ("Hailo") +All rights reserved. + +By downloading, installing, copying, or otherwise using the software, +you agree to be bound by the terms of this license. + +Hailo hereby grants You a limited, non-exclusive, non-assignable, non-transferable, revocable, non-sublicensable, license to (a) use the Licensed Software, (b) Redistribution of this software in its binary form only, both (a) and (b) solely: (i) according to the Documentation and this license ; (ii) for the purpose of developing applications for use in Your system(s) ("End-User Product(s)), and in conjunction with Hailo's proprietary products previously purchased by You from Hailo ("Products") which are utilizing the Licensed Software; + +Restrictions. You agree that except as expressly permitted by Hailo under this license, you will not, nor allow any third party on Your behalf to (a) license, rent, lease, sublicense, loan, sell or otherwise allow any access the Licensed Software; (b) modify, alter, copy, transfer, emulate or create any derivative works of the Licensed Software or of any part thereof; (c) reverse engineer, decompile, decode, decrypt, disassemble, or in any way attempt to derive source code, designs, or otherwise discover the underlying Intellectual Property or technology of the Licensed Software or any part thereof; (d) remove, alter or obscure any copyright, trademark or other proprietary rights notice, on or in, the Licensed Software and/or the Documentation; (e) use the Licensed Software for any benchmarking to be publicly published or for competing development activities, (f) distribute without this license and/or not in the Licensed software binary form (g) publish or disclose to any third party any technical features, quality, performance or benchmark test, or comparative analyses relating to the Licensed Software ("Technical Data"), or (h) utilize the Licensed Software with any product(s) other than End User Products in conjunction with the Product(s) as described hereunder and under the Documentation, or (i) use the Licensed Software in any manner that would cause the Licensed Software and/or the Product to become subject to an Open Source License. "Open Source License" includes, without limitation, a software license that requires as a condition of use, modification, and/or distribution, of such software that the Licensed Software be (1) disclosed or distributed in source code form; (2) be licensed under other and/or terms other than this license; + +Hailo Technologies Ltd. disclaims any warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose. +This software is provided on an "AS IS" basis, and Hailo has no obligation to provide maintenance, support, updates, enhancements, or modifications diff --git a/hailort/hailort_server/CMakeLists.txt b/hailort/hailort_server/CMakeLists.txt new file mode 100644 index 0000000..d288066 --- /dev/null +++ b/hailort/hailort_server/CMakeLists.txt @@ -0,0 +1,55 @@ +cmake_minimum_required(VERSION 3.0.0) + +include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/spdlog.cmake) +include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/readerwriterqueue.cmake) + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +set(HAILORT_SERVER_SOURCES + hailort_server.cpp + ${HRPC_CPP_SOURCES} + ${HRPC_PROTOCOL_CPP_SOURCES} + ${HAILORT_COMMON_OS_DIR}/os_utils.cpp + ${HAILORT_SERVICE_DIR}/cng_buffer_pool.cpp + ${HAILORT_COMMON_DIR}/common/event_internal.cpp + ${HAILO_FULL_OS_DIR}/event.cpp # TODO HRT-10681: move to common + ${DRIVER_OS_DIR}/driver_os_specific.cpp + ${HAILO_OS_DIR}/file_descriptor.cpp + ${HAILO_OS_DIR}/mmap_buffer.cpp + ${HAILORT_SRC_DIR}/vdma/pcie_session.cpp + ${HAILORT_SRC_DIR}/vdma/memory/descriptor_list.cpp + ${HAILORT_SRC_DIR}/vdma/memory/mapped_buffer.cpp + ${HAILORT_SRC_DIR}/vdma/memory/dma_able_buffer.cpp + ${HAILORT_SRC_DIR}/vdma/memory/vdma_edge_layer.cpp + ${HAILORT_SRC_DIR}/vdma/driver/hailort_driver.cpp + ${HAILORT_SRC_DIR}/vdma/channel/interrupts_dispatcher.cpp + ${HAILORT_SRC_DIR}/vdma/channel/transfer_launcher.cpp + ${HAILORT_SRC_DIR}/vdma/channel/boundary_channel.cpp + ${HAILORT_SRC_DIR}/vdma/channel/channels_group.cpp + ${HAILORT_SRC_DIR}/stream_common/transfer_common.cpp +) +if(WIN32) + # hailort_driver.cpp in windows depends on string_conversion + # dma_able_buffer.cpp in windows depends on virtual_alloc_guard + set(HAILORT_SERVER_SOURCES ${HAILORT_SERVER_SOURCES} + ${HAILORT_COMMON_OS_DIR}/string_conversion.cpp + ${HAILO_FULL_OS_DIR}/virtual_alloc_guard.cpp) +endif() + +add_executable(hailort_server ${HAILORT_SERVER_SOURCES}) +target_include_directories(hailort_server PRIVATE + ${HAILORT_SRC_DIR} + ${COMMON_INC_DIR} + ${DRIVER_INC_DIR} +) +target_compile_options(hailort_server PRIVATE ${HAILORT_COMPILE_OPTIONS}) +set_property(TARGET hailort_server PROPERTY CXX_STANDARD 14) +set_property(TARGET hailort_server PROPERTY INSTALL_RPATH "$ORIGIN" "../lib/") # Link with a relative libhailort +target_link_libraries(hailort_server PRIVATE + libhailort + Threads::Threads + rpc_proto + spdlog::spdlog + readerwriterqueue +) \ No newline at end of file diff --git a/hailort/hailort_server/hailort_server.cpp b/hailort/hailort_server/hailort_server.cpp new file mode 100644 index 0000000..d8cc83f --- /dev/null +++ b/hailort/hailort_server/hailort_server.cpp @@ -0,0 +1,539 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file hailo_server.cpp + * @brief Hailo Server + **/ + +#include "hailort_server.hpp" +#include "hrpc/server.hpp" +#include "hailo/vdevice.hpp" +#include "hailo/infer_model.hpp" +#include "hrpc_protocol/serializer.hpp" +#include "net_flow/ops/nms_post_process.hpp" +#include "hailort_service/service_resource_manager.hpp" + +#include +#include + +using namespace hailort; + +// TODO: These macros should be merged with the grpc macros, also change them to TRY +#define CHECK_EXPECTED_AS_HRPC_STATUS(_exepcted, T) \ + do { \ + if (!_exepcted) { \ + LOGGER__ERROR("CHECK_EXPECTED_AS_HRPC_STATUS failed, status: {}", _exepcted.status()); \ + auto reply = T::serialize_reply(_exepcted.status()); \ + if (reply) return reply; \ + LOGGER__CRITICAL("Failed to create reply with status: {}", reply.status()); \ + return make_unexpected(HAILO_INTERNAL_FAILURE); \ + } \ + } while (0) +#define CHECK_SUCCESS_AS_HRPC_STATUS(_status, T) \ + do { \ + if (_status != HAILO_SUCCESS) { \ + LOGGER__ERROR("CHECK_SUCCESS_AS_HRPC_STATUS failed, status: {}", _status); \ + auto reply = T::serialize_reply(_status); \ + if (reply) return reply; \ + LOGGER__CRITICAL("Failed to create reply with status: {}", reply.status()); \ + return make_unexpected(HAILO_INTERNAL_FAILURE); \ + } \ + } while (0) + +#define __HAILO_CONCAT(x, y) x ## y +#define _HAILO_CONCAT(x, y) __HAILO_CONCAT(x, y) + +#define _TRY_AS_HRPC_STATUS(expected_var_name, var_decl, expr, ...) \ + auto expected_var_name = (expr); \ + CHECK_EXPECTED_AS_HRPC_STATUS(expected_var_name, __VA_ARGS__); \ + var_decl = expected_var_name.release() + +#define TRY_AS_HRPC_STATUS(var_decl, expr, ...) _TRY_AS_HRPC_STATUS(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__) + +#ifdef NDEBUG +#define LOGGER_PATTERN ("[%n] [%^%l%$] %v") +#else +#define LOGGER_PATTERN ("[%Y-%m-%d %X.%e] [%P] [%t] [%n] [%^%l%$] [%s:%#] [%!] %v") +#endif +#define BUFFER_POOL_SIZE (10) // TODO: this may hurt performance, should be configurable + +struct InferModelInfo +{ + std::unordered_map input_streams_sizes; + std::unordered_map output_streams_sizes; + std::vector inputs_names; + std::vector outputs_names; +}; + +void init_logger(const std::string &name) +{ + auto console_sink = make_shared_nothrow(); + console_sink->set_level(spdlog::level::info); + console_sink->set_pattern(LOGGER_PATTERN); + spdlog::set_default_logger(make_shared_nothrow(name, console_sink)); +} + +void hrpc::HailoRTServer::cleanup_infer_model_hef_buffers(const std::vector &infer_model_handles) +{ + for (const auto &infer_model_handle : infer_model_handles) { + auto hef_buffers_iter = m_hef_buffers_per_infer_model.find(infer_model_handle); + if (m_hef_buffers_per_infer_model.end() != hef_buffers_iter) { + m_hef_buffers_per_infer_model.erase(infer_model_handle); + } + } +} + +void hrpc::HailoRTServer::cleanup_cim_buffer_pools(const std::vector &cim_handles) +{ + for (const auto &cim_handle : cim_handles) { + auto buffer_pool_iter = m_buffer_pool_per_cim.find(cim_handle); + if (m_buffer_pool_per_cim.end() != buffer_pool_iter) { + m_buffer_pool_per_cim.erase(cim_handle); + } + } +} + +hailo_status hrpc::HailoRTServer::cleanup_client_resources(RpcConnection client_connection) +{ + std::set pids = {SINGLE_CLIENT_PID}; + auto cim_handles = ServiceResourceManager::get_instance().resources_handles_by_pids(pids); + (void)ServiceResourceManager::get_instance().release_by_pid(SINGLE_CLIENT_PID); + cleanup_cim_buffer_pools(cim_handles); + + auto infer_model_handles = ServiceResourceManager::get_instance().resources_handles_by_pids(pids); + (void)ServiceResourceManager::get_instance().release_by_pid(SINGLE_CLIENT_PID); + (void)ServiceResourceManager::get_instance().release_by_pid(SINGLE_CLIENT_PID); + cleanup_infer_model_hef_buffers(infer_model_handles); + m_infer_model_to_info_id.clear(); + + (void)ServiceResourceManager::get_instance().release_by_pid(SINGLE_CLIENT_PID); + CHECK_SUCCESS(client_connection.close()); + return HAILO_SUCCESS; +} + +Expected> hrpc::HailoRTServer::create_unique() +{ + TRY(auto connection_context, ConnectionContext::create_shared(true)); + auto res = make_unique_nothrow(connection_context); + CHECK_NOT_NULL(res, HAILO_OUT_OF_HOST_MEMORY); + return res; +} + +int main() +{ + init_logger("HailoRT-Server"); + TRY(auto server, hrpc::HailoRTServer::create_unique()); + hrpc::Dispatcher dispatcher; + + // TODO: add a server implementation class, with resources heiracrhy and more + auto &infer_model_to_info_id = server->get_infer_model_to_info_id(); + auto &buffer_pool_per_cim = server->get_buffer_pool_per_cim(); + + // Because the infer model is created with a hef buffer, we need to keep the buffer until the configure stage. + // Here I keep it until the infer model is destroyed + auto &hef_buffers = server->get_hef_buffers(); + + dispatcher.register_action(HailoRpcActionID::VDEVICE__CREATE, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + TRY_AS_HRPC_STATUS(auto vdevice_params, CreateVDeviceSerializer::deserialize_request(request), CreateVDeviceSerializer); + TRY_AS_HRPC_STATUS(auto vdevice, VDevice::create(vdevice_params), CreateVDeviceSerializer); + + auto &manager = ServiceResourceManager::get_instance(); + auto id = manager.register_resource(SINGLE_CLIENT_PID, std::move(vdevice)); + auto reply = CreateVDeviceSerializer::serialize_reply(HAILO_SUCCESS, id); + return reply; + }); + dispatcher.register_action(HailoRpcActionID::VDEVICE__DESTROY, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &manager = ServiceResourceManager::get_instance(); + TRY_AS_HRPC_STATUS(auto vdevice_handle, DestroyVDeviceSerializer::deserialize_request(request), DestroyVDeviceSerializer); + (void)manager.release_resource(vdevice_handle, SINGLE_CLIENT_PID); + TRY_AS_HRPC_STATUS(auto reply, DestroyVDeviceSerializer::serialize_reply(HAILO_SUCCESS), DestroyVDeviceSerializer); + return reply; + }); + dispatcher.register_action(HailoRpcActionID::VDEVICE__CREATE_INFER_MODEL, + [&hef_buffers] (const MemoryView &request, hrpc::ServerContextPtr server_context) -> Expected { + TRY_AS_HRPC_STATUS(auto tuple, CreateInferModelSerializer::deserialize_request(request), CreateInferModelSerializer); + auto vdevice_handle = std::get<0>(tuple); + uint64_t hef_size = std::get<1>(tuple); + + assert(hef_size <= SIZE_MAX); + TRY_AS_HRPC_STATUS(auto hef_buffer, Buffer::create(static_cast(hef_size), BufferStorageParams::create_dma()), CreateInferModelSerializer); + + auto status = server_context->connection().read_buffer(MemoryView(hef_buffer)); + CHECK_SUCCESS_AS_HRPC_STATUS(status, CreateInferModelSerializer); + + auto &vdevice_manager = ServiceResourceManager::get_instance(); + auto lambda = [view = MemoryView(hef_buffer)] (std::shared_ptr vdevice) { + return vdevice->create_infer_model(view); + }; + auto infer_model = vdevice_manager.execute>>(vdevice_handle, lambda); + CHECK_EXPECTED_AS_HRPC_STATUS(infer_model, CreateInferModelSerializer); + + auto &infer_model_manager = ServiceResourceManager::get_instance(); + auto infer_model_id = infer_model_manager.register_resource(SINGLE_CLIENT_PID, std::move(infer_model.release())); + hef_buffers.emplace(infer_model_id, std::move(hef_buffer)); + + TRY_AS_HRPC_STATUS(auto reply, CreateInferModelSerializer::serialize_reply(HAILO_SUCCESS, infer_model_id), CreateInferModelSerializer); + return reply; + }); + dispatcher.register_action(HailoRpcActionID::INFER_MODEL__DESTROY, + [&hef_buffers] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &manager = ServiceResourceManager::get_instance(); + TRY_AS_HRPC_STATUS(auto infer_model_handle, DestroyInferModelSerializer::deserialize_request(request), DestroyInferModelSerializer); + hef_buffers.erase(infer_model_handle); + (void)manager.release_resource(infer_model_handle, SINGLE_CLIENT_PID); + TRY_AS_HRPC_STATUS(auto reply, DestroyInferModelSerializer::serialize_reply(HAILO_SUCCESS), DestroyInferModelSerializer); + return reply; + }); + dispatcher.register_action(HailoRpcActionID::INFER_MODEL__CREATE_CONFIGURED_INFER_MODEL, + [&buffer_pool_per_cim, &infer_model_to_info_id] + (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &infer_model_manager = ServiceResourceManager::get_instance(); + + TRY_AS_HRPC_STATUS(auto request_params, CreateConfiguredInferModelSerializer::deserialize_request(request), CreateConfiguredInferModelSerializer); + const auto &infer_model_handle = request_params.infer_model_handle; + const auto &vdevice_handle = request_params.vdevice_handle; + + auto lambda = [&request_params] (std::shared_ptr infer_model) -> Expected { + const auto &input_streams_formats = request_params.input_streams_params; + const auto &output_streams_formats = request_params.output_streams_params; + for (const auto &input_stream_format : input_streams_formats) { + TRY(auto input, infer_model->input(input_stream_format.first)); + + input.set_format_order(static_cast(input_stream_format.second.format_order)); + input.set_format_type(static_cast(input_stream_format.second.format_type)); + if (INVALID_NMS_CONFIG != input_stream_format.second.nms_score_threshold) { + input.set_nms_score_threshold(input_stream_format.second.nms_score_threshold); + } + if (INVALID_NMS_CONFIG != input_stream_format.second.nms_iou_threshold) { + input.set_nms_iou_threshold(input_stream_format.second.nms_iou_threshold); + } + if (static_cast(INVALID_NMS_CONFIG) != input_stream_format.second.nms_max_proposals_per_class) { + input.set_nms_max_proposals_per_class(input_stream_format.second.nms_max_proposals_per_class); + } + if (static_cast(INVALID_NMS_CONFIG) != input_stream_format.second.nms_max_accumulated_mask_size) { + input.set_nms_max_accumulated_mask_size(input_stream_format.second.nms_max_accumulated_mask_size); + } + } + + for (const auto &output_stream_format : output_streams_formats) { + TRY(auto output, infer_model->output(output_stream_format.first)); + output.set_format_order(static_cast(output_stream_format.second.format_order)); + output.set_format_type(static_cast(output_stream_format.second.format_type)); + if (INVALID_NMS_CONFIG != output_stream_format.second.nms_score_threshold) { + output.set_nms_score_threshold(output_stream_format.second.nms_score_threshold); + } + if (INVALID_NMS_CONFIG != output_stream_format.second.nms_iou_threshold) { + output.set_nms_iou_threshold(output_stream_format.second.nms_iou_threshold); + } + if (static_cast(INVALID_NMS_CONFIG) != output_stream_format.second.nms_max_proposals_per_class) { + output.set_nms_max_proposals_per_class(output_stream_format.second.nms_max_proposals_per_class); + } + if (static_cast(INVALID_NMS_CONFIG) != output_stream_format.second.nms_max_accumulated_mask_size) { + output.set_nms_max_accumulated_mask_size(output_stream_format.second.nms_max_accumulated_mask_size); + } + } + + infer_model->set_batch_size(request_params.batch_size); + infer_model->set_power_mode(request_params.power_mode); + infer_model->set_hw_latency_measurement_flags(request_params.latency_flag); + + return infer_model->configure(); + }; + + auto configured_infer_model = infer_model_manager.execute>(infer_model_handle, lambda); + CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model, CreateConfiguredInferModelSerializer); + + TRY_AS_HRPC_STATUS(auto async_queue_size, configured_infer_model->get_async_queue_size(), CreateConfiguredInferModelSerializer); + auto set_model_info_lambda = [] (std::shared_ptr infer_model) -> Expected> { + auto infer_model_info = make_shared_nothrow(); + CHECK_NOT_NULL_AS_EXPECTED(infer_model_info, HAILO_OUT_OF_HOST_MEMORY); + + for (const auto &input : infer_model->inputs()) { + infer_model_info->input_streams_sizes.emplace(input.name(), input.get_frame_size()); + infer_model_info->inputs_names.push_back(input.name()); + } + for (const auto &output : infer_model->outputs()) { + infer_model_info->output_streams_sizes.emplace(output.name(), output.get_frame_size()); + infer_model_info->outputs_names.push_back(output.name()); + } + return infer_model_info; + }; + auto model_info = infer_model_manager.execute>>(infer_model_handle, set_model_info_lambda); + CHECK_EXPECTED_AS_HRPC_STATUS(model_info, CreateConfiguredInferModelSerializer); + + auto &infer_model_infos_manager = ServiceResourceManager::get_instance(); + auto infer_model_info_id = infer_model_infos_manager.register_resource(SINGLE_CLIENT_PID, std::move(model_info.release())); + + auto &cim_manager = ServiceResourceManager::get_instance(); + auto cim_id = cim_manager.register_resource(SINGLE_CLIENT_PID, + std::move(make_shared_nothrow(configured_infer_model.release()))); + + auto buffer_pool = ServiceNetworkGroupBufferPool::create(vdevice_handle); + CHECK_EXPECTED_AS_HRPC_STATUS(buffer_pool, CreateConfiguredInferModelSerializer); + + auto buffer_pool_ptr = buffer_pool.release(); + auto get_infer_model_info_lambda = [] (std::shared_ptr infer_model_info) { + return *infer_model_info; + }; + auto infer_model_info = infer_model_infos_manager.execute>(infer_model_info_id, get_infer_model_info_lambda); + CHECK_EXPECTED_AS_HRPC_STATUS(infer_model_info, CreateConfiguredInferModelSerializer); + + for (const auto &input_name : infer_model_info->inputs_names) { + auto status = buffer_pool_ptr->allocate_pool(input_name, HAILO_DMA_BUFFER_DIRECTION_D2H, + infer_model_info->input_streams_sizes[input_name], BUFFER_POOL_SIZE); + CHECK_SUCCESS_AS_HRPC_STATUS(status, CreateConfiguredInferModelSerializer); + } + for (const auto &output_name : infer_model_info->outputs_names) { + auto status = buffer_pool_ptr->allocate_pool(output_name, HAILO_DMA_BUFFER_DIRECTION_H2D, + infer_model_info->output_streams_sizes[output_name], BUFFER_POOL_SIZE); + CHECK_SUCCESS_AS_HRPC_STATUS(status, CreateConfiguredInferModelSerializer); + } + buffer_pool_per_cim.emplace(cim_id, buffer_pool_ptr); + + infer_model_to_info_id[infer_model_handle] = infer_model_info_id; + TRY_AS_HRPC_STATUS(auto reply, + CreateConfiguredInferModelSerializer::serialize_reply(HAILO_SUCCESS, cim_id, static_cast(async_queue_size)), + CreateConfiguredInferModelSerializer); + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__DESTROY, + [&buffer_pool_per_cim] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &manager = ServiceResourceManager::get_instance(); + TRY_AS_HRPC_STATUS(auto configured_infer_model_handle, DestroyConfiguredInferModelSerializer::deserialize_request(request), DestroyInferModelSerializer); + + auto shutdown_lambda = [] (std::shared_ptr configured_infer_model) { + configured_infer_model->shutdown(); + return HAILO_SUCCESS; + }; + manager.execute(configured_infer_model_handle, shutdown_lambda); + buffer_pool_per_cim.erase(configured_infer_model_handle); + (void)manager.release_resource(configured_infer_model_handle, SINGLE_CLIENT_PID); + TRY_AS_HRPC_STATUS(auto reply, DestroyConfiguredInferModelSerializer::serialize_reply(HAILO_SUCCESS), DestroyInferModelSerializer); + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_TIMEOUT, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &cim_manager = ServiceResourceManager::get_instance(); + TRY_AS_HRPC_STATUS(auto tuple, SetSchedulerTimeoutSerializer::deserialize_request(request), SetSchedulerTimeoutSerializer); + const auto &configured_infer_model_handle = std::get<0>(tuple); + const auto &timeout = std::get<1>(tuple); + auto lambda = [timeout] (std::shared_ptr configured_infer_model) { + return configured_infer_model->set_scheduler_timeout(timeout); + }; + auto status = cim_manager.execute(configured_infer_model_handle, lambda); + TRY_AS_HRPC_STATUS(auto reply, SetSchedulerTimeoutSerializer::serialize_reply(status), SetSchedulerTimeoutSerializer); + + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_THRESHOLD, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &cim_manager = ServiceResourceManager::get_instance(); + TRY_AS_HRPC_STATUS(auto tuple, SetSchedulerThresholdSerializer::deserialize_request(request), SetSchedulerThresholdSerializer); + const auto &configured_infer_model_handle = std::get<0>(tuple); + const auto &threshold = std::get<1>(tuple); + auto lambda = [threshold] (std::shared_ptr configured_infer_model) { + return configured_infer_model->set_scheduler_threshold(threshold); + }; + auto status = cim_manager.execute(configured_infer_model_handle, lambda); + TRY_AS_HRPC_STATUS(auto reply, SetSchedulerThresholdSerializer::serialize_reply(status), SetSchedulerThresholdSerializer); + + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_PRIORITY, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &cim_manager = ServiceResourceManager::get_instance(); + TRY_AS_HRPC_STATUS(auto tuple, SetSchedulerPrioritySerializer::deserialize_request(request), SetSchedulerPrioritySerializer); + const auto &configured_infer_model_handle = std::get<0>(tuple); + const auto &priority = std::get<1>(tuple); + auto lambda = [priority] (std::shared_ptr configured_infer_model) { + return configured_infer_model->set_scheduler_priority(static_cast(priority)); + }; + auto status = cim_manager.execute(configured_infer_model_handle, lambda); + TRY_AS_HRPC_STATUS(auto reply, SetSchedulerPrioritySerializer::serialize_reply(status), SetSchedulerPrioritySerializer); + + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__GET_HW_LATENCY_MEASUREMENT, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &cim_manager = ServiceResourceManager::get_instance(); + + auto configured_infer_model_handle = GetHwLatencyMeasurementSerializer::deserialize_request(request); + CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model_handle, GetHwLatencyMeasurementSerializer); + + auto lambda = [] (std::shared_ptr configured_infer_model) { + return configured_infer_model->get_hw_latency_measurement(); + }; + + auto latency_measurement_result = cim_manager.execute>(configured_infer_model_handle.value(), lambda); + if (HAILO_NOT_AVAILABLE == latency_measurement_result.status()) { + return GetHwLatencyMeasurementSerializer::serialize_reply(HAILO_NOT_AVAILABLE); + } + CHECK_EXPECTED_AS_HRPC_STATUS(latency_measurement_result, GetHwLatencyMeasurementSerializer); + + uint32_t avg_hw_latency = static_cast(latency_measurement_result.value().avg_hw_latency.count()); + TRY_AS_HRPC_STATUS(auto reply, GetHwLatencyMeasurementSerializer::serialize_reply(latency_measurement_result.status(), avg_hw_latency), GetHwLatencyMeasurementSerializer); + + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__ACTIVATE, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &cim_manager = ServiceResourceManager::get_instance(); + + auto configured_infer_model_handle = ActivateSerializer::deserialize_request(request); + CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model_handle, ActivateSerializer); + + auto lambda = [] (std::shared_ptr configured_infer_model) { + return configured_infer_model->activate(); + }; + + auto status = cim_manager.execute(configured_infer_model_handle.value(), lambda); + TRY_AS_HRPC_STATUS(auto reply, ActivateSerializer::serialize_reply(status), ActivateSerializer); + + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__DEACTIVATE, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &cim_manager = ServiceResourceManager::get_instance(); + + auto configured_infer_model_handle = DeactivateSerializer::deserialize_request(request); + CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model_handle, DeactivateSerializer); + + auto lambda = [] (std::shared_ptr configured_infer_model) { + return configured_infer_model->deactivate(); + }; + + auto status = cim_manager.execute(configured_infer_model_handle.value(), lambda); + TRY_AS_HRPC_STATUS(auto reply, DeactivateSerializer::serialize_reply(status), DeactivateSerializer); + + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__SHUTDOWN, + [] (const MemoryView &request, hrpc::ServerContextPtr /*server_context*/) -> Expected { + auto &cim_manager = ServiceResourceManager::get_instance(); + + auto configured_infer_model_handle = ShutdownSerializer::deserialize_request(request); + CHECK_EXPECTED_AS_HRPC_STATUS(configured_infer_model_handle, ShutdownSerializer); + + auto lambda = [] (std::shared_ptr configured_infer_model) { + return configured_infer_model->shutdown(); + }; + + auto status = cim_manager.execute(configured_infer_model_handle.value(), lambda); + TRY_AS_HRPC_STATUS(auto reply, ShutdownSerializer::serialize_reply(status), ShutdownSerializer); + + return reply; + }); + dispatcher.register_action(HailoRpcActionID::CONFIGURED_INFER_MODEL__RUN_ASYNC, + [&infer_model_to_info_id, &buffer_pool_per_cim] + (const MemoryView &request, hrpc::ServerContextPtr server_context) -> Expected { + auto &cim_manager = ServiceResourceManager::get_instance(); + auto bindings_lambda = [] (std::shared_ptr configured_infer_model) { + return configured_infer_model->create_bindings(); + }; + TRY_AS_HRPC_STATUS(auto request_tuple, RunAsyncSerializer::deserialize_request(request), RunAsyncSerializer); + auto configured_infer_model_handle = std::get<0>(request_tuple); + auto infer_model_handle = std::get<1>(request_tuple); + auto callback_id = std::get<2>(request_tuple); + + auto bindings = cim_manager.execute>(configured_infer_model_handle, bindings_lambda); + CHECK_EXPECTED_AS_HRPC_STATUS(bindings, RunAsyncSerializer); + + auto infer_model_info_lambda = [] (std::shared_ptr infer_model_info) { + return *infer_model_info; + }; + auto &infer_model_infos_manager = ServiceResourceManager::get_instance(); + auto infer_model_info = infer_model_infos_manager.execute>(infer_model_to_info_id[infer_model_handle], + infer_model_info_lambda); + CHECK_EXPECTED_AS_HRPC_STATUS(infer_model_info, RunAsyncSerializer); + + std::vector inputs; // TODO: add infer vector pool + inputs.reserve(infer_model_info->inputs_names.size()); + for (const auto &input_name : infer_model_info->inputs_names) { + TRY_AS_HRPC_STATUS(auto input, bindings->input(input_name), RunAsyncSerializer); + + TRY_AS_HRPC_STATUS(auto buffer_ptr, buffer_pool_per_cim[configured_infer_model_handle]->acquire_buffer(input_name), + RunAsyncSerializer); + + auto status = server_context->connection().read_buffer(MemoryView(*buffer_ptr)); + CHECK_SUCCESS_AS_HRPC_STATUS(status, RunAsyncSerializer); + + inputs.emplace_back(buffer_ptr); + status = input.set_buffer(MemoryView(*buffer_ptr)); + CHECK_SUCCESS_AS_HRPC_STATUS(status, RunAsyncSerializer); + } + + std::vector outputs; // TODO: add infer vector pool + outputs.reserve(infer_model_info->outputs_names.size()); + for (const auto &output_name : infer_model_info->outputs_names) { + TRY_AS_HRPC_STATUS(auto buffer_ptr, buffer_pool_per_cim[configured_infer_model_handle]->acquire_buffer(output_name), + RunAsyncSerializer); + + auto output = bindings->output(output_name); + CHECK_EXPECTED_AS_HRPC_STATUS(output, RunAsyncSerializer); + + auto status = output->set_buffer(MemoryView(buffer_ptr->data(), buffer_ptr->size())); + CHECK_SUCCESS_AS_HRPC_STATUS(status, RunAsyncSerializer); + + outputs.emplace_back(buffer_ptr); + } + + auto infer_lambda = + [bindings = bindings.release(), callback_id, server_context, inputs, outputs, &buffer_pool_per_cim, configured_infer_model_handle, + infer_model_info] + (std::shared_ptr configured_infer_model) { + return configured_infer_model->run_async(bindings, + [callback_id, server_context, inputs, outputs, &buffer_pool_per_cim, configured_infer_model_handle, infer_model_info] + (const AsyncInferCompletionInfo &completion_info) { + auto status = server_context->trigger_callback(callback_id, completion_info.status, [outputs, completion_info] (hrpc::RpcConnection connection) -> hailo_status { + if (HAILO_SUCCESS == completion_info.status) { + for (auto output : outputs) { + auto status = connection.write_buffer(MemoryView(*output)); + CHECK_SUCCESS(status); + } + } + return HAILO_SUCCESS; + }); + + // HAILO_COMMUNICATION_CLOSED means the client disconnected. Server doesn't need to restart in this case. + if ((status != HAILO_SUCCESS) && (status != HAILO_COMMUNICATION_CLOSED)) { + LOGGER__CRITICAL("Error {} returned from connection.write(). Server Should restart!", status); + } + + for (uint32_t i = 0; i < inputs.size(); i++) { + status = buffer_pool_per_cim[configured_infer_model_handle]->return_to_pool(infer_model_info->inputs_names[i], inputs[i]); + if (status != HAILO_SUCCESS) { + LOGGER__CRITICAL("return_to_pool failed for input {}, status = {}. Server should restart!", infer_model_info->inputs_names[i], status); + return; + } + } + for (uint32_t i = 0; i < outputs.size(); i++) { + status = buffer_pool_per_cim[configured_infer_model_handle]->return_to_pool(infer_model_info->outputs_names[i], outputs[i]); + if (status != HAILO_SUCCESS) { + LOGGER__CRITICAL("return_to_pool failed for output {}, status = {}. Server should restart!", infer_model_info->outputs_names[i], status); + return; + } + } + }); + }; + auto job = cim_manager.execute>(configured_infer_model_handle, infer_lambda); + CHECK_EXPECTED_AS_HRPC_STATUS(job, RunAsyncSerializer); + + job->detach(); + + TRY_AS_HRPC_STATUS(auto reply, RunAsyncSerializer::serialize_reply(HAILO_SUCCESS), RunAsyncSerializer); + return reply; + }); + + server->set_dispatcher(dispatcher); + auto status = server->serve(); + if (status != HAILO_SUCCESS) { + LOGGER__ERROR("Error in serve, status = {}", status); + return status; + } + + return 0; +} diff --git a/hailort/hailort_server/hailort_server.hpp b/hailort/hailort_server/hailort_server.hpp new file mode 100644 index 0000000..f90f7d4 --- /dev/null +++ b/hailort/hailort_server/hailort_server.hpp @@ -0,0 +1,44 @@ +#ifndef HAILORT_SERVER_HPP_ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file hailort_server.hpp + * @brief RPC Hailort Server Header + **/ + +#define HAILORT_SERVER_HPP_ + +#include "hrpc/server.hpp" +#include "hailort_service/cng_buffer_pool.hpp" + +namespace hrpc +{ + +using infer_model_handle_t = uint32_t; + +class Server; +class HailoRTServer : public Server { +public: + static Expected> create_unique(); + explicit HailoRTServer(std::shared_ptr connection_context) : Server(connection_context) {}; + + std::unordered_map &get_infer_model_to_info_id() { return m_infer_model_to_info_id; }; + std::unordered_map> &get_buffer_pool_per_cim() { return m_buffer_pool_per_cim; }; + std::unordered_map &get_hef_buffers() { return m_hef_buffers_per_infer_model; }; + +private: + + std::unordered_map m_infer_model_to_info_id; + std::unordered_map> m_buffer_pool_per_cim; + std::unordered_map m_hef_buffers_per_infer_model; + virtual hailo_status cleanup_client_resources(RpcConnection client_connection) override; + void cleanup_cim_buffer_pools(const std::vector &cim_handles); + void cleanup_infer_model_hef_buffers(const std::vector &infer_model_handles); + +}; + +} // namespace hrpc + +#endif // HAILORT_SERVER_HPP_ \ No newline at end of file diff --git a/hailort/hailort_server/hailort_server.sh b/hailort/hailort_server/hailort_server.sh new file mode 100644 index 0000000..3b11dc0 --- /dev/null +++ b/hailort/hailort_server/hailort_server.sh @@ -0,0 +1,39 @@ +#! /bin/bash + +### BEGIN INIT INFO +# Provides: hailort_server +# Required-Start: $local_fs $network +# Required-Stop: $local_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: hailort_server service +# Description: Run hailort_server daemon +### END INIT INFO + +# TODO: Remove this file once the hailort_server will use systemd + +# Carry out specific functions when asked to by the system +case "$1" in + start) + echo "Starting hailort_server" + bash -c "cd /usr/bin && hailort_server &" + ;; + + stop) + echo "Stopping hailort_server..." + bash -c "killall hailort_server" + ;; + + restart) + $0 stop + sleep 1 + $0 start + ;; + + *) + echo "Usage: /etc/init.d/hailort_server {start|stop|restart}" + exit 1 + ;; +esac + +exit 0 \ No newline at end of file diff --git a/hailort/hailort_service/cng_buffer_pool.cpp b/hailort/hailort_service/cng_buffer_pool.cpp index b054164..f2a268c 100644 --- a/hailort/hailort_service/cng_buffer_pool.cpp +++ b/hailort/hailort_service/cng_buffer_pool.cpp @@ -23,26 +23,25 @@ Expected> ServiceStreamBufferPool::crea }; auto &vdevice_manager = ServiceResourceManager::get_instance(); - auto free_buffers_queue = SpscQueue::create(buffer_count, shutdown_event, DEFAULT_TRANSFER_TIMEOUT); - CHECK_EXPECTED(free_buffers_queue); + TRY(auto free_buffers_queue, + SpscQueue::create(buffer_count, shutdown_event, DEFAULT_TRANSFER_TIMEOUT)); std::vector buffers; buffers.reserve(buffer_count); for (size_t i = 0; i < buffer_count; i++) { - auto buffer = Buffer::create_shared(buffer_size, BufferStorageParams::create_dma()); - CHECK_EXPECTED(buffer); + TRY(auto buffer, Buffer::create_shared(buffer_size, BufferStorageParams::create_dma())); - auto mapped_buffer = vdevice_manager.execute>(vdevice_handle, map_buffer_lambda, buffer.value()); - CHECK_EXPECTED(mapped_buffer); + TRY(auto mapped_buffer, + vdevice_manager.execute>(vdevice_handle, map_buffer_lambda, buffer)); - auto status = free_buffers_queue->enqueue(buffer.value()); + auto status = free_buffers_queue.enqueue(buffer); CHECK_SUCCESS(status); - buffers.emplace_back(AllocatedMappedBuffer{ buffer.release(), mapped_buffer.release()}); + buffers.emplace_back(AllocatedMappedBuffer{ buffer, std::move(mapped_buffer)}); } auto buffer_pool_ptr = make_shared_nothrow(buffer_size, std::move(buffers), - free_buffers_queue.release(), buffer_count); + std::move(free_buffers_queue), buffer_count); CHECK_NOT_NULL_AS_EXPECTED(buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY); return buffer_pool_ptr; @@ -58,18 +57,9 @@ ServiceStreamBufferPool::ServiceStreamBufferPool(size_t buffer_size, std::vector Expected ServiceStreamBufferPool::acquire_buffer() { - auto buffer = m_free_buffers_queue.dequeue(DEFAULT_TRANSFER_TIMEOUT); - if (HAILO_SHUTDOWN_EVENT_SIGNALED == buffer.status()) { - return make_unexpected(buffer.status()); - } - else if (HAILO_TIMEOUT == buffer.status()) { - LOGGER__WARNING( - "Failed to acquire buffer because the buffer pool is empty. This could be caused by uneven reading and writing speeds"); - return make_unexpected(buffer.status()); - } - CHECK_EXPECTED(buffer); - - return buffer.release(); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto buffer, + m_free_buffers_queue.dequeue(DEFAULT_TRANSFER_TIMEOUT)); + return buffer; } hailo_status ServiceStreamBufferPool::return_to_pool(BufferPtr buffer) @@ -91,9 +81,7 @@ size_t ServiceStreamBufferPool::buffers_count() Expected> ServiceNetworkGroupBufferPool::create(uint32_t vdevice_handle) { - auto shutdown_event_exp = Event::create_shared(Event::State::not_signalled); - CHECK_EXPECTED(shutdown_event_exp); - auto shutdown_event = shutdown_event_exp.release(); + TRY(auto shutdown_event, Event::create_shared(Event::State::not_signalled)); auto cng_buffer_pool_ptr = make_shared_nothrow(shutdown_event, vdevice_handle); CHECK_NOT_NULL_AS_EXPECTED(cng_buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY); @@ -102,54 +90,53 @@ Expected> ServiceNetworkGroupBuff } ServiceNetworkGroupBufferPool::ServiceNetworkGroupBufferPool(EventPtr shutdown_event, uint32_t vdevice_handle) : - m_output_name_to_buffer_pool(), m_shutdown_event(shutdown_event), m_vdevice_handle(vdevice_handle) + m_stream_name_to_buffer_pool(), m_shutdown_event(shutdown_event), m_vdevice_handle(vdevice_handle) {} -hailo_status ServiceNetworkGroupBufferPool::allocate_pool(const std::string &name, size_t frame_size, size_t pool_size) +hailo_status ServiceNetworkGroupBufferPool::allocate_pool(const std::string &name, + hailo_dma_buffer_direction_t direction, size_t frame_size, size_t pool_size) { - auto buffer_pool = ServiceStreamBufferPool::create(m_vdevice_handle, frame_size, - pool_size, HAILO_DMA_BUFFER_DIRECTION_D2H, m_shutdown_event); - CHECK_EXPECTED(buffer_pool); + TRY(auto buffer_pool, ServiceStreamBufferPool::create(m_vdevice_handle, frame_size, + pool_size, direction, m_shutdown_event)); std::unique_lock lock(m_mutex); - m_output_name_to_buffer_pool[name] = buffer_pool.release(); + m_stream_name_to_buffer_pool[name] = buffer_pool; return HAILO_SUCCESS; } -hailo_status ServiceNetworkGroupBufferPool::reallocate_pool(const std::string &name, size_t frame_size) +hailo_status ServiceNetworkGroupBufferPool::reallocate_pool(const std::string &name, + hailo_dma_buffer_direction_t direction, size_t frame_size) { std::unique_lock lock(m_mutex); - auto pool_size = m_output_name_to_buffer_pool[name]->buffers_count(); - m_output_name_to_buffer_pool[name].reset(); + auto pool_size = m_stream_name_to_buffer_pool[name]->buffers_count(); + m_stream_name_to_buffer_pool[name].reset(); - auto buffer_pool = ServiceStreamBufferPool::create(m_vdevice_handle, frame_size, - pool_size, HAILO_DMA_BUFFER_DIRECTION_D2H, m_shutdown_event); - CHECK_EXPECTED(buffer_pool); - m_output_name_to_buffer_pool[name] = buffer_pool.release(); + TRY(auto buffer_pool, ServiceStreamBufferPool::create(m_vdevice_handle, frame_size, + pool_size, direction, m_shutdown_event)); + m_stream_name_to_buffer_pool[name] = buffer_pool; return HAILO_SUCCESS; } -Expected ServiceNetworkGroupBufferPool::acquire_buffer(const std::string &output_name) +Expected ServiceNetworkGroupBufferPool::acquire_buffer(const std::string &stream_name) { - CHECK_AS_EXPECTED(contains(m_output_name_to_buffer_pool, output_name), HAILO_INTERNAL_FAILURE, - "acquire_buffer() for output {} failed, output name does not exist in buffer pool", output_name); + CHECK_AS_EXPECTED(contains(m_stream_name_to_buffer_pool, stream_name), HAILO_INTERNAL_FAILURE, + "acquire_buffer() for stream {} failed, stream name does not exist in buffer pool", stream_name); std::unique_lock lock(m_mutex); - auto buffer = m_output_name_to_buffer_pool.at(output_name)->acquire_buffer(); - CHECK_EXPECTED(buffer); + TRY(auto buffer, m_stream_name_to_buffer_pool.at(stream_name)->acquire_buffer()); - return buffer.release(); + return buffer; } -hailo_status ServiceNetworkGroupBufferPool::return_to_pool(const std::string &output_name, BufferPtr buffer) +hailo_status ServiceNetworkGroupBufferPool::return_to_pool(const std::string &stream_name, BufferPtr buffer) { - CHECK(contains(m_output_name_to_buffer_pool, output_name), HAILO_INTERNAL_FAILURE, - "acquire_buffer() for output {} failed, output name does not exist in buffer pool", output_name); + CHECK(contains(m_stream_name_to_buffer_pool, stream_name), HAILO_INTERNAL_FAILURE, + "acquire_buffer() for stream {} failed, stream name does not exist in buffer pool", stream_name); std::unique_lock lock(m_mutex); - auto status = m_output_name_to_buffer_pool.at(output_name)->return_to_pool(buffer); + auto status = m_stream_name_to_buffer_pool.at(stream_name)->return_to_pool(buffer); CHECK_SUCCESS(status); return HAILO_SUCCESS; diff --git a/hailort/hailort_service/cng_buffer_pool.hpp b/hailort/hailort_service/cng_buffer_pool.hpp index 86172a3..399027a 100644 --- a/hailort/hailort_service/cng_buffer_pool.hpp +++ b/hailort/hailort_service/cng_buffer_pool.hpp @@ -4,7 +4,7 @@ **/ /** * @file cng_buffer_pool.hpp - * @brief This model represents the buffer pools for the output reads for each network group. Used in async API + * @brief This model represents the buffer pools for the streams of each network group. Used in async API **/ #ifndef _HAILO_CNG_BUFFER_POOL_HPP_ @@ -49,9 +49,9 @@ private: }; using BufferPoolPtr = std::shared_ptr; -using output_name_t = std::string; +using stream_name_t = std::string; -// This object holds a buffer pool for each output streams of the network group. +// This object holds a buffer pool for each stream of the network group. // It is used to pre-allocate all the buffers necessary for the reads from the device. // The buffers are reuseable, which also prevents allocation during inference. // The buffers are mapped to the device during their creation, which prevent lazy mapping each frame inference. @@ -61,9 +61,9 @@ class ServiceNetworkGroupBufferPool public: static Expected> create(uint32_t vdevice_handle); - hailo_status allocate_pool(const std::string &name, size_t frame_size, size_t pool_size); + hailo_status allocate_pool(const std::string &name, hailo_dma_buffer_direction_t direction, size_t frame_size, size_t pool_size); // Used in order to reallocate the pool buffers with different frame_size - hailo_status reallocate_pool(const std::string &name, size_t frame_size); + hailo_status reallocate_pool(const std::string &name, hailo_dma_buffer_direction_t direction, size_t frame_size); ServiceNetworkGroupBufferPool(ServiceNetworkGroupBufferPool &&) = delete; ServiceNetworkGroupBufferPool(const ServiceNetworkGroupBufferPool &) = delete; @@ -72,12 +72,12 @@ public: virtual ~ServiceNetworkGroupBufferPool() = default; ServiceNetworkGroupBufferPool(EventPtr shutdown_event, uint32_t vdevice_handle); - Expected acquire_buffer(const std::string &output_name); - hailo_status return_to_pool(const std::string &output_name, BufferPtr buffer); + Expected acquire_buffer(const std::string &stream_name); + hailo_status return_to_pool(const std::string &stream_name, BufferPtr buffer); hailo_status shutdown(); private: - std::unordered_map m_output_name_to_buffer_pool; + std::unordered_map m_stream_name_to_buffer_pool; EventPtr m_shutdown_event; uint32_t m_vdevice_handle; std::mutex m_mutex; diff --git a/hailort/hailort_service/hailort_rpc_service.cpp b/hailort/hailort_service/hailort_rpc_service.cpp index da67ac0..d40fc1e 100644 --- a/hailort/hailort_service/hailort_rpc_service.cpp +++ b/hailort/hailort_service/hailort_rpc_service.cpp @@ -307,26 +307,23 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD hailo_status HailoRtRpcService::create_buffer_pools_for_ng(uint32_t vdevice_handle, uint32_t ng_handle, uint32_t request_pid, bool allocate_for_raw_streams) { - auto cng_buffer_pool = ServiceNetworkGroupBufferPool::create(vdevice_handle); - CHECK_EXPECTED_AS_STATUS(cng_buffer_pool); + TRY(auto cng_buffer_pool, ServiceNetworkGroupBufferPool::create(vdevice_handle)); auto &cng_buffer_pool_manager = ServiceResourceManager::get_instance(); - auto cng_buffer_pool_handle = cng_buffer_pool_manager.register_resource(request_pid, cng_buffer_pool.release()); + auto cng_buffer_pool_handle = cng_buffer_pool_manager.register_resource(request_pid, cng_buffer_pool); CHECK(cng_buffer_pool_handle == ng_handle, HAILO_INTERNAL_FAILURE, "cng_buffer_pool_handle = {} must be equal to network_group_handle ={}", cng_buffer_pool_handle, ng_handle); if (allocate_for_raw_streams) { // For Async API - The buffer size in the pool will be the stream's hw frame size as used in the infer_model pipeline - auto min_buffer_pool_size = get_min_buffer_pool_size(ng_handle); - CHECK_EXPECTED_AS_STATUS(min_buffer_pool_size); + TRY(const auto min_buffer_pool_size, get_min_buffer_pool_size(ng_handle)); + TRY(const auto streams_infos, get_all_stream_infos(ng_handle)); - auto streams_infos = get_all_stream_infos(ng_handle); - CHECK_EXPECTED_AS_STATUS(streams_infos); - - for (const auto &stream_info : streams_infos.value()) { + for (const auto &stream_info : streams_infos) { if (stream_info.direction == HAILO_D2H_STREAM) { auto allocate_lambda = [&](std::shared_ptr cng_buffer_pool) { - return cng_buffer_pool->allocate_pool(stream_info.name, stream_info.hw_frame_size, min_buffer_pool_size.value()); + return cng_buffer_pool->allocate_pool(stream_info.name, HAILO_DMA_BUFFER_DIRECTION_D2H, + stream_info.hw_frame_size, min_buffer_pool_size); }; CHECK_SUCCESS(cng_buffer_pool_manager.execute(ng_handle, allocate_lambda)); } @@ -465,10 +462,8 @@ hailo_status HailoRtRpcService::add_input_named_buffer(const ProtoTransferReques mem_view = MemoryView::create_const(data, proto_stream_transfer_request.data().size()); } else { // The memory is not aligned to 8, therefore we need to copy the data into a buffer - auto buffer_exp = Buffer::create_shared(data, proto_stream_transfer_request.data().size(), - BufferStorageParams::create_dma()); - CHECK_EXPECTED(buffer_exp); - buffer = buffer_exp.release(); + TRY(buffer, Buffer::create_shared(data, proto_stream_transfer_request.data().size(), + BufferStorageParams::create_dma())); mem_view = MemoryView(*buffer); } @@ -487,7 +482,11 @@ hailo_status HailoRtRpcService::add_input_named_buffer(const ProtoTransferReques enqueue_cb_identifier(vdevice_handle, std::move(cb_identifier)); }; - named_buffers_callbacks.emplace(stream_name, std::make_pair(mem_view, transfer_done)); + BufferRepresentation buffer_representation {}; + buffer_representation.buffer_type = BufferType::VIEW; + buffer_representation.view = mem_view; + + named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_representation, transfer_done)); return HAILO_SUCCESS; } @@ -496,9 +495,7 @@ hailo_status HailoRtRpcService::add_output_named_buffer(const ProtoTransferReque { // Prepare output buffer auto &stream_name = proto_stream_transfer_request.stream_name(); - auto buffer_exp = acquire_buffer_from_cng_pool(ng_handle, stream_name); - CHECK_EXPECTED(buffer_exp); - auto buffer = buffer_exp.release(); + TRY(auto buffer, acquire_buffer_from_cng_pool(ng_handle, stream_name)); // Prepare callback auto cb_idx = proto_stream_transfer_request.cb_idx(); @@ -511,7 +508,11 @@ hailo_status HailoRtRpcService::add_output_named_buffer(const ProtoTransferReque enqueue_cb_identifier(vdevice_handle, std::move(cb_identifier)); }; - named_buffers_callbacks.emplace(stream_name, std::make_pair(MemoryView(*buffer), transfer_done)); + BufferRepresentation buffer_representation {}; + buffer_representation.buffer_type = BufferType::VIEW; + buffer_representation.view = MemoryView(*buffer); + + named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_representation, transfer_done)); return HAILO_SUCCESS; } @@ -566,9 +567,12 @@ Expected HailoRtRpcService::acquire_buffer_from_cng_pool(uint32_t ng_ auto lambda_acquire_buffer = [](std::shared_ptr cng_buffer_pool, const std::string &output_name) { return cng_buffer_pool->acquire_buffer(output_name); }; - auto buffer = cng_buffer_pool_manager.execute>(ng_handle, lambda_acquire_buffer, output_name); - CHECK_EXPECTED(buffer); - return buffer.release(); + TRY(auto buffer, + cng_buffer_pool_manager.execute>( + ng_handle, lambda_acquire_buffer, output_name) + ); + + return buffer; } grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_infer_async(grpc::ServerContext*, @@ -1091,6 +1095,7 @@ void serialize_op_matadata(hailort::net_flow::OpMetadata &op_metadata, ProtoOpMe nms_config_proto->set_background_removal(nms_config.background_removal); nms_config_proto->set_background_removal_index(nms_config.background_removal_index); nms_config_proto->set_cross_classes(nms_config.cross_classes); + nms_config_proto->set_bbox_only(nms_config.bbox_only); } switch (op_metadata.type()) { @@ -1379,7 +1384,8 @@ grpc::Status HailoRtRpcService::OutputVStreams_create(grpc::ServerContext *, con auto &vstream_manager = ServiceResourceManager::get_instance(); for (size_t i = 0; i < vstreams.size(); i++) { auto allocate_lambda = [&](std::shared_ptr cng_buffer_pool) { - return cng_buffer_pool->allocate_pool(vstreams[i].name(), vstreams[i].get_frame_size(), output_params.at(vstreams[i].name()).queue_size); + return cng_buffer_pool->allocate_pool(vstreams[i].name(), HAILO_DMA_BUFFER_DIRECTION_D2H, + vstreams[i].get_frame_size(), output_params.at(vstreams[i].name()).queue_size); }; CHECK_SUCCESS_AS_RPC_STATUS(cng_buffer_pool_manager.execute(network_group_handle, allocate_lambda), reply); @@ -1513,7 +1519,7 @@ grpc::Status HailoRtRpcService::OutputVStream_read(grpc::ServerContext*, const O auto buffer_exp = acquire_buffer_from_cng_pool(ng_handle, vstream_name.value()); CHECK_EXPECTED_AS_RPC_STATUS(buffer_exp, reply); - auto buffer = buffer_exp.release(); + auto buffer = buffer_exp.value(); auto lambda = [](std::shared_ptr output_vstream, MemoryView &buffer) { return output_vstream->read(std::move(buffer)); @@ -1549,10 +1555,10 @@ Expected> HailoRtRpcService::get_all_stream_inf return cng->get_all_stream_infos(); }; auto &manager = ServiceResourceManager::get_instance(); - auto expected_stream_infos = manager.execute>>(ng_handle, lambda); - CHECK_EXPECTED(expected_stream_infos); + TRY(auto stream_infos, + manager.execute>>(ng_handle, lambda)); - return expected_stream_infos.release(); + return stream_infos; } Expected> HailoRtRpcService::get_all_vstream_infos(uint32_t ng_handle) @@ -1561,10 +1567,10 @@ Expected> HailoRtRpcService::get_all_vstream_i return cng->get_all_vstream_infos(); }; auto &manager = ServiceResourceManager::get_instance(); - auto expected_vstream_infos = manager.execute>>(ng_handle, lambda); - CHECK_EXPECTED(expected_vstream_infos); + TRY(auto vstream_infos, + manager.execute>>(ng_handle, lambda)); - return expected_vstream_infos.release(); + return vstream_infos; } grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_all_stream_infos(grpc::ServerContext*, @@ -1683,10 +1689,9 @@ Expected HailoRtRpcService::get_min_buffer_pool_size(uint32_t ng_handle) return cng->get_min_buffer_pool_size(); }; auto &manager = ServiceResourceManager::get_instance(); - auto min_buffer_pool_size = manager.execute>(ng_handle, lambda); - CHECK_EXPECTED(min_buffer_pool_size); + TRY(auto min_buffer_pool_size, manager.execute>(ng_handle, lambda)); - return min_buffer_pool_size.release(); + return min_buffer_pool_size; } grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_min_buffer_pool_size(grpc::ServerContext*, @@ -1899,10 +1904,9 @@ Expected HailoRtRpcService::output_vstream_name(uint32_t vstream_ha return output_vstream->name(); }; auto &manager = ServiceResourceManager::get_instance(); - auto name = manager.execute>(vstream_handle, lambda); - CHECK_EXPECTED(name); + TRY(auto name, manager.execute>(vstream_handle, lambda)); - return name.release(); + return name; } Expected HailoRtRpcService::output_vstream_frame_size(uint32_t vstream_handle) @@ -1911,10 +1915,9 @@ Expected HailoRtRpcService::output_vstream_frame_size(uint32_t vstream_h return output_vstream->get_frame_size(); }; auto &manager = ServiceResourceManager::get_instance(); - auto frame_size = manager.execute>(vstream_handle, lambda); - CHECK_EXPECTED(frame_size); + TRY(auto frame_size, manager.execute>(vstream_handle, lambda)); - return frame_size.release(); + return frame_size; } grpc::Status HailoRtRpcService::OutputVStream_name(grpc::ServerContext*, const VStream_name_Request *request, @@ -2194,15 +2197,12 @@ grpc::Status HailoRtRpcService::OutputVStream_set_nms_iou_threshold(grpc::Server hailo_status HailoRtRpcService::update_buffer_size_in_pool(uint32_t vstream_handle, uint32_t network_group_handle) { - auto vstream_name = output_vstream_name(vstream_handle); - CHECK_EXPECTED(vstream_name); - - auto frame_size = output_vstream_frame_size(vstream_handle); - CHECK_EXPECTED(frame_size); + TRY(const auto vstream_name, output_vstream_name(vstream_handle)); + TRY(const auto frame_size, output_vstream_frame_size(vstream_handle)); auto &cng_buffer_pool_manager = ServiceResourceManager::get_instance(); auto allocate_lambda = [&](std::shared_ptr cng_buffer_pool) { - return cng_buffer_pool->reallocate_pool(vstream_name.release(), frame_size.release()); + return cng_buffer_pool->reallocate_pool(vstream_name, HAILO_DMA_BUFFER_DIRECTION_D2H, frame_size); }; CHECK_SUCCESS(cng_buffer_pool_manager.execute(network_group_handle, allocate_lambda)); diff --git a/hailort/hailort_service/service_resource_manager.hpp b/hailort/hailort_service/service_resource_manager.hpp index 320b722..82b88d1 100644 --- a/hailort/hailort_service/service_resource_manager.hpp +++ b/hailort/hailort_service/service_resource_manager.hpp @@ -19,6 +19,8 @@ #include #include +#define SINGLE_CLIENT_PID (0) + namespace hailort { @@ -48,10 +50,7 @@ public: K execute(uint32_t handle, Func &lambda, Args... args) { std::unique_lock lock(m_mutex); - auto resource_expected = resource_lookup(handle); - CHECK_EXPECTED(resource_expected); - auto resource = resource_expected.release(); - + TRY(auto resource, resource_lookup(handle)); assert(contains(m_resources_mutexes, handle)); std::shared_lock resource_lock(m_resources_mutexes[handle]); lock.unlock(); @@ -64,10 +63,7 @@ public: hailo_status execute(uint32_t handle, Func &lambda, Args... args) { std::unique_lock lock(m_mutex); - auto resource_expected = resource_lookup(handle); - CHECK_EXPECTED_AS_STATUS(resource_expected); - auto resource = resource_expected.release(); - + TRY(auto resource, resource_lookup(handle)); assert(contains(m_resources_mutexes, handle)); std::shared_lock resource_lock(m_resources_mutexes[handle]); lock.unlock(); @@ -90,10 +86,7 @@ public: Expected dup_handle(uint32_t handle, uint32_t pid) { std::unique_lock lock(m_mutex); - auto resource_expected = resource_lookup(handle); - CHECK_EXPECTED(resource_expected); - auto resource = resource_expected.release(); - + TRY(auto resource, resource_lookup(handle)); assert(contains(m_resources_mutexes, handle)); std::unique_lock resource_lock(m_resources_mutexes[handle]); resource->pids.insert(pid); @@ -118,7 +111,7 @@ public: { std::unique_lock resource_lock(m_resources_mutexes[handle]); resource->pids.erase(pid); - if (all_pids_dead(resource)) { + if ((SINGLE_CLIENT_PID == pid) || all_pids_dead(resource)) { release_resource = true; res = resource->resource; m_resources.erase(handle); diff --git a/hailort/hailort_service/vdevice_callbacks_queue.hpp b/hailort/hailort_service/vdevice_callbacks_queue.hpp index 5eaab7f..821c71a 100644 --- a/hailort/hailort_service/vdevice_callbacks_queue.hpp +++ b/hailort/hailort_service/vdevice_callbacks_queue.hpp @@ -29,14 +29,12 @@ class VDeviceCallbacksQueue final public: static Expected> create(uint32_t max_queue_size) { - auto shutdown_event_exp = Event::create_shared(Event::State::not_signalled); - CHECK_EXPECTED(shutdown_event_exp); - auto shutdown_event = shutdown_event_exp.release(); + TRY(auto shutdown_event, Event::create_shared(Event::State::not_signalled)); - auto cb_ids_queue = SpscQueue::create(max_queue_size, shutdown_event, HAILO_INFINITE_TIMEOUT); - CHECK_EXPECTED(cb_ids_queue); + TRY(auto cb_ids_queue, + SpscQueue::create(max_queue_size, shutdown_event, HAILO_INFINITE_TIMEOUT)); - auto queue_ptr = make_unique_nothrow(cb_ids_queue.release(), shutdown_event); + auto queue_ptr = make_unique_nothrow(std::move(cb_ids_queue), shutdown_event); CHECK_AS_EXPECTED(nullptr != queue_ptr, HAILO_OUT_OF_HOST_MEMORY); return queue_ptr; @@ -57,16 +55,8 @@ public: Expected dequeue() { - auto callback_id = m_callbacks_ids_queue.dequeue(); - if (HAILO_SHUTDOWN_EVENT_SIGNALED == callback_id.status()) { - return make_unexpected(callback_id.status()); - } - else if (HAILO_TIMEOUT == callback_id.status()) { - LOGGER__WARNING("Failed to dequeue callback_id because the queue is empty, status={}", HAILO_TIMEOUT); - return make_unexpected(callback_id.status()); - } - CHECK_EXPECTED(callback_id); - + TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto callback_id, + m_callbacks_ids_queue.dequeue()); return callback_id; } diff --git a/hailort/hailortcli/benchmark_command.cpp b/hailort/hailortcli/benchmark_command.cpp index 57fa385..fb0bef2 100644 --- a/hailort/hailortcli/benchmark_command.cpp +++ b/hailort/hailortcli/benchmark_command.cpp @@ -65,35 +65,32 @@ hailo_status BenchmarkCommand::execute() std::cout << "Starting Measurements..." << std::endl; std::cout << "Measuring FPS in hw_only mode" << std::endl; - auto hw_only_mode_info = hw_only_mode(); - CHECK_EXPECTED_AS_STATUS(hw_only_mode_info, "hw_only measuring failed"); + TRY(auto hw_only_mode_info, hw_only_mode(), "hw_only measuring failed"); std::cout << "Measuring FPS " << (!m_not_measure_power ? "and Power " : "") << "in streaming mode" << std::endl; - auto streaming_mode_info = fps_streaming_mode(); - CHECK_EXPECTED_AS_STATUS(streaming_mode_info, "FPS in streaming mode failed"); + TRY(auto streaming_mode_info, fps_streaming_mode(), "FPS in streaming mode failed"); // TODO - HRT-6931 - measure latency only in the case of single device. std::cout << "Measuring HW Latency" << std::endl; - auto latency_info = latency(); - CHECK_EXPECTED_AS_STATUS(latency_info, "Latency measuring failed"); + TRY(auto latency_info, latency(), "Latency measuring failed"); - assert(hw_only_mode_info->network_group_results().size() == streaming_mode_info->network_group_results().size()); - assert(latency_info->network_group_results().size() == streaming_mode_info->network_group_results().size()); + assert(hw_only_mode_info.network_group_results().size() == streaming_mode_info.network_group_results().size()); + assert(latency_info.network_group_results().size() == streaming_mode_info.network_group_results().size()); std::cout << std::endl; std::cout << "=======" << std::endl; std::cout << "Summary" << std::endl; std::cout << "=======" << std::endl; - for (auto &hw_only_res : hw_only_mode_info->network_group_results()) { + for (auto &hw_only_res : hw_only_mode_info.network_group_results()) { auto network_group_name = hw_only_res.network_group_name(); - auto streaming_res = std::find_if(streaming_mode_info->network_group_results().begin(), streaming_mode_info->network_group_results().end(), + auto streaming_res = std::find_if(streaming_mode_info.network_group_results().begin(), streaming_mode_info.network_group_results().end(), [network_group_name] (NetworkGroupInferResult &infer_results) { return (infer_results.network_group_name() == network_group_name); }); - CHECK(streaming_mode_info->network_group_results().end() != streaming_res, HAILO_INTERNAL_FAILURE, "Failed to fun streaming results for network group {}", network_group_name); + CHECK(streaming_mode_info.network_group_results().end() != streaming_res, HAILO_INTERNAL_FAILURE, "Failed to fun streaming results for network group {}", network_group_name); - auto latency_res = std::find_if(latency_info->network_group_results().begin(), latency_info->network_group_results().end(), + auto latency_res = std::find_if(latency_info.network_group_results().begin(), latency_info.network_group_results().end(), [network_group_name] (NetworkGroupInferResult &infer_results) { return (infer_results.network_group_name() == network_group_name); }); - CHECK(latency_info->network_group_results().end() != latency_res, HAILO_INTERNAL_FAILURE, "Failed to fun latency results for network group {}", network_group_name); + CHECK(latency_info.network_group_results().end() != latency_res, HAILO_INTERNAL_FAILURE, "Failed to fun latency results for network group {}", network_group_name); std::cout << "FPS (hw_only) = " << hw_only_res.fps().value() <data(); const auto &power_units = pair.second->power_units(); @@ -116,11 +113,9 @@ hailo_status BenchmarkCommand::execute() if (!m_csv_file_path.empty()){ m_params.csv_output = m_csv_file_path; - auto printer = InferStatsPrinter::create(m_params, false); - CHECK_EXPECTED_AS_STATUS(printer, "Failed to initialize infer stats printer"); - printer->print_benchmark_csv_header(); - printer->print_benchmark_csv(hw_only_mode_info.value(), - streaming_mode_info.value(), latency_info.value()); + TRY(auto printer, InferStatsPrinter::create(m_params, false), "Failed to initialize infer stats printer"); + printer.print_benchmark_csv_header(); + printer.print_benchmark_csv(hw_only_mode_info, streaming_mode_info, latency_info); } return HAILO_SUCCESS; } diff --git a/hailort/hailortcli/board_config_command.cpp b/hailort/hailortcli/board_config_command.cpp index 8619880..11baca0 100644 --- a/hailort/hailortcli/board_config_command.cpp +++ b/hailort/hailortcli/board_config_command.cpp @@ -35,13 +35,12 @@ hailo_status BoardConfigReadSubcommand::execute_on_device(Device &device) CHECK_SUCCESS(status, "'board-config read' command should get a specific device-id."); - auto buffer = device.read_board_config(); - CHECK_EXPECTED_AS_STATUS(buffer, "Failed reading board config from device"); + TRY(auto buffer, device.read_board_config(), "Failed reading board config from device"); auto output_file = std::ofstream(m_output_file_path, std::ios::out | std::ios::binary); CHECK(output_file.is_open(), HAILO_OPEN_FILE_FAILURE, "Failed opening output file {} with errno: {}", m_output_file_path, errno); - output_file.write(reinterpret_cast(buffer->data()), buffer->size()); + output_file.write(reinterpret_cast(buffer.data()), buffer.size()); CHECK(output_file.good(), HAILO_FILE_OPERATION_FAILURE, "Failed writing board config into file {}.", m_output_file_path); return HAILO_SUCCESS; @@ -57,10 +56,8 @@ BoardConfigWriteSubcommand::BoardConfigWriteSubcommand(CLI::App &parent_app) : hailo_status BoardConfigWriteSubcommand::execute_on_device(Device &device) { - auto buffer = read_binary_file(m_input_file_path); - CHECK_EXPECTED_AS_STATUS(buffer); - - hailo_status status = device.write_board_config(MemoryView(buffer.value())); + TRY(auto buffer, read_binary_file(m_input_file_path)); + hailo_status status = device.write_board_config(MemoryView(buffer)); CHECK_SUCCESS(status, "Failed writing board config to device."); return HAILO_SUCCESS; diff --git a/hailort/hailortcli/command.cpp b/hailort/hailortcli/command.cpp index dae8600..96c45eb 100644 --- a/hailort/hailortcli/command.cpp +++ b/hailort/hailortcli/command.cpp @@ -34,13 +34,21 @@ hailo_status ContainerCommand::execute() } DeviceCommand::DeviceCommand(CLI::App *app) : - Command(app) + Command(app), + m_show_stdout(true) { add_device_options(m_app, m_device_params); } +void DeviceCommand::pre_execute() +{ + // Do nothing by default +} + hailo_status DeviceCommand::execute() { + pre_execute(); + auto devices = create_devices(m_device_params); if (!devices) { return devices.status(); @@ -52,7 +60,9 @@ hailo_status DeviceCommand::execute_on_devices(std::vectorget_dev_id() << std::endl; + if (m_show_stdout) { + std::cout << "Executing on device: " << device->get_dev_id() << std::endl; + } auto execute_status = execute_on_device(*device); if (HAILO_SUCCESS != execute_status) { std::cerr << "Failed to execute on device: " << device->get_dev_id() << ". status= " << execute_status << std::endl; @@ -66,9 +76,8 @@ hailo_status DeviceCommand::validate_specific_device_is_given() { if ((1 != m_device_params.device_ids.size()) || contains(m_device_params.device_ids, std::string("*"))) { // No specific device-id given, make sure there is only 1 device on the machine. - auto scan_res = Device::scan(); - CHECK_EXPECTED_AS_STATUS(scan_res, "Failed to scan for devices"); - if (1 != scan_res->size()) { + TRY(auto scan_res, Device::scan(), "Failed to scan for devices"); + if (1 != scan_res.size()) { return HAILO_INVALID_OPERATION; } } diff --git a/hailort/hailortcli/command.hpp b/hailort/hailortcli/command.hpp index 2002a8d..eb4e051 100644 --- a/hailort/hailortcli/command.hpp +++ b/hailort/hailortcli/command.hpp @@ -65,7 +65,9 @@ public: protected: hailo_device_params m_device_params; + bool m_show_stdout; // Set to false in subclasses to disable this class' prints to stdout + virtual void pre_execute(); // Override this function to do any pre-execution setup virtual hailo_status execute_on_device(Device &device) = 0; hailo_status execute_on_devices(std::vector> &devices); hailo_status validate_specific_device_is_given(); diff --git a/hailort/hailortcli/download_action_list_command.cpp b/hailort/hailortcli/download_action_list_command.cpp index 7d57108..a1256ed 100644 --- a/hailort/hailortcli/download_action_list_command.cpp +++ b/hailort/hailortcli/download_action_list_command.cpp @@ -36,13 +36,8 @@ DownloadActionListCommand::DownloadActionListCommand(CLI::App &parent_app) : hailo_status DownloadActionListCommand::execute(Device &device, const std::string &output_file_path, const ConfiguredNetworkGroupVector &network_groups, const std::string &hef_file_path) { - auto expected_action_list_json = init_json_object(device, hef_file_path); - CHECK_EXPECTED_AS_STATUS(expected_action_list_json); - auto action_list_json = expected_action_list_json.value(); - - auto network_groups_list_json = parse_network_groups(device, network_groups); - CHECK_EXPECTED_AS_STATUS(network_groups_list_json); - action_list_json["network_groups"] = network_groups_list_json.release(); + TRY(auto action_list_json, init_json_object(device, hef_file_path)); + TRY(action_list_json["network_groups"], parse_network_groups(device, network_groups)); return write_to_json(action_list_json, output_file_path); } @@ -50,9 +45,7 @@ hailo_status DownloadActionListCommand::execute(Device &device, const std::strin hailo_status DownloadActionListCommand::execute(Device &device, std::shared_ptr network_group, uint16_t batch_size, ordered_json &action_list_json_param, double fps, uint32_t network_group_index) { - auto expected_network_groups_list_json = parse_network_group(device, network_group, network_group_index); - CHECK_EXPECTED_AS_STATUS(expected_network_groups_list_json); - auto network_groups_list_json = expected_network_groups_list_json.release(); + TRY(auto network_groups_list_json, parse_network_group(device, network_group, network_group_index)); network_groups_list_json[0]["batch_size"] = batch_size; network_groups_list_json[0]["fps"] = fps; action_list_json_param["runs"] += network_groups_list_json[0]; @@ -73,30 +66,25 @@ hailo_status DownloadActionListCommand::write_to_json(ordered_json &action_list_ Expected DownloadActionListCommand::init_json_object(Device &device, const std::string &hef_file_path) { ordered_json action_list_json = {}; - auto curr_time = CliCommon::current_time_to_string(); - CHECK_EXPECTED(curr_time); + TRY(auto curr_time, CliCommon::current_time_to_string()); + TRY(auto chip_arch, device.get_architecture()); - auto chip_arch = device.get_architecture(); - CHECK_EXPECTED(chip_arch); unsigned int clock_cycle = 0; // TODO - HRT-8046 Implement extended device info for hailo15 - if (HAILO_ARCH_HAILO15H == chip_arch.value()) { + if (HAILO_ARCH_HAILO15H == chip_arch) { clock_cycle = HAILO15_VPU_CORE_CPU_DEFAULT_FREQ_MHZ; } else { - auto extended_info = device.get_extended_device_information(); - CHECK_EXPECTED(extended_info); - clock_cycle = (extended_info->neural_network_core_clock_rate / NN_CORE_TO_TIMER_FREQ_FACTOR) / MHz; + TRY(auto extended_info, device.get_extended_device_information()); + clock_cycle = (extended_info.neural_network_core_clock_rate / NN_CORE_TO_TIMER_FREQ_FACTOR) / MHz; } action_list_json["version"] = ACTION_LIST_FORMAT_VERSION(); - action_list_json["creation_time"] = curr_time.release(); + action_list_json["creation_time"] = curr_time; action_list_json["clock_cycle_MHz"] = clock_cycle; action_list_json["hef"] = json({}); if (!hef_file_path.empty()) { - auto hef_info = parse_hef_metadata(hef_file_path); - CHECK_EXPECTED(hef_info); - action_list_json["hef"] = hef_info.release(); + TRY(action_list_json["hef"], parse_hef_metadata(hef_file_path)); } action_list_json["runs"] = ordered_json::array(); @@ -123,12 +111,11 @@ Expected DownloadActionListCommand::parse_hef_metadata(const std:: CHECK_AS_EXPECTED(is_valid_hef(hef_file_path), HAILO_INTERNAL_FAILURE, "Hef '{}' is not valid", hef_file_path); - auto hef_md5 = calc_md5_hexdigest(hef_file_path); - CHECK_EXPECTED(hef_md5); + TRY(auto hef_md5, calc_md5_hexdigest(hef_file_path)); ordered_json hef_info_json = { {"path", hef_file_path}, - {"file_hash", hef_md5.release()} + {"file_hash", hef_md5} }; return hef_info_json; @@ -144,13 +131,12 @@ bool DownloadActionListCommand::is_valid_hef(const std::string &hef_file_path) Expected DownloadActionListCommand::calc_md5_hexdigest(const std::string &hef_file_path) { - auto hef_bin = read_binary_file(hef_file_path); - CHECK_EXPECTED(hef_bin); + TRY(auto hef_bin, read_binary_file(hef_file_path)); MD5_CTX md5_ctx{}; MD5_SUM_t md5_sum{}; MD5_Init(&md5_ctx); - MD5_Update(&md5_ctx, hef_bin->data(), hef_bin->size()); + MD5_Update(&md5_ctx, hef_bin.data(), hef_bin.size()); MD5_Final(md5_sum, &md5_ctx); const bool LOWERCASE = false; @@ -300,6 +286,18 @@ Expected DownloadActionListCommand::parse_action_data(uint32_t bas data_json = *reinterpret_cast(action); action_length_local = sizeof(CONTEXT_SWITCH_DEFS__activate_ddr_buffer_output_data_t); break; + case CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_INPUT: + data_json = *reinterpret_cast(action); + action_length_local = sizeof(CONTEXT_SWITCH_DEFS__activate_cache_input_data_t); + break; + case CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_OUTPUT: + data_json = *reinterpret_cast(action); + action_length_local = sizeof(CONTEXT_SWITCH_DEFS__activate_cache_output_data_t); + break; + case CONTEXT_SWITCH_DEFS__ACTION_TYPE_WAIT_FOR_CACHE_UPDATED: + data_json = json({}); + action_length_local = 0; + break; case CONTEXT_SWITCH_DEFS__ACTION_TYPE_CHANGE_VDMA_TO_STREAM_MAPPING: data_json = *reinterpret_cast(action); action_length_local = sizeof(CONTEXT_SWITCH_DEFS__change_vdma_to_stream_mapping_data_t); @@ -402,12 +400,11 @@ Expected DownloadActionListCommand::parse_single_action(uint32_t b static const bool DONT_SET_SUB_ACTION_INDEX = false; uint32_t action_data_length = 0; - auto json = parse_action_data(base_address, &context_action_list[current_buffer_offset], current_buffer_offset, &action_data_length, - action_header->action_type, time_stamp_local, 0, DONT_SET_SUB_ACTION_INDEX, is_repeated, num_repeated, sub_action_type); - CHECK_EXPECTED(json); + TRY(auto json, parse_action_data(base_address, &context_action_list[current_buffer_offset], current_buffer_offset, &action_data_length, + action_header->action_type, time_stamp_local, 0, DONT_SET_SUB_ACTION_INDEX, is_repeated, num_repeated, sub_action_type)); *action_length = static_cast(action_length_local + action_data_length); *time_stamp = time_stamp_local; - return json.release(); + return json; } Expected DownloadActionListCommand::parse_context(Device &device, uint32_t network_group_id, @@ -417,47 +414,44 @@ Expected DownloadActionListCommand::parse_context(Device &device, uint32_t action_list_base_address = 0; uint32_t batch_counter = 0; - auto action_list = device.download_context_action_list(network_group_id, converted_context_type, context_index, - &action_list_base_address, &batch_counter); - CHECK_EXPECTED(action_list); + TRY(auto action_list, device.download_context_action_list(network_group_id, converted_context_type, context_index, + &action_list_base_address, &batch_counter)); // Needs to fit in 2 bytes due to firmware limitation of action list size - CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(action_list->size()), HAILO_INTERNAL_FAILURE, - "Action list size is expected to fit in 2B. actual size is {}", action_list->size()); + CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(action_list.size()), HAILO_INTERNAL_FAILURE, + "Action list size is expected to fit in 2B. actual size is {}", action_list.size()); ordered_json context_json { {"action_list_base_address", action_list_base_address}, - {"action_list_size", action_list->size() }, + {"action_list_size", action_list.size() }, {"batch_counter", batch_counter}, {"context_name", context_name}, }; ordered_json action_list_json; uint16_t current_buffer_offset = 0; - while (current_buffer_offset < action_list->size()) { + while (current_buffer_offset < action_list.size()) { bool is_repeated = false; uint8_t num_repeated = 0; CONTEXT_SWITCH_DEFS__ACTION_TYPE_t sub_action_type = CONTEXT_SWITCH_DEFS__ACTION_TYPE_COUNT; uint32_t single_action_length = 0; uint32_t timestamp = 0; - auto action_json = parse_single_action(action_list_base_address, action_list->data(), - current_buffer_offset, &single_action_length, &is_repeated, &num_repeated, &sub_action_type, ×tamp); - CHECK_EXPECTED(action_json); + TRY(auto action_json, parse_single_action(action_list_base_address, action_list.data(), + current_buffer_offset, &single_action_length, &is_repeated, &num_repeated, &sub_action_type, ×tamp)); current_buffer_offset = (uint16_t)(current_buffer_offset + single_action_length); - action_list_json.emplace_back(action_json.release()); + action_list_json.emplace_back(std::move(action_json)); if (is_repeated) { for (uint8_t index_in_repeated_block = 0; index_in_repeated_block < num_repeated; index_in_repeated_block++) { uint32_t sub_action_length = 0; - auto repeated_action_json = parse_single_repeated_action(action_list_base_address, - action_list->data() + current_buffer_offset, current_buffer_offset, &sub_action_length, - sub_action_type, timestamp, index_in_repeated_block); - CHECK_EXPECTED(repeated_action_json); + TRY(auto repeated_action_json, parse_single_repeated_action(action_list_base_address, + action_list.data() + current_buffer_offset, current_buffer_offset, &sub_action_length, + sub_action_type, timestamp, index_in_repeated_block)); current_buffer_offset = (uint16_t)(current_buffer_offset + sub_action_length); - action_list_json.emplace_back(repeated_action_json.release()); + action_list_json.emplace_back(std::move(repeated_action_json)); } } } - CHECK_AS_EXPECTED(current_buffer_offset == action_list->size(), HAILO_INTERNAL_FAILURE, + CHECK_AS_EXPECTED(current_buffer_offset == action_list.size(), HAILO_INTERNAL_FAILURE, "PARSING ERROR ! Reached forbidden memory space"); context_json["actions"] = action_list_json; @@ -473,24 +467,21 @@ double DownloadActionListCommand::get_accumulator_mean_value(const AccumulatorPt Expected DownloadActionListCommand::parse_network_groups(Device &device, const ConfiguredNetworkGroupVector &network_groups) { - const auto number_of_dynamic_contexts_per_network_group = device.get_number_of_dynamic_contexts_per_network_group(); - CHECK_EXPECTED(number_of_dynamic_contexts_per_network_group); + TRY(const auto number_of_dynamic_contexts_per_network_group, device.get_number_of_dynamic_contexts_per_network_group()); - auto number_of_network_groups = (uint32_t)number_of_dynamic_contexts_per_network_group->size(); + auto number_of_network_groups = (uint32_t)number_of_dynamic_contexts_per_network_group.size(); ordered_json network_group_list_json; for (uint32_t network_group_index = 0; network_group_index < number_of_network_groups; network_group_index++) { auto &network_group = (network_group_index < network_groups.size()) ? network_groups[network_group_index] : nullptr; - auto expected_json_file = parse_network_group(device, network_group, network_group_index); - CHECK_EXPECTED(expected_json_file); - network_group_list_json.emplace_back(expected_json_file.value()); + TRY(auto json_file, parse_network_group(device, network_group, network_group_index)); + network_group_list_json.emplace_back(std::move(json_file)); } return network_group_list_json; } Expected DownloadActionListCommand::parse_network_group(Device &device, const std::shared_ptr network_group, uint32_t network_group_id) { - const auto number_of_dynamic_contexts_per_network_group = device.get_number_of_dynamic_contexts_per_network_group(); - CHECK_EXPECTED(number_of_dynamic_contexts_per_network_group); + TRY(const auto number_of_dynamic_contexts_per_network_group, device.get_number_of_dynamic_contexts_per_network_group()); ordered_json network_group_list_json; // TODO: network_group_name via Hef::get_network_groups_names (HRT-5997) @@ -510,30 +501,26 @@ Expected DownloadActionListCommand::parse_network_group(Device &de network_group->get_deactivation_time_accumulator()); } - auto activation_context_json = parse_context(device, network_group_id, - CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION, 0, "activation"); - CHECK_EXPECTED(activation_context_json); - network_group_json["contexts"].emplace_back(activation_context_json.release()); + TRY(auto activation_context_json, parse_context(device, network_group_id, + CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION, 0, "activation")); + network_group_json["contexts"].emplace_back(std::move(activation_context_json)); - auto preliminary_context_json = parse_context(device, network_group_id, - CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY, 0, "preliminary"); - CHECK_EXPECTED(preliminary_context_json); - network_group_json["contexts"].emplace_back(preliminary_context_json.release()); + TRY(auto preliminary_context_json, parse_context(device, network_group_id, + CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY, 0, "preliminary")); + network_group_json["contexts"].emplace_back(std::move(preliminary_context_json)); - const auto dynamic_contexts_count = number_of_dynamic_contexts_per_network_group.value()[network_group_id]; + const auto dynamic_contexts_count = number_of_dynamic_contexts_per_network_group[network_group_id]; for (uint16_t context_index = 0; context_index < dynamic_contexts_count; context_index++) { - auto context_json = parse_context(device, network_group_id, + TRY(auto context_json, parse_context(device, network_group_id, CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC, context_index, - fmt::format("dynamic_{}", context_index)); - CHECK_EXPECTED(context_json); + fmt::format("dynamic_{}", context_index))); - network_group_json["contexts"].emplace_back(context_json.release()); + network_group_json["contexts"].emplace_back(std::move(context_json)); } - auto batch_switching_context_json = parse_context(device, network_group_id, - CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING, 0, "batch_switching"); - CHECK_EXPECTED(batch_switching_context_json); - network_group_json["contexts"].emplace_back(batch_switching_context_json.release()); + TRY(auto batch_switching_context_json, parse_context(device, network_group_id, + CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING, 0, "batch_switching")); + network_group_json["contexts"].emplace_back(std::move(batch_switching_context_json)); network_group_list_json.emplace_back(network_group_json); @@ -595,6 +582,19 @@ void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_ddr_buffer_output_data j["stream_index"] = data.stream_index; } +void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_cache_input_data_t &data) +{ + j = unpack_vdma_channel_id(data); + j["stream_index"] = data.stream_index; +} + +void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_cache_output_data_t &data) +{ + j = unpack_vdma_channel_id(data); + j["stream_index"] = data.stream_index; +} + + // Needs to be backwards compatible, so we use "channel_index" instead of "vdma_channel_index". void to_json(json& j, const CONTEXT_SWITCH_DEFS__fetch_cfg_channel_descriptors_action_data_t& data) { uint8_t engine_index = 0; diff --git a/hailort/hailortcli/download_action_list_command.hpp b/hailort/hailortcli/download_action_list_command.hpp index 0cbb885..cc88691 100644 --- a/hailort/hailortcli/download_action_list_command.hpp +++ b/hailort/hailortcli/download_action_list_command.hpp @@ -84,6 +84,9 @@ static std::pair mapping[] = { {CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_INTER_CONTEXT_OUTPUT, "activate_inter_context_output"}, {CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_DDR_BUFFER_INPUT, "activate_ddr_buffer_input"}, {CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_DDR_BUFFER_OUTPUT, "activate_ddr_buffer_output"}, + {CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_INPUT, "activate_cache_input"}, + {CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_OUTPUT, "activate_cache_output"}, + {CONTEXT_SWITCH_DEFS__ACTION_TYPE_WAIT_FOR_CACHE_UPDATED, "wait_for_cache_updated"}, {CONTEXT_SWITCH_DEFS__ACTION_TYPE_DEACTIVATE_VDMA_CHANNEL, "deactivate_vdma_channel"}, {CONTEXT_SWITCH_DEFS__ACTION_TYPE_VALIDATE_VDMA_CHANNEL, "validate_vdma_channel"}, {CONTEXT_SWITCH_DEFS__ACTION_TYPE_CHANGE_VDMA_TO_STREAM_MAPPING, "change_vdma_to_stream_mapping"}, @@ -137,6 +140,8 @@ void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_ddr_buffer_input_data_ void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_boundary_output_data_t &data); void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_inter_context_output_data_t &data); void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_ddr_buffer_output_data_t &data); +void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_cache_input_data_t &data); +void to_json(json &j, const CONTEXT_SWITCH_DEFS__activate_cache_output_data_t &data); void to_json(json &j, const CONTEXT_SWITCH_DEFS__enable_lcu_action_default_data_t &data); void to_json(json &j, const CONTEXT_SWITCH_DEFS__enable_lcu_action_non_default_data_t &data); void to_json(json &j, const CONTEXT_SWITCH_DEFS__disable_lcu_action_data_t &data); diff --git a/hailort/hailortcli/fw_config_command.cpp b/hailort/hailortcli/fw_config_command.cpp index 6c5ef40..68d0f81 100644 --- a/hailort/hailortcli/fw_config_command.cpp +++ b/hailort/hailortcli/fw_config_command.cpp @@ -22,12 +22,11 @@ hailo_status FwConfigReadSubcommand::execute_on_device(Device &device) CHECK_SUCCESS(status, "'fw-config read' command should get a specific device-id."); - auto user_config_buffer = device.read_user_config(); - CHECK_EXPECTED_AS_STATUS(user_config_buffer, "Failed reading user config from device"); + TRY(auto user_config_buffer, device.read_user_config(), "Failed reading user config from device"); status = FwConfigJsonSerializer::deserialize_config( - *reinterpret_cast(user_config_buffer->data()), - user_config_buffer->size(), m_output_file); + *reinterpret_cast(user_config_buffer.data()), + user_config_buffer.size(), m_output_file); CHECK_SUCCESS(status); return HAILO_SUCCESS; @@ -43,20 +42,16 @@ FwConfigWriteSubcommand::FwConfigWriteSubcommand(CLI::App &parent_app) : hailo_status FwConfigWriteSubcommand::execute_on_device(Device &device) { - auto config_buffer = Buffer::create(FLASH_USER_CONFIG_SECTION_SIZE); - CHECK_EXPECTED_AS_STATUS(config_buffer); - - auto config_size = FwConfigJsonSerializer::serialize_config( - *reinterpret_cast(config_buffer->data()), config_buffer->size(), m_input_file); - CHECK_EXPECTED_AS_STATUS(config_size); + TRY(auto config_buffer, Buffer::create(FLASH_USER_CONFIG_SECTION_SIZE)); + TRY(auto config_size, FwConfigJsonSerializer::serialize_config( + *reinterpret_cast(config_buffer.data()), config_buffer.size(), m_input_file)); - // We only need to write config_size.value() bytes from config_buffer, so we "resize" the buffer - CHECK(config_buffer->size() >= config_size.value(), HAILO_INTERNAL_FAILURE, - "Unexpected config size {} (max_size={})", config_size.value(), config_buffer->size()); - auto resized_config_buffer = Buffer::create(config_buffer->data(), config_size.value()); - CHECK_EXPECTED_AS_STATUS(resized_config_buffer); + // We only need to write 'config_size' bytes from config_buffer, so we "resize" the buffer + CHECK(config_buffer.size() >= config_size, HAILO_INTERNAL_FAILURE, + "Unexpected config size {} (max_size={})", config_size, config_buffer.size()); + TRY(auto resized_config_buffer, Buffer::create(config_buffer.data(), config_size)); - hailo_status status = device.write_user_config(MemoryView(resized_config_buffer.value())); + hailo_status status = device.write_user_config(MemoryView(resized_config_buffer)); CHECK_SUCCESS(status, "Failed writing user firmware configuration to device"); return HAILO_SUCCESS; @@ -74,17 +69,15 @@ FwConfigSerializeSubcommand::FwConfigSerializeSubcommand(CLI::App &parent_app) : hailo_status FwConfigSerializeSubcommand::execute() { - auto config_buffer = Buffer::create(FLASH_USER_CONFIG_SECTION_SIZE); - CHECK_EXPECTED_AS_STATUS(config_buffer); + TRY(auto config_buffer, Buffer::create(FLASH_USER_CONFIG_SECTION_SIZE)); - USER_CONFIG_header_t *config_header = reinterpret_cast(config_buffer->data()); - auto config_size = FwConfigJsonSerializer::serialize_config(*config_header, config_buffer->size(), m_input_file); - CHECK_EXPECTED_AS_STATUS(config_size); + USER_CONFIG_header_t *config_header = reinterpret_cast(config_buffer.data()); + TRY(auto config_size, FwConfigJsonSerializer::serialize_config(*config_header, config_buffer.size(), m_input_file)); std::ofstream ofs(m_output_file, std::ios::out | std::ios::binary); CHECK(ofs.good(), HAILO_OPEN_FILE_FAILURE, "Failed opening file: {}, with errno: {}", m_output_file, errno); - ofs.write(reinterpret_cast(config_header), config_size.value()); + ofs.write(reinterpret_cast(config_header), config_size); CHECK(ofs.good(), HAILO_FILE_OPERATION_FAILURE, "Failed writing binary firmware configuration to file: {}, with errno: {}", m_output_file, errno); diff --git a/hailort/hailortcli/fw_config_serializer.cpp b/hailort/hailortcli/fw_config_serializer.cpp index a78981a..4f4271c 100644 --- a/hailort/hailortcli/fw_config_serializer.cpp +++ b/hailort/hailortcli/fw_config_serializer.cpp @@ -110,18 +110,17 @@ hailo_status FwConfigJsonSerializer::dump_config(const ordered_json &config_json hailo_status FwConfigJsonSerializer::deserialize_config(const USER_CONFIG_header_t &user_config_header, size_t config_size, const std::string &file_path) { try { - auto categories = get_deserialize_vector(); - CHECK_EXPECTED_AS_STATUS(categories); + TRY(const auto categories, get_deserialize_vector()); ordered_json config_json; size_t current_deserialized_data_size = 0; uintptr_t current_entry_offset = (uintptr_t)(&(user_config_header.entries)); for (size_t i = 0; i < user_config_header.entry_count; i++) { USER_CONFIG_ENTRY_t *config_entry = reinterpret_cast(current_entry_offset); - CHECK(config_entry->category < categories->size(), HAILO_INTERNAL_FAILURE, - "Category id is out of bounds. Category id = {}, Max category id = {}", config_entry->category, (categories->size()-1)); + CHECK(config_entry->category < categories.size(), HAILO_INTERNAL_FAILURE, + "Category id is out of bounds. Category id = {}, Max category id = {}", config_entry->category, (categories.size()-1)); - auto category = categories.value()[config_entry->category]; + auto category = categories[config_entry->category]; CHECK(config_entry->entry_id < category.size(), HAILO_INTERNAL_FAILURE, "Entry id is out of bounds. Entry id = {}, Max entry id = {}", config_entry->entry_id, (category.size() - 1)); @@ -156,74 +155,51 @@ hailo_status FwConfigJsonSerializer::deserialize_entry(ordered_json &config_json entry_definition["size"].get(); if (deserialize_as == "str") { - auto str_val = deserialize_str(entry_value, size); - CHECK_EXPECTED_AS_STATUS(str_val); - config_json[category_name][entry_name] = str_val.value(); + TRY(config_json[category_name][entry_name], deserialize_str(entry_value, size)); } else if (deserialize_as == "bool") { - auto bool_val = deserialize_bool(entry_value, size); - CHECK_EXPECTED_AS_STATUS(bool_val); - config_json[category_name][entry_name] = bool_val.value(); + TRY(config_json[category_name][entry_name], deserialize_bool(entry_value, size)); } else if (deserialize_as == "int") { - auto int_val = deserialize_int(entry_value, size); - CHECK_EXPECTED_AS_STATUS(int_val); - config_json[category_name][entry_name] = int_val.value(); + TRY(config_json[category_name][entry_name], deserialize_int(entry_value, size)); } else if (deserialize_as == "i2c_speed") { - auto i2c_speed_val = deserialize_i2c_speed(entry_value, size); - CHECK_EXPECTED_AS_STATUS(i2c_speed_val); - config_json[category_name][entry_name]["value"] = i2c_speed_val.value(); + TRY(config_json[category_name][entry_name]["value"], deserialize_i2c_speed(entry_value, size)); } else if (deserialize_as == "supported_aspm_states") { - auto supported_aspm_states_val = deserialize_supported_aspm_states(entry_value, size); - CHECK_EXPECTED_AS_STATUS(supported_aspm_states_val); - config_json[category_name][entry_name]["value"] = supported_aspm_states_val.value(); + TRY(config_json[category_name][entry_name]["value"], deserialize_supported_aspm_states(entry_value, size)); } else if (deserialize_as == "supported_aspm_l1_substates") { - auto supported_aspm_l1_substates_val = deserialize_supported_aspm_l1_substates(entry_value, size); - CHECK_EXPECTED_AS_STATUS(supported_aspm_l1_substates_val); - config_json[category_name][entry_name]["value"] = supported_aspm_l1_substates_val.value(); + TRY(config_json[category_name][entry_name]["value"], + deserialize_supported_aspm_l1_substates(entry_value, size)); } else if (deserialize_as == "ipv4") { - auto ipv4_val = deserialize_ipv4(entry_value, size); - CHECK_EXPECTED_AS_STATUS(ipv4_val); - config_json[category_name][entry_name]["value"] = ipv4_val.value(); + TRY(config_json[category_name][entry_name]["value"], deserialize_ipv4(entry_value, size)); } else if (deserialize_as == "mac_address") { - auto mac_address_val = deserialize_mac_address(entry_value, size); - CHECK_EXPECTED_AS_STATUS(mac_address_val); - config_json[category_name][entry_name]["value"] = mac_address_val.value(); + TRY(config_json[category_name][entry_name]["value"], deserialize_mac_address(entry_value, size)); } else if (deserialize_as == "clock_frequency") { - auto clock_frequency_val = deserialize_clock_frequency(entry_value, size); - CHECK_EXPECTED_AS_STATUS(clock_frequency_val); - config_json[category_name][entry_name]["value"] = clock_frequency_val.value(); + TRY(config_json[category_name][entry_name]["value"], + deserialize_clock_frequency(entry_value, size)); } else if (deserialize_as == "logger_level") { - auto logger_level = deserialize_logger_level(entry_value, size); - CHECK_EXPECTED_AS_STATUS(logger_level); - config_json[category_name][entry_name]["value"] = logger_level.value(); + TRY(config_json[category_name][entry_name]["value"], deserialize_logger_level(entry_value, size)); } else if (deserialize_as == "watchdog_mode") { - auto watchdog_mode_val = deserialize_watchdog_mode(entry_value, size); - CHECK_EXPECTED_AS_STATUS(watchdog_mode_val); - config_json[category_name][entry_name]["value"] = watchdog_mode_val.value(); + TRY(config_json[category_name][entry_name]["value"], deserialize_watchdog_mode(entry_value, size)); } else if (deserialize_as == "overcurrent_parameters_source") { - auto overcurrent_parameters_source_val = deserialize_overcurrent_parameters_source(entry_value, size); - CHECK_EXPECTED_AS_STATUS(overcurrent_parameters_source_val); - config_json[category_name][entry_name]["value"] = overcurrent_parameters_source_val.value(); + TRY(config_json[category_name][entry_name]["value"], + deserialize_overcurrent_parameters_source(entry_value, size)); } else if (deserialize_as == "temperature_parameters_source") { - auto temperature_parameters_source_val = deserialize_temperature_parameters_source(entry_value, size); - CHECK_EXPECTED_AS_STATUS(temperature_parameters_source_val); - config_json[category_name][entry_name]["value"] = temperature_parameters_source_val.value(); + TRY(config_json[category_name][entry_name]["value"], + deserialize_temperature_parameters_source(entry_value, size)); } else if (deserialize_as == "conversion_time") { - auto conversion_time_val = deserialize_conversion_time(entry_value, size); - CHECK_EXPECTED_AS_STATUS(conversion_time_val); - config_json[category_name][entry_name]["value"] = conversion_time_val.value(); + TRY(config_json[category_name][entry_name]["value"], + deserialize_conversion_time(entry_value, size)); } else { LOGGER__ERROR("Failed deserializing entry. Serialization format {} not found", deserialize_as); @@ -240,9 +216,8 @@ Expected FwConfigJsonSerializer::deserialize_str(uint8_t *entry_value, uin Expected FwConfigJsonSerializer::deserialize_bool(uint8_t *entry_value, uint32_t size) { - auto bool_val = get_int_value(entry_value, size); - CHECK_EXPECTED(bool_val); - json bool_str = bool_val.value() ? true : false; + TRY(const auto bool_val, get_int_value(entry_value, size)); + json bool_str = bool_val ? true : false; return bool_str; } @@ -273,10 +248,9 @@ Expected FwConfigJsonSerializer::deserialize_mac_address(uint8_t *entry_va Expected FwConfigJsonSerializer::deserialize_supported_aspm_states(uint8_t *entry_value, uint32_t size) { - auto aspm_state = get_int_value(entry_value, size); - CHECK_EXPECTED(aspm_state); + TRY(const auto aspm_state, get_int_value(entry_value, size)); - switch (static_cast(aspm_state.value())) { + switch (static_cast(aspm_state)) { case ASPM_DISABLED: return json("ASPM DISABLED"); case ASPM_L1_ONLY: @@ -291,10 +265,9 @@ Expected FwConfigJsonSerializer::deserialize_supported_aspm_states(uint8_t Expected FwConfigJsonSerializer::deserialize_supported_aspm_l1_substates(uint8_t *entry_value, uint32_t size) { - auto aspm_l1_substate = get_int_value(entry_value, size); - CHECK_EXPECTED(aspm_l1_substate); + TRY(const auto aspm_l1_substate, get_int_value(entry_value, size)); - switch (static_cast(aspm_l1_substate.value())) { + switch (static_cast(aspm_l1_substate)) { case ASPM_L1_SUBSTATES_DISABLED: return json("ASPM L1 SUBSTATES DISABLED"); case ASPM_L1_SUBSTATES_L11_ONLY: @@ -309,10 +282,9 @@ Expected FwConfigJsonSerializer::deserialize_supported_aspm_l1_substates(u Expected FwConfigJsonSerializer::deserialize_clock_frequency(uint8_t *entry_value, uint32_t size) { - auto clock_frequency = get_int_value(entry_value, size); - CHECK_EXPECTED(clock_frequency); + TRY(const auto clock_frequency, get_int_value(entry_value, size)); - switch (clock_frequency.value()) { + switch (clock_frequency) { case SOC__NN_CLOCK_400MHz: return json("400MHZ"); case SOC__NN_CLOCK_375MHz: @@ -341,10 +313,9 @@ Expected FwConfigJsonSerializer::deserialize_clock_frequency(uint8_t *entr Expected FwConfigJsonSerializer::deserialize_watchdog_mode(uint8_t *entry_value, uint32_t size) { - auto watchdog_mode = get_int_value(entry_value, size); - CHECK_EXPECTED(watchdog_mode); + TRY(const auto watchdog_mode, get_int_value(entry_value, size)); - switch (static_cast(watchdog_mode.value())) { + switch (static_cast(watchdog_mode)) { case WD_SERVICE_MODE_HW_SW: return json("WD MODE HW SW"); case WD_SERVICE_MODE_HW_ONLY: @@ -357,10 +328,9 @@ Expected FwConfigJsonSerializer::deserialize_watchdog_mode(uint8_t *entry_ Expected FwConfigJsonSerializer::deserialize_i2c_speed(uint8_t *entry_value, uint32_t size) { - auto i2c_speed = get_int_value(entry_value, size); - CHECK_EXPECTED(i2c_speed); + TRY(const auto i2c_speed, get_int_value(entry_value, size)); - switch (static_cast(i2c_speed.value())) { + switch (static_cast(i2c_speed)) { case I2C_SPEED_STANDARD: return json("I2C SPEED STANDARD"); case I2C_SPEED_FAST: @@ -373,10 +343,9 @@ Expected FwConfigJsonSerializer::deserialize_i2c_speed(uint8_t *entry_valu Expected FwConfigJsonSerializer::deserialize_logger_level(uint8_t *entry_value, uint32_t size) { - auto logger_level = get_int_value(entry_value, size); - CHECK_EXPECTED(logger_level); + TRY(const auto logger_level, get_int_value(entry_value, size)); - switch (static_cast(logger_level.value())) { + switch (static_cast(logger_level)) { case FW_LOGGER_LEVEL_TRACE: return json("TRACE"); case FW_LOGGER_LEVEL_DEBUG: @@ -397,10 +366,9 @@ Expected FwConfigJsonSerializer::deserialize_logger_level(uint8_t *entry_v Expected FwConfigJsonSerializer::deserialize_overcurrent_parameters_source(uint8_t *entry_value, uint32_t size) { - auto overcurrent_parameters_source = get_int_value(entry_value, size); - CHECK_EXPECTED(overcurrent_parameters_source); + TRY(const auto overcurrent_parameters_source, get_int_value(entry_value, size)); - switch (static_cast(overcurrent_parameters_source.value())) { + switch (static_cast(overcurrent_parameters_source)) { case OVERCURRENT_PARAMETERS_SOURCE_FW_VALUES: return json("FW VALUES"); case OVERCURRENT_PARAMETERS_SOURCE_USER_CONFIG_VALUES: @@ -417,10 +385,9 @@ Expected FwConfigJsonSerializer::deserialize_overcurrent_parameters_source Expected FwConfigJsonSerializer::deserialize_temperature_parameters_source(uint8_t *entry_value, uint32_t size) { - auto temperature_parameters_source = get_int_value(entry_value, size); - CHECK_EXPECTED(temperature_parameters_source); + TRY(const auto temperature_parameters_source, get_int_value(entry_value, size)); - switch (static_cast(temperature_parameters_source.value())) { + switch (static_cast(temperature_parameters_source)) { case TEMPERATURE_PROTECTION_PARAMETERS_SOURCE_FW_VALUES: return json("FW VALUES"); case TEMPERATURE_PROTECTION_PARAMETERS_SOURCE_USER_CONFIG_VALUES: @@ -433,9 +400,8 @@ Expected FwConfigJsonSerializer::deserialize_temperature_parameters_source Expected FwConfigJsonSerializer::deserialize_conversion_time(uint8_t *entry_value, uint32_t size) { - auto conversion_time = get_int_value(entry_value, size); - CHECK_EXPECTED(conversion_time); - auto conversion_time_value = static_cast(conversion_time.value()); + TRY(const auto conversion_time, get_int_value(entry_value, size)); + auto conversion_time_value = static_cast(conversion_time); if (conversion_time_value == OVERCURRENT_CONVERSION_PERIOD_140US || conversion_time_value == OVERCURRENT_CONVERSION_PERIOD_204US || @@ -459,21 +425,18 @@ Expected FwConfigJsonSerializer::deserialize_int(uint8_t *entry_value, uin switch (size) { case sizeof(uint8_t): { - auto uint8_val = get_int_value(entry_value, size); - CHECK_EXPECTED(uint8_val); - return json(uint8_val.value()); + TRY(const auto uint8_val, get_int_value(entry_value, size)); + return json(uint8_val); } case sizeof(uint16_t): { - auto uint16_val = get_int_value(entry_value, size); - CHECK_EXPECTED(uint16_val); - return json(uint16_val.value()); + TRY(const auto uint16_val, get_int_value(entry_value, size)); + return json(uint16_val); } case sizeof(uint32_t): { - auto uint32_val = get_int_value(entry_value, size); - CHECK_EXPECTED(uint32_val); - return json(uint32_val.value()); + TRY(const auto uint32_val, get_int_value(entry_value, size)); + return json(uint32_val); } default: LOGGER__ERROR("Failed deserializing int value"); @@ -487,20 +450,17 @@ Expected FwConfigJsonSerializer::serialize_config(USER_CONFIG_header_t size_t data_size = sizeof(USER_CONFIG_header_t); try { - auto config_json = FwConfigJsonSerializer::read_json_file(file_path); - CHECK_EXPECTED(config_json); + TRY_V(const auto config_json, FwConfigJsonSerializer::read_json_file(file_path)); + TRY(auto definitions, FwConfigJsonSerializer::get_serialize_map()); - auto definitions = FwConfigJsonSerializer::get_serialize_map(); - CHECK_EXPECTED(definitions); - - user_config_header.version = definitions.value()["version"]["value"].get(); + user_config_header.version = definitions["version"]["value"].get(); user_config_header.magic = USER_CONFIG_MAGIC; user_config_header.entry_count = 0; uintptr_t current_entry_offset = (uintptr_t)(&(user_config_header.entries)); - for (auto &config_category : config_json->items()) { + for (auto &config_category : config_json.items()) { for (auto &config_entry : config_category.value().items()) { - ordered_json entry_definition = definitions.value()[config_category.key()][config_entry.key()]; + ordered_json entry_definition = definitions[config_category.key()][config_entry.key()]; USER_CONFIG_ENTRY_t *curr_entry = (USER_CONFIG_ENTRY_t *)current_entry_offset; curr_entry->entry_size = entry_definition.contains("length") ? (entry_definition["length"].get() * entry_definition["size"].get()) : @@ -603,6 +563,7 @@ hailo_status FwConfigJsonSerializer::serialize_str(USER_CONFIG_ENTRY_t &entry, c CHECK(entry.entry_size >= str.length(), HAILO_INVALID_ARGUMENT, "Failed serializing string value {}. String length must be equal or shorter than {}", str, entry.entry_size); + memset(&(entry.value), 0, entry.entry_size); memcpy(&(entry.value), str.c_str(), str.length()); return HAILO_SUCCESS; diff --git a/hailort/hailortcli/fw_control_command.cpp b/hailort/hailortcli/fw_control_command.cpp index 61dcc32..77e0e73 100644 --- a/hailort/hailortcli/fw_control_command.cpp +++ b/hailort/hailortcli/fw_control_command.cpp @@ -73,9 +73,7 @@ static bool extended_device_information_is_array_not_empty(uint8_t *array_for_pr static hailo_status print_extended_device_information(Device &device) { - auto extended_info_expected = device.get_extended_device_information(); - CHECK_EXPECTED_AS_STATUS(extended_info_expected, "Failed identify"); - auto device_info = extended_info_expected.release(); + TRY(auto device_info, device.get_extended_device_information()); // Print Board Extended information std::cout << "Boot source: " << extended_device_information_boot_string(device_info.boot_source) << std::endl; @@ -144,6 +142,8 @@ static std::string identity_arch_string(const hailo_device_identity_t &identity) return "PLUTO"; case HAILO_ARCH_HAILO15M: return "HAILO15M"; + case HAILO_ARCH_HAILO10H: + return "HAILO10H"; default: return "Unknown"; } @@ -167,9 +167,7 @@ FwControlIdentifyCommand::FwControlIdentifyCommand(CLI::App &parent_app) : hailo_status FwControlIdentifyCommand::execute_on_device(Device &device) { - auto identity_expected = device.identify(); - CHECK_EXPECTED_AS_STATUS(identity_expected, "Failed identify"); - auto identity = identity_expected.release(); + TRY(const auto identity, device.identify()); // Print board information std::cout << "Identifying board" << std::endl; diff --git a/hailort/hailortcli/fw_logger_command.cpp b/hailort/hailortcli/fw_logger_command.cpp index e6e3614..2d23ff8 100644 --- a/hailort/hailortcli/fw_logger_command.cpp +++ b/hailort/hailortcli/fw_logger_command.cpp @@ -14,67 +14,81 @@ FwLoggerCommand::FwLoggerCommand(CLI::App &parent_app) : DeviceCommand(parent_app.add_subcommand("fw-logger", "Download fw logs to a file")), - m_should_overwrite(false) + m_should_overwrite(false), + m_stdout(false), + m_continuos(false) { m_app->add_option("output_file", m_output_file, "File path to write binary firmware log into") ->required(); m_app->add_flag("--overwrite", m_should_overwrite, "Should overwrite the file or not"); + m_app->add_flag("--stdout", m_stdout, "Write the output to stdout instead of a file"); + m_app->add_flag("--continuos", m_continuos, "Write to file/stdout, until the process is killed"); } -hailo_status write_logs_to_file(Device &device, std::ofstream &ofs, hailo_cpu_id_t cpu_id){ +hailo_status FwLoggerCommand::write_logs(Device &device, std::ostream *os, hailo_cpu_id_t cpu_id) +{ auto still_has_logs = true; static const auto buffer_size = AMOUNT_OF_BYTES_TO_READ; - - auto expected_buffer = Buffer::create(buffer_size); - CHECK_EXPECTED_AS_STATUS(expected_buffer); - Buffer buffer = expected_buffer.release(); - while(still_has_logs) { - MemoryView response_view(buffer); - auto response_size_expected = device.read_log(response_view, cpu_id); - CHECK_EXPECTED_AS_STATUS(response_size_expected); + TRY(auto buffer, Buffer::create(buffer_size)); - auto response_size = response_size_expected.release(); + while (still_has_logs || m_continuos) { + MemoryView response_view(buffer); + TRY(const auto response_size, device.read_log(response_view, cpu_id)); if (response_size == 0) { still_has_logs = false; - } - else { - ofs.write((char *)buffer.data(), response_size); - CHECK(ofs.good(), HAILO_FILE_OPERATION_FAILURE, + } else { + os->write((char *)buffer.data(), response_size); + CHECK(os->good(), HAILO_FILE_OPERATION_FAILURE, "Failed writing firmware logger to output file, with errno: {}", errno); + os->flush(); } } return HAILO_SUCCESS; } +void FwLoggerCommand::pre_execute() +{ + if (m_stdout) { + // We want only the binary data from the logger to be written to stdout + DeviceCommand::m_show_stdout = false; + } +} + hailo_status FwLoggerCommand::execute_on_device(Device &device) { auto status = validate_specific_device_is_given(); CHECK_SUCCESS(status, "'fw-logger' command should get a specific device-id"); - - auto ofs_flags = std::ios::out | std::ios::binary; - if (!m_should_overwrite){ - ofs_flags |= std::ios::app; + // Initialization dependency + std::ofstream ofs; + std::ostream *os = nullptr; + if (m_stdout) { + os = &std::cout; + } else { + auto ofs_flags = std::ios::out | std::ios::binary; + if (!m_should_overwrite){ + ofs_flags |= std::ios::app; + } + ofs.open(m_output_file, ofs_flags); + CHECK(ofs.good(), HAILO_OPEN_FILE_FAILURE, "Failed opening file: {}, with errno: {}", m_output_file, errno); + os = &ofs; } - std::ofstream ofs(m_output_file, ofs_flags); - CHECK(ofs.good(), HAILO_OPEN_FILE_FAILURE, "Failed opening file: {}, with errno: {}", m_output_file, errno); - if (Device::Type::ETH == device.get_type()) { LOGGER__ERROR("Read FW log is not supported over Eth device"); return HAILO_INVALID_OPERATION; } - + if (Device::Type::INTEGRATED != device.get_type()) { - status = write_logs_to_file(device, ofs, HAILO_CPU_ID_0); + status = write_logs(device, os, HAILO_CPU_ID_0); if (status != HAILO_SUCCESS){ return status; } } - status = write_logs_to_file(device, ofs, HAILO_CPU_ID_1); + status = write_logs(device, os, HAILO_CPU_ID_1); if (status != HAILO_SUCCESS){ return status; } diff --git a/hailort/hailortcli/fw_logger_command.hpp b/hailort/hailortcli/fw_logger_command.hpp index 8ed82e6..bb5568a 100644 --- a/hailort/hailortcli/fw_logger_command.hpp +++ b/hailort/hailortcli/fw_logger_command.hpp @@ -24,11 +24,16 @@ public: explicit FwLoggerCommand(CLI::App &parent_app); protected: + virtual void pre_execute() override; virtual hailo_status execute_on_device(Device &device) override; private: std::string m_output_file; bool m_should_overwrite; + bool m_stdout; + bool m_continuos; + + hailo_status write_logs(Device &device, std::ostream *os, hailo_cpu_id_t cpu_id); }; #endif /* _HAILO_FW_LOGGER_COMMAND_COMMAND_HPP_ */ diff --git a/hailort/hailortcli/hailortcli.cpp b/hailort/hailortcli/hailortcli.cpp index ed246d7..a031807 100644 --- a/hailort/hailortcli/hailortcli.cpp +++ b/hailort/hailortcli/hailortcli.cpp @@ -61,13 +61,10 @@ Expected>> create_devices(const hailo_device { std::vector> res; - auto device_ids = get_device_ids(device_params); - CHECK_EXPECTED(device_ids); - - for (auto device_id : device_ids.value()) { - auto device = Device::create(device_id); - CHECK_EXPECTED(device); - res.emplace_back(device.release()); + TRY(const auto device_ids, get_device_ids(device_params)); + for (auto device_id : device_ids) { + TRY(auto device, Device::create(device_id)); + res.emplace_back(std::move(device)); } return res; diff --git a/hailort/hailortcli/measure_nnc_performance_command.cpp b/hailort/hailortcli/measure_nnc_performance_command.cpp index eefed1a..da352bc 100644 --- a/hailort/hailortcli/measure_nnc_performance_command.cpp +++ b/hailort/hailortcli/measure_nnc_performance_command.cpp @@ -67,49 +67,44 @@ Expected> get_configure_params(con hailo_status HwInferEstimatorCommand::execute() { - auto devices = create_devices(m_params.vdevice_params.device_params); - CHECK_EXPECTED_AS_STATUS(devices, "Failed creating device"); + TRY(auto devices, create_devices(m_params.vdevice_params.device_params), "Failed creating device"); /* This function supports controls for multiple devices. We validate there is only 1 device generated as we are on a single device flow */ - CHECK(1 == devices->size(), HAILO_INTERNAL_FAILURE, "Hw infer command support only one physical device"); - auto &device = devices.value()[0]; + CHECK(1 == devices.size(), HAILO_INTERNAL_FAILURE, "Hw infer command support only one physical device"); + auto &device = devices[0]; - auto hef = Hef::create(m_params.hef_path.c_str()); - CHECK_EXPECTED_AS_STATUS(hef, "Failed reading hef file {}", m_params.hef_path); + TRY(auto hef, + Hef::create(m_params.hef_path.c_str()), "Failed reading hef file {}", m_params.hef_path); - auto interface = device->get_default_streams_interface(); - CHECK_EXPECTED_AS_STATUS(interface, "Failed to get default streams interface"); + TRY(const auto interface, device->get_default_streams_interface(), "Failed to get default streams interface"); - auto configure_params = get_configure_params(m_params, hef.value(), interface.value()); - CHECK_EXPECTED_AS_STATUS(configure_params); + TRY(auto configure_params, get_configure_params(m_params, hef, interface)); /* Use Env var to configure all desc list with max depth */ setenv("HAILO_CONFIGURE_FOR_HW_INFER","Y",1); - auto network_group_list = device->configure(hef.value(), configure_params.value()); - CHECK_EXPECTED_AS_STATUS(network_group_list, "Failed configure device from hef"); + TRY(auto network_group_list, + device->configure(hef, configure_params), "Failed configure device from hef"); unsetenv("HAILO_CONFIGURE_FOR_HW_INFER"); - CHECK(1 == network_group_list->size(), HAILO_INVALID_OPERATION, + CHECK(1 == network_group_list.size(), HAILO_INVALID_OPERATION, "HW Inference is not supported on HEFs with multiple network groups"); - auto network_group_ptr = network_group_list.value()[0]; + auto network_group_ptr = network_group_list[0]; std::cout << "Starting HW infer Estimator..." << std::endl; - - auto results = network_group_ptr->run_hw_infer_estimator(); - CHECK_EXPECTED_AS_STATUS(results); + TRY(const auto results, network_group_ptr->run_hw_infer_estimator()); std::cout << std::endl; std::cout << "======================" << std::endl; std::cout << " Summary" << std::endl; std::cout << "======================" << std::endl; - std::cout << "Batch count: " << results->batch_count << std::endl; - std::cout << "Total transfer size [KB]: " << (results->total_transfer_size / BYTES_TO_KILOBYTES) << std::endl; - std::cout << "Total frames passed: " << results->total_frames_passed << std::endl; - std::cout << "Total time [s]: " << results->time_sec << std::endl; - std::cout << "Total FPS [1/s]: " << results->fps << std::endl; - std::cout << "BW [Gbps]: " << results->BW_Gbps << std::endl; + std::cout << "Batch count: " << results.batch_count << std::endl; + std::cout << "Total transfer size [KB]: " << (results.total_transfer_size / BYTES_TO_KILOBYTES) << std::endl; + std::cout << "Total frames passed: " << results.total_frames_passed << std::endl; + std::cout << "Total time [s]: " << results.time_sec << std::endl; + std::cout << "Total FPS [1/s]: " << results.fps << std::endl; + std::cout << "BW [Gbps]: " << results.BW_Gbps << std::endl; std::cout << "======================" << std::endl; std::cout << " End of report" << std::endl; diff --git a/hailort/hailortcli/mon_command.cpp b/hailort/hailortcli/mon_command.cpp index cb75fa6..7efdb4e 100644 --- a/hailort/hailortcli/mon_command.cpp +++ b/hailort/hailortcli/mon_command.cpp @@ -230,25 +230,20 @@ hailo_status MonCommand::run_monitor() signal(SIGINT, signit_handler); std::chrono::milliseconds time_interval = DEFAULT_SCHEDULER_MON_INTERVAL + EPSILON_TIME; - auto terminal_line_width_expected = get_terminal_line_width(); - CHECK_EXPECTED_AS_STATUS(terminal_line_width_expected); - auto terminal_line_width = terminal_line_width_expected.release(); + TRY(const auto terminal_line_width, get_terminal_line_width()); AlternativeTerminal alt_terminal; while (keep_running) { bool print_warning_msg = true; // Will change to false only if mon directory is valid and there are updated files in it. - - auto mon_dir_valid = Filesystem::is_directory(SCHEDULER_MON_TMP_DIR); - CHECK_EXPECTED_AS_STATUS(mon_dir_valid); + TRY(const auto mon_dir_valid, Filesystem::is_directory(SCHEDULER_MON_TMP_DIR)); std::vector mon_messages; - if (mon_dir_valid.value()) { - auto scheduler_mon_files = Filesystem::get_latest_files_in_dir_flat(SCHEDULER_MON_TMP_DIR, time_interval); - CHECK_EXPECTED_AS_STATUS(scheduler_mon_files); - print_warning_msg = scheduler_mon_files->empty(); + if (mon_dir_valid) { + TRY(auto scheduler_mon_files, Filesystem::get_latest_files_in_dir_flat(SCHEDULER_MON_TMP_DIR, time_interval)); + print_warning_msg = scheduler_mon_files.empty(); - mon_messages.reserve(scheduler_mon_files->size()); - for (const auto &mon_file : scheduler_mon_files.value()) { + mon_messages.reserve(scheduler_mon_files.size()); + for (const auto &mon_file : scheduler_mon_files) { auto file = LockedFile::create(mon_file, "r"); if (HAILO_SUCCESS != file.status()) { LOGGER__ERROR("Failed to open and lock file {}, with status: {}", mon_file, file.status()); diff --git a/hailort/hailortcli/parse_hef_command.cpp b/hailort/hailortcli/parse_hef_command.cpp index 51f830e..6840b18 100644 --- a/hailort/hailortcli/parse_hef_command.cpp +++ b/hailort/hailortcli/parse_hef_command.cpp @@ -24,10 +24,8 @@ ParseHefCommand::ParseHefCommand(CLI::App &parent_app) : hailo_status ParseHefCommand::execute() { - auto is_dir = Filesystem::is_directory(m_hef_path.c_str()); - CHECK_EXPECTED_AS_STATUS(is_dir, "Failed checking if path is directory"); - - if (is_dir.value()){ + TRY(const auto is_dir, Filesystem::is_directory(m_hef_path.c_str()), "Failed checking if path is directory"); + if (is_dir) { return ParseHefCommand::parse_hefs_infos_dir(m_hef_path, m_parse_streams, m_parse_vstreams); } else { return ParseHefCommand::parse_hefs_info(m_hef_path, m_parse_streams, m_parse_vstreams); @@ -36,13 +34,9 @@ hailo_status ParseHefCommand::execute() hailo_status ParseHefCommand::parse_hefs_info(const std::string &hef_path, bool stream_infos, bool vstream_infos) { - auto hef_exp = Hef::create(hef_path); - CHECK_EXPECTED_AS_STATUS(hef_exp, "Failed to parse HEF"); - auto hef = hef_exp.release(); - - auto hef_info = hef.get_description(stream_infos, vstream_infos); - CHECK_EXPECTED_AS_STATUS(hef_info, "Failed to parse HEF"); - std::cout << hef_info.release(); + TRY(const auto hef, Hef::create(hef_path)); + TRY(const auto hef_info, hef.get_description(stream_infos, vstream_infos)); + std::cout << hef_info; return HAILO_SUCCESS; } @@ -50,10 +44,9 @@ hailo_status ParseHefCommand::parse_hefs_infos_dir(const std::string &hef_path, { bool contains_hef = false; std::string hef_dir = hef_path; - const auto files = Filesystem::get_files_in_dir_flat(hef_dir); - CHECK_EXPECTED_AS_STATUS(files); + TRY(const auto files, Filesystem::get_files_in_dir_flat(hef_dir)); - for (const auto &full_path : files.value()) { + for (const auto &full_path : files) { if (Filesystem::has_suffix(full_path, ".hef")) { contains_hef = true; std::cout << std::string(80, '*') << std::endl << "Parsing " << full_path << ":"<< std::endl; diff --git a/hailort/hailortcli/run2/io_wrappers.hpp b/hailort/hailortcli/run2/io_wrappers.hpp index f3337ab..3c19d32 100644 --- a/hailort/hailortcli/run2/io_wrappers.hpp +++ b/hailort/hailortcli/run2/io_wrappers.hpp @@ -153,28 +153,27 @@ private: { const uint8_t const_byte = 0xAB; - auto constant_buffer = Buffer::create_shared(frame_size, const_byte, BufferStorageParams::create_dma()); - CHECK_EXPECTED(constant_buffer); + TRY(auto constant_buffer, + Buffer::create_shared(frame_size, const_byte, BufferStorageParams::create_dma())); - return std::vector{constant_buffer.release()}; + return std::vector{ constant_buffer }; } static Expected> create_dataset_from_input_file(const std::string &file_path, size_t frame_size) { - auto buffer = read_binary_file(file_path); - CHECK_EXPECTED(buffer); - CHECK_AS_EXPECTED(0 == (buffer->size() % frame_size), HAILO_INVALID_ARGUMENT, + TRY(auto buffer, read_binary_file(file_path)); + CHECK_AS_EXPECTED(0 == (buffer.size() % frame_size), HAILO_INVALID_ARGUMENT, "Input file ({}) size {} must be a multiple of the frame size {}", - file_path, buffer->size(), frame_size); + file_path, buffer.size(), frame_size); std::vector dataset; - const size_t frames_count = buffer->size() / frame_size; + const size_t frames_count = buffer.size() / frame_size; dataset.reserve(frames_count); for (size_t i = 0; i < frames_count; i++) { const auto offset = frame_size * i; - auto frame_buffer = Buffer::create_shared(buffer->data() + offset, frame_size, BufferStorageParams::create_dma()); - CHECK_EXPECTED(frame_buffer); - dataset.emplace_back(frame_buffer.release()); + TRY(auto frame_buffer, + Buffer::create_shared(buffer.data() + offset, frame_size, BufferStorageParams::create_dma())); + dataset.emplace_back(frame_buffer); } return dataset; @@ -183,9 +182,9 @@ private: static Expected> dma_map_dataset(const std::vector &dataset, VDevice &vdevice) { std::vector dataset_mapped_buffers; for (const auto &buffer : dataset) { - auto mapped_buffer = DmaMappedBuffer::create(vdevice, buffer->data(), buffer->size(), HAILO_DMA_BUFFER_DIRECTION_H2D); - CHECK_EXPECTED(mapped_buffer); - dataset_mapped_buffers.emplace_back(mapped_buffer.release()); + TRY(auto mapped_buffer, + DmaMappedBuffer::create(vdevice, buffer->data(), buffer->size(), HAILO_DMA_BUFFER_DIRECTION_H2D)); + dataset_mapped_buffers.emplace_back(std::move(mapped_buffer)); } return dataset_mapped_buffers; } diff --git a/hailort/hailortcli/run2/live_stats.cpp b/hailort/hailortcli/run2/live_stats.cpp index b68799f..30715af 100644 --- a/hailort/hailortcli/run2/live_stats.cpp +++ b/hailort/hailortcli/run2/live_stats.cpp @@ -132,9 +132,9 @@ Expected> LiveStats::get_last_measured_fps_per_network_group CHECK_AS_EXPECTED(contains(m_tracks, NETWORK_STATS_LEVEL), HAILO_NOT_AVAILABLE); for (size_t network_stats_track_index = 0; network_stats_track_index < m_tracks[NETWORK_STATS_LEVEL].size(); network_stats_track_index++) { - auto expected_fps = m_tracks[NETWORK_STATS_LEVEL][network_stats_track_index]->get_last_measured_fps(); - CHECK_EXPECTED(expected_fps); - last_measured_fpss.emplace_back(expected_fps.release()); + TRY(auto fps, + m_tracks[NETWORK_STATS_LEVEL][network_stats_track_index]->get_last_measured_fps()); + last_measured_fpss.emplace_back(fps); } return last_measured_fpss; diff --git a/hailort/hailortcli/run2/measurement_live_track.cpp b/hailort/hailortcli/run2/measurement_live_track.cpp index f098cfe..9278e39 100644 --- a/hailort/hailortcli/run2/measurement_live_track.cpp +++ b/hailort/hailortcli/run2/measurement_live_track.cpp @@ -24,23 +24,19 @@ Expected> MeasurementLiveTrack::create_sha { std::shared_ptr power_measurement = nullptr; if (measure_power) { - auto power_measurement_exp = PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__POWER); - CHECK_EXPECTED(power_measurement_exp); - power_measurement = power_measurement_exp.release(); + TRY(power_measurement, + PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__POWER)); } std::shared_ptr current_measurement = nullptr; if (measure_current) { - auto current_measurement_exp = PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__CURRENT); - CHECK_EXPECTED(current_measurement_exp); - current_measurement = current_measurement_exp.release(); + TRY(current_measurement, + PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__CURRENT)); } std::shared_ptr temp_measurement = nullptr; if (measure_temp) { - auto temp_measurement_exp = TemperatureMeasurement::create_shared(device); - CHECK_EXPECTED(temp_measurement_exp); - temp_measurement = temp_measurement_exp.release(); + TRY(temp_measurement, TemperatureMeasurement::create_shared(device)); } auto ptr = make_shared_nothrow(power_measurement, current_measurement, temp_measurement, device.get_dev_id()); diff --git a/hailort/hailortcli/run2/network_runner.cpp b/hailort/hailortcli/run2/network_runner.cpp index 86845f7..266f26a 100644 --- a/hailort/hailortcli/run2/network_runner.cpp +++ b/hailort/hailortcli/run2/network_runner.cpp @@ -138,12 +138,8 @@ Expected NetworkRunner::get_network_group_name(const NetworkParams Expected> FullAsyncNetworkRunner::create_shared(VDevice &vdevice, NetworkParams params) { - auto infer_model = vdevice.create_infer_model(params.hef_path); - CHECK_EXPECTED(infer_model); - auto infer_model_ptr = infer_model.release(); - - auto expected_net_group_name = get_network_group_name(params, infer_model_ptr->hef()); - CHECK_EXPECTED(expected_net_group_name); + TRY(auto infer_model_ptr, vdevice.create_infer_model(params.hef_path)); + TRY(auto net_group_name, get_network_group_name(params, infer_model_ptr->hef())); /* Configure Params */ infer_model_ptr->set_batch_size(params.batch_size); @@ -163,10 +159,9 @@ Expected> FullAsyncNetworkRunner::create }); auto input_params = (input_params_it == params.vstream_params.end()) ? VStreamParams() : *input_params_it; - auto input_config = infer_model_ptr->input(input_name); - CHECK_EXPECTED(input_config); - input_config->set_format_order(input_params.params.user_buffer_format.order); - input_config->set_format_type(input_params.params.user_buffer_format.type); + TRY(auto input_config, infer_model_ptr->input(input_name)); + input_config.set_format_order(input_params.params.user_buffer_format.order); + input_config.set_format_type(input_params.params.user_buffer_format.type); } for (const auto &output_name : infer_model_ptr->get_output_names()) { auto output_params_it = std::find_if(params.vstream_params.begin(), params.vstream_params.end(), @@ -175,18 +170,16 @@ Expected> FullAsyncNetworkRunner::create }); auto output_params = (output_params_it == params.vstream_params.end()) ? VStreamParams() : *output_params_it; - auto output_config = infer_model_ptr->output(output_name); - CHECK_EXPECTED(output_config); - output_config->set_format_order(output_params.params.user_buffer_format.order); - output_config->set_format_type(output_params.params.user_buffer_format.type); + TRY(auto output_config, infer_model_ptr->output(output_name)); + output_config.set_format_order(output_params.params.user_buffer_format.order); + output_config.set_format_type(output_params.params.user_buffer_format.type); } - auto configured_model = infer_model_ptr->configure(); - CHECK_EXPECTED(configured_model); - auto configured_infer_model_ptr = make_shared_nothrow(configured_model.release()); + TRY(auto configured_model, infer_model_ptr->configure()); + auto configured_infer_model_ptr = make_shared_nothrow(std::move(configured_model)); CHECK_NOT_NULL_AS_EXPECTED(configured_infer_model_ptr, HAILO_OUT_OF_HOST_MEMORY); - auto res = make_shared_nothrow(params, expected_net_group_name.value(), vdevice, + auto res = make_shared_nothrow(params, net_group_name, vdevice, infer_model_ptr, configured_infer_model_ptr); CHECK_NOT_NULL_AS_EXPECTED(res, HAILO_OUT_OF_HOST_MEMORY); @@ -216,35 +209,27 @@ Expected> NetworkRunner::create_shared(VDevice &v std::shared_ptr net_runner_ptr = nullptr; if (InferenceMode::FULL_ASYNC == final_net_params.mode) { - auto runner_exp = FullAsyncNetworkRunner::create_shared(vdevice, final_net_params); - CHECK_EXPECTED(runner_exp); - net_runner_ptr = runner_exp.release(); + TRY(net_runner_ptr, FullAsyncNetworkRunner::create_shared(vdevice, final_net_params)); } else { - auto hef = Hef::create(final_net_params.hef_path); - CHECK_EXPECTED(hef); - - auto expected_net_group_name = get_network_group_name(final_net_params, hef.value()); - CHECK_EXPECTED(expected_net_group_name); - - auto cfg_params = vdevice.create_configure_params(hef.value(), expected_net_group_name.value()); - CHECK_EXPECTED(cfg_params); - cfg_params->batch_size = final_net_params.batch_size; + TRY(auto hef, Hef::create(final_net_params.hef_path)); + TRY(auto net_group_name, get_network_group_name(final_net_params, hef)); + TRY(auto cfg_params, vdevice.create_configure_params(hef, net_group_name)); + cfg_params.batch_size = final_net_params.batch_size; if (final_net_params.batch_size == HAILO_DEFAULT_BATCH_SIZE) { // Changing batch_size to 1 (after configuring the vdevice) - as we iterate over 'final_net_params.batch_size' in latency measurements scenarios final_net_params.batch_size = 1; } if (final_net_params.measure_hw_latency) { - cfg_params->latency |= HAILO_LATENCY_MEASURE; + cfg_params.latency |= HAILO_LATENCY_MEASURE; } if (final_net_params.is_async()) { - for (auto &stream_name_params_pair : cfg_params->stream_params_by_name) { + for (auto &stream_name_params_pair : cfg_params.stream_params_by_name) { stream_name_params_pair.second.flags = HAILO_STREAM_FLAGS_ASYNC; } } - auto cfgr_net_groups = vdevice.configure(hef.value(), {{expected_net_group_name.value(), cfg_params.value()}}); - CHECK_EXPECTED(cfgr_net_groups); - assert(1 == cfgr_net_groups->size()); - auto cfgr_net_group = cfgr_net_groups.value()[0]; + TRY(auto cfgr_net_groups, vdevice.configure(hef, {{ net_group_name, cfg_params }})); + assert(1 == cfgr_net_groups.size()); + auto cfgr_net_group = cfgr_net_groups[0]; if (HAILO_SCHEDULING_ALGORITHM_NONE != final_net_params.scheduling_algorithm) { CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_threshold(final_net_params.scheduler_threshold)); @@ -260,11 +245,10 @@ Expected> NetworkRunner::create_shared(VDevice &v for (auto &vstream_params : final_net_params.vstream_params) { vstreams_params.emplace(vstream_params.name, vstream_params.params); } - auto vstreams = create_vstreams(*cfgr_net_group, vstreams_params); - CHECK_EXPECTED(vstreams); + TRY(auto vstreams, create_vstreams(*cfgr_net_group, vstreams_params)); - auto net_runner = make_shared_nothrow(final_net_params, expected_net_group_name.value(), vdevice, - std::move(vstreams->first), std::move(vstreams->second), cfgr_net_group); + auto net_runner = make_shared_nothrow(final_net_params, net_group_name, vdevice, + std::move(vstreams.first), std::move(vstreams.second), cfgr_net_group); CHECK_NOT_NULL_AS_EXPECTED(net_runner, HAILO_OUT_OF_HOST_MEMORY); net_runner_ptr = std::static_pointer_cast(net_runner); break; @@ -279,7 +263,7 @@ Expected> NetworkRunner::create_shared(VDevice &v auto output_streams = cfgr_net_group->get_output_streams(); CHECK_AS_EXPECTED(output_streams.size() > 0, HAILO_INTERNAL_FAILURE); - auto net_runner = make_shared_nothrow(final_net_params, expected_net_group_name.value(), vdevice, + auto net_runner = make_shared_nothrow(final_net_params, net_group_name, vdevice, std::move(input_streams), std::move(output_streams), cfgr_net_group); CHECK_NOT_NULL_AS_EXPECTED(net_runner, HAILO_OUT_OF_HOST_MEMORY); net_runner_ptr = std::static_pointer_cast(net_runner); @@ -331,7 +315,7 @@ hailo_status NetworkRunner::run(EventPtr shutdown_event, LiveStats &live_stats, if (!ang_exp) { activation_barrier.terminate(); } - CHECK_EXPECTED_AS_STATUS(ang_exp); + CHECK_EXPECTED_AS_STATUS(ang_exp); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here ang = ang_exp.release(); } } @@ -350,12 +334,11 @@ hailo_status NetworkRunner::run(EventPtr shutdown_event, LiveStats &live_stats, if ((InferenceMode::RAW_ASYNC_SINGLE_THREAD == m_params.mode) || (InferenceMode::FULL_ASYNC == m_params.mode)) { return run_single_thread_async_infer(shutdown_event, net_live_track); } else { - auto threads = start_inference_threads(shutdown_event, net_live_track); - CHECK_EXPECTED_AS_STATUS(threads); + TRY(auto threads, start_inference_threads(shutdown_event, net_live_track)); CHECK_SUCCESS(shutdown_event->wait(HAILO_INFINITE_TIMEOUT)); stop(); - return wait_for_threads(threads.value()); + return wait_for_threads(threads); } } @@ -390,9 +373,8 @@ Expected, std::vector>> Netwo size_t match_count = 0; std::map input_vstreams_params; - auto input_vstreams_info = net_group.get_input_vstream_infos(); - CHECK_EXPECTED(input_vstreams_info); - for (auto &input_vstream_info : input_vstreams_info.value()) { + TRY(auto input_vstreams_info, net_group.get_input_vstream_infos()); + for (auto &input_vstream_info : input_vstreams_info) { if (params.end() != params.find(input_vstream_info.name)) { match_count++; input_vstreams_params.emplace(input_vstream_info.name, params.at(input_vstream_info.name)); @@ -402,9 +384,8 @@ Expected, std::vector>> Netwo } std::map output_vstreams_params; - auto output_vstreams_info = net_group.get_output_vstream_infos(); - CHECK_EXPECTED(output_vstreams_info); - for (auto &output_vstream_info : output_vstreams_info.value()) { + TRY(auto output_vstreams_info, net_group.get_output_vstream_infos()); + for (auto &output_vstream_info : output_vstreams_info) { if (params.end() != params.find(output_vstream_info.name)) { match_count++; output_vstreams_params.emplace(output_vstream_info.name, params.at(output_vstream_info.name)); @@ -415,13 +396,10 @@ Expected, std::vector>> Netwo CHECK(match_count == params.size(), make_unexpected(HAILO_INVALID_ARGUMENT), "One of the params has an invalid vStream name"); - auto input_vstreams = VStreamsBuilder::create_input_vstreams(net_group, input_vstreams_params); - CHECK_EXPECTED(input_vstreams); + TRY(auto input_vstreams, VStreamsBuilder::create_input_vstreams(net_group, input_vstreams_params)); + TRY(auto output_vstreams, VStreamsBuilder::create_output_vstreams(net_group, output_vstreams_params)); - auto output_vstreams = VStreamsBuilder::create_output_vstreams(net_group, output_vstreams_params); - CHECK_EXPECTED(output_vstreams); - - return {{input_vstreams.release(), output_vstreams.release()}};//TODO: move? copy elision? + return std::make_pair(std::move(input_vstreams), std::move(output_vstreams)); } const std::vector NetworkRunner::ALLOWED_INFERENCE_RETURN_VALUES{ @@ -444,24 +422,22 @@ Expected>> FullSyncNetworkRunner::start std::vector> threads; for (auto &input_vstream : m_input_vstreams) { const auto vstream_params = get_params(input_vstream.name()); - auto writer = WriterWrapper::create(input_vstream, vstream_params, m_vdevice, - m_overall_latency_meter, m_params.framerate, SYNC_API); - CHECK_EXPECTED(writer); + TRY(auto writer, WriterWrapper::create(input_vstream, vstream_params, m_vdevice, + m_overall_latency_meter, m_params.framerate, SYNC_API)); threads.emplace_back(std::make_unique>("WRITE", - [this, writer = writer.release(), shutdown_event]() mutable { + [this, writer, shutdown_event]() mutable { return run_write(writer, shutdown_event, m_latency_barrier); })); } bool first = true; //TODO: check with multiple outputs for (auto &output_vstream : m_output_vstreams) { - auto reader = ReaderWrapper::create(output_vstream, m_vdevice, - m_overall_latency_meter, first ? net_live_track : nullptr, SYNC_API); - CHECK_EXPECTED(reader); + TRY(auto reader, ReaderWrapper::create(output_vstream, m_vdevice, + m_overall_latency_meter, first ? net_live_track : nullptr, SYNC_API)); threads.emplace_back(std::make_unique>("READ", - [this, reader=reader.release(), shutdown_event]() mutable { + [this, reader, shutdown_event]() mutable { return run_read(reader, shutdown_event, m_latency_barrier); })); first = false; @@ -552,8 +528,8 @@ Expected FullAsyncNetworkRunner::create_infer_job(const Configure if (m_overall_latency_meter) { m_overall_latency_meter->add_start_sample(std::chrono::steady_clock::now().time_since_epoch()); } - auto job = m_configured_infer_model->run_async(bindings, [=, &inference_status] (const AsyncInferCompletionInfo &completion_info) { + TRY(auto job, m_configured_infer_model->run_async(bindings, [=, &inference_status] (const AsyncInferCompletionInfo &completion_info) { if (HAILO_SUCCESS != completion_info.status) { inference_status = completion_info.status; if (HAILO_STREAM_ABORT != completion_info.status) { @@ -569,9 +545,8 @@ Expected FullAsyncNetworkRunner::create_infer_job(const Configure so there's a circular dependency */ net_live_track->progress(); } - }); - CHECK_EXPECTED(job); - return job.release(); + })); + return job; } hailo_status FullAsyncNetworkRunner::run_single_thread_async_infer(EventPtr shutdown_event, @@ -590,13 +565,10 @@ hailo_status FullAsyncNetworkRunner::run_single_thread_async_infer(EventPtr shut status = m_configured_infer_model->set_scheduler_priority(m_params.scheduler_priority); CHECK_SUCCESS(status); } else { - auto guard_exp = ConfiguredInferModelActivationGuard::create(m_configured_infer_model); - CHECK_EXPECTED_AS_STATUS(guard_exp); - guard = guard_exp.release(); + TRY(guard, ConfiguredInferModelActivationGuard::create(m_configured_infer_model)); } - auto bindings = m_configured_infer_model->create_bindings(); - CHECK_EXPECTED_AS_STATUS(bindings); + TRY(auto bindings, m_configured_infer_model->create_bindings()); std::unordered_map input_buffers; // Keys are inputs names std::vector output_buffers; @@ -604,40 +576,36 @@ hailo_status FullAsyncNetworkRunner::run_single_thread_async_infer(EventPtr shut const uint8_t const_byte = 0xAB; for (const auto &name : get_input_names()) { - auto input_config = m_infer_model->input(name); - CHECK_EXPECTED_AS_STATUS(input_config); + TRY(auto input_config, m_infer_model->input(name)); auto params = get_params(name); - auto buffer = params.input_file_path.empty() ? - Buffer::create(input_config->get_frame_size(), const_byte, BufferStorageParams::create_dma()) : - read_binary_file(params.input_file_path, BufferStorageParams::create_dma()); - CHECK_EXPECTED_AS_STATUS(buffer); - CHECK(0 == (buffer->size() % input_config->get_frame_size()), HAILO_INVALID_ARGUMENT, - "Size of data for input '{}' must be a multiple of the frame size {}. Received - {}", name, input_config->get_frame_size(), buffer->size()); - input_buffers.emplace(name, buffer.release()); - - for (uint32_t i = 0; i < (input_buffers.at(name).size() % input_config->get_frame_size()); i++) { - auto mapped_buffer = DmaMappedBuffer::create(m_vdevice, input_buffers.at(name).data() + (i * input_config->get_frame_size()), - input_config->get_frame_size(), HAILO_DMA_BUFFER_DIRECTION_H2D); - CHECK_EXPECTED_AS_STATUS(mapped_buffer); - dma_mapped_buffers.emplace_back(mapped_buffer.release()); + Buffer buffer {}; + if (params.input_file_path.empty()) { + TRY(buffer, Buffer::create(input_config.get_frame_size(), const_byte, BufferStorageParams::create_dma())); + } else { + TRY(buffer, read_binary_file(params.input_file_path, BufferStorageParams::create_dma())); + } + CHECK(0 == (buffer.size() % input_config.get_frame_size()), HAILO_INVALID_ARGUMENT, + "Size of data for input '{}' must be a multiple of the frame size {}. Received - {}", name, input_config.get_frame_size(), buffer.size()); + input_buffers.emplace(name, std::move(buffer)); + + for (uint32_t i = 0; i < (input_buffers.at(name).size() % input_config.get_frame_size()); i++) { + TRY(auto mapped_buffer, DmaMappedBuffer::create(m_vdevice, input_buffers.at(name).data() + (i * input_config.get_frame_size()), + input_config.get_frame_size(), HAILO_DMA_BUFFER_DIRECTION_H2D)); + dma_mapped_buffers.emplace_back(std::move(mapped_buffer)); } } for (const auto &name : get_output_names()) { - auto output_config = m_infer_model->output(name); - CHECK_EXPECTED_AS_STATUS(output_config); - - auto buffer = Buffer::create(output_config->get_frame_size(), 0, BufferStorageParams::create_dma()); - CHECK_EXPECTED_AS_STATUS(buffer); - output_buffers.emplace_back(buffer.release()); + TRY(auto output_config, m_infer_model->output(name)); + TRY(auto buffer, Buffer::create(output_config.get_frame_size(), 0, BufferStorageParams::create_dma())); + output_buffers.emplace_back(std::move(buffer)); - auto mapped_buffer = DmaMappedBuffer::create(m_vdevice, output_buffers.back().data(), output_buffers.back().size(), - HAILO_DMA_BUFFER_DIRECTION_D2H); - CHECK_EXPECTED_AS_STATUS(mapped_buffer); - dma_mapped_buffers.emplace_back(mapped_buffer.release()); + TRY(auto mapped_buffer, DmaMappedBuffer::create(m_vdevice, output_buffers.back().data(), output_buffers.back().size(), + HAILO_DMA_BUFFER_DIRECTION_D2H)); + dma_mapped_buffers.emplace_back(std::move(mapped_buffer)); - CHECK_SUCCESS(bindings->output(name)->set_buffer(MemoryView(output_buffers.back()))); + CHECK_SUCCESS(bindings.output(name)->set_buffer(MemoryView(output_buffers.back()))); } FramerateThrottle frame_rate_throttle(m_params.framerate); @@ -648,17 +616,14 @@ hailo_status FullAsyncNetworkRunner::run_single_thread_async_infer(EventPtr shut while (HAILO_TIMEOUT == shutdown_event->wait(std::chrono::milliseconds(0)) && (HAILO_SUCCESS == inference_status)) { for (uint32_t frames_in_cycle = 0; frames_in_cycle < m_params.batch_size; frames_in_cycle++) { for (const auto &name : get_input_names()) { - auto input_config = m_infer_model->input(name); - CHECK_EXPECTED_AS_STATUS(input_config); - auto offset = (frame_id % (input_buffers.at(name).size() / input_config->get_frame_size())) * input_config->get_frame_size(); - CHECK_SUCCESS(bindings->input(name)->set_buffer(MemoryView(input_buffers.at(name).data() + offset, - input_config->get_frame_size()))); + TRY(auto input_config, m_infer_model->input(name)); + auto offset = (frame_id % (input_buffers.at(name).size() / input_config.get_frame_size())) * input_config.get_frame_size(); + CHECK_SUCCESS(bindings.input(name)->set_buffer(MemoryView(input_buffers.at(name).data() + offset, + input_config.get_frame_size()))); } frame_id++; if (HAILO_SUCCESS == m_configured_infer_model->wait_for_async_ready(DEFAULT_TRANSFER_TIMEOUT)) { - auto job_exp = create_infer_job(*bindings, net_live_track, frame_rate_throttle, inference_status); - CHECK_EXPECTED_AS_STATUS(job_exp); - last_job = job_exp.release(); + TRY(last_job, create_infer_job(bindings, net_live_track, frame_rate_throttle, inference_status)); last_job.detach(); } } @@ -689,18 +654,17 @@ Expected>> RawNetworkRunner::start_infe std::vector> threads; for (auto &input_stream : m_input_streams) { const auto stream_params = get_params(input_stream.get().name()); - auto writer = WriterWrapper::create(input_stream.get(), stream_params, m_vdevice, - m_overall_latency_meter, m_params.framerate, async_streams); - CHECK_EXPECTED(writer); + TRY(auto writer, WriterWrapper::create(input_stream.get(), stream_params, m_vdevice, + m_overall_latency_meter, m_params.framerate, async_streams)); if (async_streams) { threads.emplace_back(std::make_unique>("WRITE_ASYNC", - [this, writer = writer.release(), shutdown_event]() mutable { + [this, writer, shutdown_event]() mutable { return run_write_async(writer, shutdown_event, m_latency_barrier); })); } else { threads.emplace_back(std::make_unique>("WRITE", - [this, writer = writer.release(), shutdown_event]() mutable { + [this, writer, shutdown_event]() mutable { return run_write(writer, shutdown_event, m_latency_barrier); })); } @@ -708,18 +672,17 @@ Expected>> RawNetworkRunner::start_infe bool first = true; //TODO: check with multiple outputs for (auto &output_stream : m_output_streams) { - auto reader = ReaderWrapper::create(output_stream.get(), m_vdevice, - m_overall_latency_meter, first ? net_live_track : nullptr, async_streams); - CHECK_EXPECTED(reader); + TRY(auto reader, ReaderWrapper::create(output_stream.get(), m_vdevice, + m_overall_latency_meter, first ? net_live_track : nullptr, async_streams)); if (async_streams) { threads.emplace_back(std::make_unique>("READ_ASYNC", - [this, reader=reader.release(), shutdown_event]() mutable { + [this, reader, shutdown_event]() mutable { return run_read_async(reader, shutdown_event, m_latency_barrier); })); } else { threads.emplace_back(std::make_unique>("READ", - [this, reader=reader.release(), shutdown_event]() mutable { + [this, reader, shutdown_event]() mutable { return run_read(reader, shutdown_event, m_latency_barrier); })); } @@ -739,37 +702,29 @@ hailo_status RawNetworkRunner::run_single_thread_async_infer(EventPtr shutdown_e std::vector output_semaphores; bool is_first_output = true; for (auto &output_stream : m_output_streams) { - auto reader_wrapper = ReaderWrapper::create(output_stream.get(), m_vdevice, - m_overall_latency_meter, is_first_output ? net_live_track : nullptr, ASYNC_API); - CHECK_EXPECTED_AS_STATUS(reader_wrapper); + TRY(auto reader_wrapper, ReaderWrapper::create(output_stream.get(), m_vdevice, + m_overall_latency_meter, is_first_output ? net_live_track : nullptr, ASYNC_API)); is_first_output = false; - auto max_queue_size = reader_wrapper.value()->get().get_async_max_queue_size(); - CHECK_EXPECTED_AS_STATUS(max_queue_size); + TRY(auto max_queue_size, reader_wrapper->get().get_async_max_queue_size()); + TRY(auto semaphore, Semaphore::create_shared(static_cast(max_queue_size))); - auto semaphore = Semaphore::create_shared(static_cast(*max_queue_size)); - CHECK_EXPECTED_AS_STATUS(semaphore); - - output_semaphores.emplace_back(semaphore.release()); - reader_wrappers.emplace_back(reader_wrapper.release()); + output_semaphores.emplace_back(semaphore); + reader_wrappers.emplace_back(reader_wrapper); } // Build input wrappers std::vector> writer_wrappers; std::vector input_semaphores; for (auto &input_stream : m_input_streams) { - auto writer_wrapper = WriterWrapper::create(input_stream.get(), - get_params(input_stream.get().name()), m_vdevice, m_overall_latency_meter, m_params.framerate, ASYNC_API); - CHECK_EXPECTED_AS_STATUS(writer_wrapper); - - auto max_queue_size = writer_wrapper.value()->get().get_async_max_queue_size(); - CHECK_EXPECTED_AS_STATUS(max_queue_size); + TRY(auto writer_wrapper, WriterWrapper::create(input_stream.get(), + get_params(input_stream.get().name()), m_vdevice, m_overall_latency_meter, m_params.framerate, ASYNC_API)); - auto semaphore = Semaphore::create_shared(static_cast(*max_queue_size)); - CHECK_EXPECTED_AS_STATUS(semaphore); + TRY(auto max_queue_size, writer_wrapper->get().get_async_max_queue_size()); + TRY(auto semaphore, Semaphore::create_shared(static_cast(max_queue_size))); - input_semaphores.emplace_back(semaphore.release()); - writer_wrappers.emplace_back(writer_wrapper.release()); + input_semaphores.emplace_back(semaphore); + writer_wrappers.emplace_back(writer_wrapper); } // Build waitables list with reference to previous input/output semaphores. @@ -788,17 +743,16 @@ hailo_status RawNetworkRunner::run_single_thread_async_infer(EventPtr shutdown_e // Inference while (true) { - auto wait_index = wait_group.wait_any(HAILORTCLI_DEFAULT_TIMEOUT); - CHECK_EXPECTED_AS_STATUS(wait_index); + TRY(auto wait_index, wait_group.wait_any(HAILORTCLI_DEFAULT_TIMEOUT)); - if (*wait_index == shutdown_index) { + if (wait_index == shutdown_index) { // Stopping the network so we won't get timeout on the flush. The async operations may still be active // (until network deactivation). stop(); break; - } else if ((*wait_index >= output_index_start) && (*wait_index < input_index_start)) { + } else if ((wait_index >= output_index_start) && (wait_index < input_index_start)) { // output is ready - const size_t output_index = *wait_index - output_index_start; + const size_t output_index = wait_index - output_index_start; auto status = reader_wrappers[output_index]->read_async( [semaphore=output_semaphores[output_index]](const OutputStream::CompletionInfo &) { (void)semaphore->signal(); @@ -807,7 +761,7 @@ hailo_status RawNetworkRunner::run_single_thread_async_infer(EventPtr shutdown_e CHECK_SUCCESS(status); } else { // input is ready - const size_t input_index = *wait_index - input_index_start; + const size_t input_index = wait_index - input_index_start; auto status = writer_wrappers[input_index]->write_async( [semaphore=input_semaphores[input_index]](const InputStream::CompletionInfo &) { (void)semaphore->signal(); diff --git a/hailort/hailortcli/run2/network_runner.hpp b/hailort/hailortcli/run2/network_runner.hpp index 9601172..22d8b85 100644 --- a/hailort/hailortcli/run2/network_runner.hpp +++ b/hailort/hailortcli/run2/network_runner.hpp @@ -186,9 +186,7 @@ protected: // sync_event will be used to send one frame at a time EventPtr sync_event = nullptr; if (m_params.measure_hw_latency || m_params.measure_overall_latency) { - auto sync_event_exp = Event::create_shared(Event::State::not_signalled); - CHECK_EXPECTED_AS_STATUS(sync_event_exp); - sync_event = sync_event_exp.release(); + TRY(sync_event, Event::create_shared(Event::State::not_signalled)); } while (true) { @@ -263,9 +261,7 @@ protected: // sync_event will be used to send one frame at a time EventPtr sync_event = nullptr; if (m_params.measure_hw_latency || m_params.measure_overall_latency) { - auto sync_event_exp = Event::create_shared(Event::State::not_signalled); - CHECK_EXPECTED_AS_STATUS(sync_event_exp); - sync_event = sync_event_exp.release(); + TRY(sync_event, Event::create_shared(Event::State::not_signalled)); } while (true) { diff --git a/hailort/hailortcli/run2/run2_command.cpp b/hailort/hailortcli/run2/run2_command.cpp index 4bfdf06..6914e5a 100644 --- a/hailort/hailortcli/run2/run2_command.cpp +++ b/hailort/hailortcli/run2/run2_command.cpp @@ -670,10 +670,10 @@ std::string format_measure_fw_actions_output_path(const std::string &base_output Expected> get_single_physical_device(VDevice &vdevice) { - auto expected_physical_devices = vdevice.get_physical_devices(); - CHECK_EXPECTED(expected_physical_devices); - CHECK_AS_EXPECTED(1 == expected_physical_devices->size(), HAILO_INVALID_OPERATION, "Operation not allowed for multi-device"); - auto &res = expected_physical_devices->at(0); + TRY(auto physical_devices, vdevice.get_physical_devices()); + CHECK_AS_EXPECTED(1 == physical_devices.size(), HAILO_INVALID_OPERATION, + "Operation not allowed for multi-device"); + auto &res = physical_devices.at(0); return std::move(res); } @@ -712,16 +712,11 @@ Expected> Run2::create_vdevice() Expected>> Run2::init_and_run_net_runners(VDevice *vdevice) { std::vector> net_runners; - - auto shutdown_event_exp = Event::create_shared(Event::State::not_signalled); - CHECK_EXPECTED(shutdown_event_exp); - auto shutdown_event = shutdown_event_exp.release(); + TRY(auto shutdown_event, Event::create_shared(Event::State::not_signalled)); // create network runners for (auto &net_params : get_network_params()) { - auto expected_net_runner = NetworkRunner::create_shared(*vdevice, net_params); - CHECK_EXPECTED(expected_net_runner); - auto net_runner = expected_net_runner.release(); + TRY(auto net_runner, NetworkRunner::create_shared(*vdevice, net_params)); net_runners.emplace_back(net_runner); } @@ -743,15 +738,24 @@ Expected>> Run2::init_and_run_net_run activation_barrier.arrive_and_wait(); if (get_measure_power() || get_measure_current() || get_measure_temp()) { - auto physical_devices = vdevice->get_physical_devices(); - CHECK_EXPECTED(physical_devices); - - for (auto &device : physical_devices.value()) { - auto measurement_live_track = MeasurementLiveTrack::create_shared(device.get(), get_measure_power(), - get_measure_current(), get_measure_temp()); - CHECK_EXPECTED(measurement_live_track); - - live_stats->add(measurement_live_track.release(), 2); + TRY(auto physical_devices, vdevice->get_physical_devices()); + + for (auto &device : physical_devices) { + TRY(const auto identity, device.get().identify()); + CHECK_AS_EXPECTED(HailoRTCommon::is_power_measurement_supported(identity.device_architecture) || !(get_measure_power()), + HAILO_INVALID_OPERATION, "HW arch {} does not support power measurement. Disable the power-measure option", + HailoRTCommon::get_device_arch_str(identity.device_architecture)); + CHECK_AS_EXPECTED(HailoRTCommon::is_current_measurement_supported(identity.device_architecture) || !(get_measure_current()), + HAILO_INVALID_OPERATION, "HW arch {} does not support current measurement. Disable the current-measure option", + HailoRTCommon::get_device_arch_str(identity.device_architecture)); + CHECK_AS_EXPECTED(HailoRTCommon::is_temp_measurement_supported(identity.device_architecture) || !(get_measure_temp()), + HAILO_INVALID_OPERATION, "HW arch {} does not support temperature measurement. Disable the temp-measure option", + HailoRTCommon::get_device_arch_str(identity.device_architecture)); + + TRY(auto measurement_live_track, MeasurementLiveTrack::create_shared(device.get(), + get_measure_power(), get_measure_current(), get_measure_temp())); + + live_stats->add(measurement_live_track, 2); } } @@ -764,9 +768,7 @@ Expected>> Run2::init_and_run_net_run if (!get_output_json_path().empty()){ live_stats->dump_stats(get_output_json_path(), get_str_infer_mode(get_mode())); } - auto expected_fps_per_network = live_stats->get_last_measured_fps_per_network_group(); - CHECK_EXPECTED(expected_fps_per_network); - auto fps_per_network = expected_fps_per_network.release(); + TRY(auto fps_per_network, live_stats->get_last_measured_fps_per_network_group()); for (size_t network_runner_index = 0; network_runner_index < fps_per_network.size(); network_runner_index++) { net_runners[network_runner_index]->set_last_measured_fps(fps_per_network[network_runner_index]); } @@ -793,10 +795,7 @@ hailo_status Run2Command::execute() LOGGER__WARNING("\"hailortcli run2\" is not optimized for single model usage. It is recommended to use \"hailortcli run\" command for a single model"); } - auto expected_vdevice = app->create_vdevice(); - CHECK_EXPECTED_AS_STATUS(expected_vdevice); - auto vdevice = expected_vdevice.release(); - + TRY(auto vdevice, app->create_vdevice()); std::vector batch_sizes_to_run = { app->get_network_params()[0].batch_size }; if(app->get_measure_fw_actions() && app->get_network_params()[0].batch_size == HAILO_DEFAULT_BATCH_SIZE) { // In case measure-fw-actions is enabled and no batch size was provided - we want to run with batch sizes 1,2,4,8,16 @@ -807,12 +806,9 @@ hailo_status Run2Command::execute() ordered_json action_list_json; if (app->get_measure_fw_actions()) { - auto device = get_single_physical_device(*vdevice); - CHECK_EXPECTED_AS_STATUS(device); - - auto expected_action_list_json = DownloadActionListCommand::init_json_object(device.release(), app->get_network_params()[0].hef_path); - CHECK_EXPECTED_AS_STATUS(expected_action_list_json); - action_list_json = expected_action_list_json.release(); + TRY(auto device, get_single_physical_device(*vdevice)); + TRY(action_list_json, + DownloadActionListCommand::init_json_object(device, app->get_network_params()[0].hef_path)); runtime_data_output_path = format_measure_fw_actions_output_path( app->get_measure_fw_actions_output_path(), app->get_network_params()[0].hef_path); } @@ -821,23 +817,16 @@ hailo_status Run2Command::execute() for (auto batch_size : batch_sizes_to_run) { if(app->get_measure_fw_actions()) { app->set_batch_size(batch_size); - - auto device = get_single_physical_device(*vdevice); - CHECK_EXPECTED_AS_STATUS(device); - - auto status = DownloadActionListCommand::set_batch_to_measure(device.release(), RUNTIME_DATA_BATCH_INDEX_TO_MEASURE_DEFAULT); + TRY(auto device, get_single_physical_device(*vdevice)); + auto status = DownloadActionListCommand::set_batch_to_measure(device, RUNTIME_DATA_BATCH_INDEX_TO_MEASURE_DEFAULT); CHECK_SUCCESS(status); } - auto expected_net_runners = app->init_and_run_net_runners(vdevice.get()); - CHECK_EXPECTED_AS_STATUS(expected_net_runners); - auto net_runners = expected_net_runners.release(); - + TRY(auto net_runners, app->init_and_run_net_runners(vdevice.get())); if(app->get_measure_fw_actions()) { // Collecting runtime data - auto device = get_single_physical_device(*vdevice); - CHECK_EXPECTED_AS_STATUS(device); - - auto status = DownloadActionListCommand::execute(device.release(), net_runners[0]->get_configured_network_group(), batch_size, action_list_json, net_runners[0]->get_last_measured_fps(), network_group_index); + TRY(auto device, get_single_physical_device(*vdevice)); + auto status = DownloadActionListCommand::execute(device, net_runners[0]->get_configured_network_group(), + batch_size, action_list_json, net_runners[0]->get_last_measured_fps(), network_group_index); CHECK_SUCCESS(status); network_group_index++; diff --git a/hailort/hailortcli/run_command.cpp b/hailort/hailortcli/run_command.cpp index 55b62a1..ef6074a 100644 --- a/hailort/hailortcli/run_command.cpp +++ b/hailort/hailortcli/run_command.cpp @@ -44,10 +44,8 @@ std::condition_variable wait_for_exit_cv; constexpr uint32_t DEFAULT_TIME_TO_RUN_SECONDS = 5; #ifndef HAILO_EMULATOR -constexpr std::chrono::milliseconds TIME_TO_WAIT_FOR_CONFIG(300); #define HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS (HAILO_DEFAULT_VSTREAM_TIMEOUT_MS) #else /* ifndef HAILO_EMULATOR */ -constexpr std::chrono::milliseconds TIME_TO_WAIT_FOR_CONFIG(30000); #define HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS (HAILO_DEFAULT_VSTREAM_TIMEOUT_MS * 100) #endif /* ifndef HAILO_EMULATOR */ static const char *RUNTIME_DATA_OUTPUT_PATH_HEF_PLACE_HOLDER = ""; @@ -449,21 +447,18 @@ Expected>> create_input_vstreams const inference_runner_params ¶ms) { std::map> res; - auto network_infos = configured_net_group.get_network_infos(); - CHECK_EXPECTED(network_infos); - for (auto &network_info : network_infos.value()) { + TRY(const auto network_infos, configured_net_group.get_network_infos()); + for (const auto &network_info : network_infos) { auto quantized = (params.transform.format_type != HAILO_FORMAT_TYPE_FLOAT32); - auto input_vstreams_params = configured_net_group.make_input_vstream_params(quantized, - params.transform.format_type, HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, network_info.name); - CHECK_EXPECTED(input_vstreams_params); + TRY(auto input_vstreams_params, configured_net_group.make_input_vstream_params(quantized, + params.transform.format_type, HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, network_info.name)); - for (auto &vstream_params : input_vstreams_params.value()) { + for (auto &vstream_params : input_vstreams_params) { vstream_params.second.pipeline_elements_stats_flags = inference_runner_params_to_pipeline_elem_stats_flags(params.pipeline_stats); vstream_params.second.vstream_stats_flags = inference_runner_params_to_vstream_stats_flags(params.pipeline_stats); } - auto input_vstreams = VStreamsBuilder::create_input_vstreams(configured_net_group, input_vstreams_params.value()); - CHECK_EXPECTED(input_vstreams); - res.emplace(network_info.name, input_vstreams.release()); + TRY(auto input_vstreams, VStreamsBuilder::create_input_vstreams(configured_net_group, input_vstreams_params)); + res.emplace(network_info.name, std::move(input_vstreams)); } return res; } @@ -472,28 +467,24 @@ Expected>> create_output_vstrea const inference_runner_params ¶ms) { std::map> res; - auto network_infos = configured_net_group.get_network_infos(); - CHECK_EXPECTED(network_infos); - for (auto &network_info : network_infos.value()) { + TRY(const auto network_infos, configured_net_group.get_network_infos()); + for (const auto &network_info : network_infos) { // Data is not quantized if format_type is explicitly float32, or if an output is NMS (which also enforces float32 output) // We don't cover a case of multiple outputs where only some of them are NMS (no such model currently), and anyway it is handled in run2 - auto vstream_infos = configured_net_group.get_output_vstream_infos(); - CHECK_EXPECTED(vstream_infos); - auto nms_output = std::any_of(vstream_infos->begin(), vstream_infos->end(), [] (const hailo_vstream_info_t &output_info) { + TRY(const auto vstream_infos, configured_net_group.get_output_vstream_infos()); + auto nms_output = std::any_of(vstream_infos.begin(), vstream_infos.end(), [] (const hailo_vstream_info_t &output_info) { return HailoRTCommon::is_nms(output_info); }); auto quantized = ((params.transform.format_type != HAILO_FORMAT_TYPE_FLOAT32) && !nms_output); - auto output_vstreams_params = configured_net_group.make_output_vstream_params(quantized, - params.transform.format_type, HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, network_info.name); - CHECK_EXPECTED(output_vstreams_params); + TRY(auto output_vstreams_params, configured_net_group.make_output_vstream_params(quantized, + params.transform.format_type, HAILORTCLI_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, network_info.name)); - for (auto &vstream_params : output_vstreams_params.value()) { + for (auto &vstream_params : output_vstreams_params) { vstream_params.second.pipeline_elements_stats_flags = inference_runner_params_to_pipeline_elem_stats_flags(params.pipeline_stats); vstream_params.second.vstream_stats_flags = inference_runner_params_to_vstream_stats_flags(params.pipeline_stats); } - auto output_vstreams = VStreamsBuilder::create_output_vstreams(configured_net_group, output_vstreams_params.value()); - CHECK_EXPECTED(output_vstreams); - res.emplace(network_info.name, output_vstreams.release()); + TRY(auto output_vstreams, VStreamsBuilder::create_output_vstreams(configured_net_group, output_vstreams_params)); + res.emplace(network_info.name, std::move(output_vstreams)); } return res; } @@ -501,12 +492,10 @@ Expected>> create_output_vstrea Expected>>> create_input_streams(ConfiguredNetworkGroup &configured_net_group) { std::map>> res; - auto network_infos = configured_net_group.get_network_infos(); - CHECK_EXPECTED(network_infos); - for (auto &network_info : network_infos.value()) { - auto input_streams = configured_net_group.get_input_streams_by_network(network_info.name); - CHECK_EXPECTED(input_streams); - res.emplace(network_info.name, input_streams.release()); + TRY(const auto network_infos, configured_net_group.get_network_infos()); + for (const auto &network_info : network_infos) { + TRY(auto input_streams, configured_net_group.get_input_streams_by_network(network_info.name)); + res.emplace(network_info.name, std::move(input_streams)); } return res; } @@ -514,12 +503,10 @@ Expected>> Expected>>> create_output_streams(ConfiguredNetworkGroup &configured_net_group) { std::map>> res; - auto network_infos = configured_net_group.get_network_infos(); - CHECK_EXPECTED(network_infos); - for (auto &network_info : network_infos.value()) { - auto output_streams = configured_net_group.get_output_streams_by_network(network_info.name); - CHECK_EXPECTED(output_streams); - res.emplace(network_info.name, output_streams.release()); + TRY(const auto network_infos, configured_net_group.get_network_infos()); + for (const auto &network_info : network_infos) { + TRY(auto output_streams, configured_net_group.get_output_streams_by_network(network_info.name)); + res.emplace(network_info.name, std::move(output_streams)); } return res; } @@ -532,9 +519,8 @@ Expected> create_output_buffers( std::map dst_data; for (auto &recv_objects : recv_objects_per_network) { for (auto &recv_object : recv_objects.second) { - auto buffer = Buffer::create_shared(recv_object.get().get_frame_size()); - CHECK_EXPECTED(buffer); - dst_data[recv_object.get().name()] = buffer.release(); + TRY(dst_data[recv_object.get().name()], + Buffer::create_shared(recv_object.get().get_frame_size())); } } @@ -643,9 +629,8 @@ static hailo_status run_streaming_impl(std::shared_ptr c std::vector> results; - auto network_progress_bar_exp = progress_bar.create_network_progress_bar(configured_net_group, network_name); - CHECK_EXPECTED_AS_STATUS(network_progress_bar_exp); - auto network_progress_bar = network_progress_bar_exp.release(); + TRY(auto network_progress_bar, + progress_bar.create_network_progress_bar(configured_net_group, network_name)); const auto start = std::chrono::high_resolution_clock::now(); // Launch async read/writes @@ -712,12 +697,10 @@ static hailo_status run_streaming_impl(std::shared_ptr c // TODO: HRT-7798 if (!params.vdevice_params.multi_process_service) { - auto network_input_streams = configured_net_group->get_input_streams_by_network(network_name); - CHECK_EXPECTED_AS_STATUS(network_input_streams); - inference_result.m_total_send_frame_size = total_send_frame_size(network_input_streams.value()); - auto network_output_streams = configured_net_group->get_output_streams_by_network(network_name); - CHECK_EXPECTED_AS_STATUS(network_output_streams); - inference_result.m_total_recv_frame_size = total_recv_frame_size(network_output_streams.value()); + TRY(auto network_input_streams, configured_net_group->get_input_streams_by_network(network_name)); + inference_result.m_total_send_frame_size = total_send_frame_size(network_input_streams); + TRY(auto network_output_streams, configured_net_group->get_output_streams_by_network(network_name)); + inference_result.m_total_recv_frame_size = total_recv_frame_size(network_output_streams); } if (params.measure_latency) { @@ -731,9 +714,8 @@ static hailo_status run_streaming_impl(std::shared_ptr c } if (params.measure_overall_latency) { - auto overall_latency = overall_latency_meter.get_latency(true); - CHECK_EXPECTED_AS_STATUS(overall_latency); - inference_result.m_overall_latency = std::make_unique(*overall_latency); + TRY(auto overall_latency, overall_latency_meter.get_latency(true)); + inference_result.m_overall_latency = std::make_unique(std::move(overall_latency)); } return HAILO_SUCCESS; @@ -759,10 +741,7 @@ static Expected run_streaming(const std::vector> networks_results; // Map of networks results for each network group networks_results.reserve(configured_net_groups.size()); - auto progress_bar_exp = InferProgress::create(params, std::chrono::seconds(1)); - CHECK_EXPECTED(progress_bar_exp); - auto progress_bar = progress_bar_exp.release(); - + TRY(auto progress_bar, InferProgress::create(params, std::chrono::seconds(1))); for (size_t network_group_index = 0; network_group_index < configured_net_groups.size(); network_group_index++) { networks_threads_status.emplace_back(); networks_results.emplace_back(); @@ -851,15 +830,13 @@ static Expected run_inference(const std::vector>>(in_vstreams.release()); + TRY(auto in_vstreams, create_input_vstreams(*configured_net_groups[network_group_index], params)); + auto in_vstreams_ptr = make_shared_nothrow>>(std::move(in_vstreams)); CHECK_NOT_NULL_AS_EXPECTED(in_vstreams_ptr, HAILO_OUT_OF_HOST_MEMORY); input_vstreams[network_group_index] = in_vstreams_ptr; - auto out_vstreams = create_output_vstreams(*configured_net_groups[network_group_index], params); - CHECK_EXPECTED(out_vstreams); - auto out_vstreams_ptr = make_shared_nothrow>>(out_vstreams.release()); + TRY(auto out_vstreams, create_output_vstreams(*configured_net_groups[network_group_index], params)); + auto out_vstreams_ptr = make_shared_nothrow>>(std::move(out_vstreams)); CHECK_NOT_NULL_AS_EXPECTED(out_vstreams_ptr, HAILO_OUT_OF_HOST_MEMORY); output_vstreams[network_group_index] = out_vstreams_ptr; @@ -879,9 +856,7 @@ static Expected run_inference(const std::vector(configured_net_groups, input_datasets, @@ -896,7 +871,7 @@ static Expected run_inference(const std::vector run_inference(const std::vector()); for (size_t network_group_index = 0; network_group_index < configured_net_groups.size(); network_group_index++) { - auto input_streams = create_input_streams(*configured_net_groups[network_group_index]); - CHECK_EXPECTED(input_streams); - input_streams_refs[network_group_index] = input_streams.release(); - auto output_streams = create_output_streams(*configured_net_groups[network_group_index]); - output_streams_refs[network_group_index] = output_streams.release(); - - auto network_group_output_buffers = create_output_buffers(output_streams_refs[network_group_index]); - CHECK_EXPECTED(network_group_output_buffers); - output_buffers[network_group_index] = network_group_output_buffers.release(); + TRY(input_streams_refs[network_group_index], create_input_streams(*configured_net_groups[network_group_index])); + TRY(output_streams_refs[network_group_index], create_output_streams(*configured_net_groups[network_group_index])); + TRY(output_buffers[network_group_index], create_output_buffers(output_streams_refs[network_group_index])); } return run_streaming(configured_net_groups, input_datasets, output_buffers, @@ -929,19 +898,6 @@ static Expected run_inference(const std::vector> activate_network_group(ConfiguredNetworkGroup &network_group) -{ - hailo_activate_network_group_params_t network_group_params = {}; - auto activated_network_group = network_group.activate(network_group_params); - CHECK_EXPECTED(activated_network_group, "Failed activating network group"); - - // Wait for configuration - // TODO: HRT-2492 wait for config in a normal way - std::this_thread::sleep_for(TIME_TO_WAIT_FOR_CONFIG); - - return activated_network_group; -} - static Expected> create_constant_dataset( const std::pair, std::vector> &input_infos, const hailo_transform_params_t &trans_params, InferMode mode) @@ -1000,35 +956,31 @@ static Expected> create_dataset_from_files( const auto file_path_it = file_paths.find(stream_name); CHECK_AS_EXPECTED(file_paths.end() != file_path_it, HAILO_INVALID_ARGUMENT, "Missing input file for input: {}", stream_name); - auto host_buffer = read_binary_file(file_path_it->second); - CHECK_EXPECTED(host_buffer, "Failed reading file {}", file_path_it->second); - CHECK_AS_EXPECTED((host_buffer->size() % host_frame_size) == 0, HAILO_INVALID_ARGUMENT, - "Input file ({}) size {} must be a multiple of the frame size {} ({})", file_path_it->second, host_buffer->size(), host_frame_size, stream_name); + TRY(auto host_buffer, read_binary_file(file_path_it->second)); + CHECK_AS_EXPECTED((host_buffer.size() % host_frame_size) == 0, HAILO_INVALID_ARGUMENT, + "Input file ({}) size {} must be a multiple of the frame size {} ({})", file_path_it->second, host_buffer.size(), host_frame_size, stream_name); if (InferMode::HW_ONLY == mode) { auto matching_stream_info = std::find_if(input_infos.first.begin(), input_infos.first.end(), [&stream_name] (const auto &stream_info) { return std::string(stream_info.name) == stream_name; }); CHECK_AS_EXPECTED(matching_stream_info != input_infos.first.end(), HAILO_INVALID_OPERATION, "Failed to find raw-stream with name {}.", stream_name); - const size_t frames_count = (host_buffer->size() / host_frame_size); + const size_t frames_count = (host_buffer.size() / host_frame_size); const size_t hw_frame_size = matching_stream_info->hw_frame_size; const size_t hw_buffer_size = frames_count * hw_frame_size; - auto hw_buffer = Buffer::create_shared(hw_buffer_size); - CHECK_EXPECTED(hw_buffer); - - auto transform_context = InputTransformContext::create(*matching_stream_info, trans_params); - CHECK_EXPECTED(transform_context); + TRY(auto hw_buffer, Buffer::create_shared(hw_buffer_size)); + TRY(auto transform_context, InputTransformContext::create(*matching_stream_info, trans_params)); for (size_t i = 0; i < frames_count; i++) { - MemoryView host_data(static_cast(host_buffer->data() + (i*host_frame_size)), host_frame_size); - MemoryView hw_data(static_cast(hw_buffer.value()->data() + (i*hw_frame_size)), hw_frame_size); + MemoryView host_data(static_cast(host_buffer.data() + (i*host_frame_size)), host_frame_size); + MemoryView hw_data(static_cast(hw_buffer->data() + (i*hw_frame_size)), hw_frame_size); - auto status = transform_context.value()->transform(host_data, hw_data); + auto status = transform_context->transform(host_data, hw_data); CHECK_SUCCESS_AS_EXPECTED(status); } - dataset[stream_name] = hw_buffer.release(); + dataset[stream_name] = std::move(hw_buffer); } else { - auto host_buffer_shared = make_shared_nothrow(host_buffer.release()); + auto host_buffer_shared = make_shared_nothrow(std::move(host_buffer)); CHECK_NOT_NULL_AS_EXPECTED(host_buffer_shared, HAILO_OUT_OF_HOST_MEMORY); dataset[stream_name] = host_buffer_shared; } @@ -1052,30 +1004,25 @@ static Expected>> create_dataset( // Vector of len(ng.conut), each element is pair of all input_stream_infos, and all input_vstream_infos std::vector, std::vector>> input_infos; for (auto &network_group : network_groups) { - auto expected_all_streams_infos = network_group->get_all_stream_infos(); - CHECK_EXPECTED(expected_all_streams_infos); - auto &all_stream_infos = expected_all_streams_infos.value(); + TRY(const auto all_streams_infos, network_group->get_all_stream_infos()); std::vector group_input_stream_infos; - std::copy_if(all_stream_infos.begin(), all_stream_infos.end(), std::back_inserter(group_input_stream_infos), [](const auto &info) { + std::copy_if(all_streams_infos.begin(), all_streams_infos.end(), std::back_inserter(group_input_stream_infos), [](const auto &info) { return info.direction == HAILO_H2D_STREAM; }); - auto expected_input_vstreams_infos = network_group->get_input_vstream_infos(); - CHECK_EXPECTED(expected_input_vstreams_infos); - input_infos.push_back({group_input_stream_infos, expected_input_vstreams_infos.release()}); + TRY(const auto input_vstreams_infos, network_group->get_input_vstream_infos()); + input_infos.push_back({group_input_stream_infos, std::move(input_vstreams_infos)}); } if (!params.inputs_name_and_file_path.empty()) { for (auto &group_input_infos : input_infos) { - auto network_group_dataset = create_dataset_from_files(group_input_infos, params.inputs_name_and_file_path, - trans_params, params.mode); - CHECK_EXPECTED(network_group_dataset); - results.emplace_back(network_group_dataset.release()); + TRY(auto network_group_dataset, create_dataset_from_files(group_input_infos, params.inputs_name_and_file_path, + trans_params, params.mode)); + results.emplace_back(std::move(network_group_dataset)); } } else { for (auto &group_input_infos : input_infos) { - auto network_group_dataset = create_constant_dataset(group_input_infos, trans_params, params.mode); - CHECK_EXPECTED(network_group_dataset); - results.emplace_back(network_group_dataset.release()); + TRY(auto network_group_dataset, create_constant_dataset(group_input_infos, trans_params, params.mode)); + results.emplace_back(std::move(network_group_dataset)); } } return results; @@ -1087,11 +1034,8 @@ Expected activate_and_run_single_device( const inference_runner_params ¶ms) { CHECK_AS_EXPECTED(1 == network_groups.size(), HAILO_INVALID_OPERATION, "Inference is not supported on HEFs with multiple network groups"); - auto activated_net_group = activate_network_group(*network_groups[0]); - CHECK_EXPECTED(activated_net_group, "Failed activate network_group"); - - auto input_dataset = create_dataset(network_groups, params); - CHECK_EXPECTED(input_dataset, "Failed creating input dataset"); + TRY(auto activated_net_group, network_groups[0]->activate(), "Failed activate network_group"); + TRY(auto input_dataset, create_dataset(network_groups, params)); hailo_power_measurement_types_t measurement_type = HAILO_POWER_MEASUREMENT_TYPES__MAX_ENUM; bool should_measure_power = false; @@ -1103,48 +1047,43 @@ Expected activate_and_run_single_device( should_measure_power = true; } - std::shared_ptr long_power_measurement = nullptr; + std::shared_ptr long_power_measurement_ptr = nullptr; if (should_measure_power) { - auto long_power_measurement_exp = PowerMeasurementSubcommand::start_power_measurement(device, - HAILO_DVM_OPTIONS_AUTO, - measurement_type, params.power_measurement.sampling_period, params.power_measurement.averaging_factor); - CHECK_EXPECTED(long_power_measurement_exp); - long_power_measurement = make_shared_nothrow(long_power_measurement_exp.release()); - CHECK_NOT_NULL_AS_EXPECTED(long_power_measurement, HAILO_OUT_OF_HOST_MEMORY); + TRY(auto long_power_measurement, PowerMeasurementSubcommand::start_power_measurement(device, + HAILO_DVM_OPTIONS_AUTO, measurement_type, params.power_measurement.sampling_period, params.power_measurement.averaging_factor)); + long_power_measurement_ptr = make_shared_nothrow(std::move(long_power_measurement)); + CHECK_NOT_NULL_AS_EXPECTED(long_power_measurement_ptr, HAILO_OUT_OF_HOST_MEMORY); } bool should_measure_temp = params.measure_temp; - auto temp_measure = TemperatureMeasurement::create_shared(device); - CHECK_EXPECTED(temp_measure); + TRY(auto temp_measure, TemperatureMeasurement::create_shared(device)); if (should_measure_temp) { - auto status = temp_measure.value()->start_measurement(); + auto status = temp_measure->start_measurement(); CHECK_SUCCESS_AS_EXPECTED(status, "Failed to get chip's temperature"); } - auto infer_result = run_inference(network_groups, input_dataset.value(), params); - CHECK_EXPECTED(infer_result, "Error failed running inference"); + TRY(auto inference_result, run_inference(network_groups, input_dataset, params), "Error while running inference"); - InferResult inference_result(infer_result.release()); std::vector> device_refs; device_refs.push_back(device); inference_result.initialize_measurements(device_refs); if (should_measure_power) { - auto status = long_power_measurement->stop(); + auto status = long_power_measurement_ptr->stop(); CHECK_SUCCESS_AS_EXPECTED(status); if (params.power_measurement.measure_current) { - status = inference_result.set_current_measurement(device.get_dev_id(), std::move(long_power_measurement)); + status = inference_result.set_current_measurement(device.get_dev_id(), std::move(long_power_measurement_ptr)); CHECK_SUCCESS_AS_EXPECTED(status); } else { - status = inference_result.set_power_measurement(device.get_dev_id(), std::move(long_power_measurement)); + status = inference_result.set_power_measurement(device.get_dev_id(), std::move(long_power_measurement_ptr)); CHECK_SUCCESS_AS_EXPECTED(status); } } if (should_measure_temp) { - temp_measure.value()->stop_measurement(); - auto temp_measure_p = make_shared_nothrow(temp_measure.value()->get_data()); + temp_measure->stop_measurement(); + auto temp_measure_p = make_shared_nothrow(temp_measure->get_data()); CHECK_NOT_NULL_AS_EXPECTED(temp_measure_p, HAILO_OUT_OF_HOST_MEMORY); auto status = inference_result.set_temp_measurement(device.get_dev_id(), std::move(temp_measure_p)); CHECK_SUCCESS_AS_EXPECTED(status); @@ -1158,9 +1097,8 @@ Expected get_min_inferred_frames_count(InferResult &inference_result) size_t min_frames_count = UINT32_MAX; for (auto &network_group_results : inference_result.network_group_results()) { for (const auto &network_results_pair : network_group_results.results_per_network()) { - auto frames_count = network_group_results.frames_count(network_results_pair.first); - CHECK_EXPECTED(frames_count); - min_frames_count = std::min(frames_count.value(), min_frames_count); + TRY(const auto frames_count, network_group_results.frames_count(network_results_pair.first)); + min_frames_count = std::min(frames_count, min_frames_count); } } return min_frames_count; @@ -1168,38 +1106,29 @@ Expected get_min_inferred_frames_count(InferResult &inference_result) Expected run_command_hef_single_device(const inference_runner_params ¶ms) { - auto devices = create_devices(params.vdevice_params.device_params); - CHECK_EXPECTED(devices, "Failed creating device"); + TRY(auto devices, create_devices(params.vdevice_params.device_params), "Failed creating device"); /* This function supports controls for multiple devices. We validate there is only 1 device generated as we are on a single device flow */ - CHECK_AS_EXPECTED(1 == devices->size(), HAILO_INTERNAL_FAILURE); - auto &device = devices.value()[0]; - - auto hef = Hef::create(params.hef_path.c_str()); - CHECK_EXPECTED(hef, "Failed reading hef file {}", params.hef_path); + CHECK_AS_EXPECTED(1 == devices.size(), HAILO_INTERNAL_FAILURE); + auto &device = devices[0]; - auto interface = device->get_default_streams_interface(); - CHECK_EXPECTED(interface, "Failed to get default streams interface"); - - auto configure_params = get_configure_params(params, hef.value(), interface.value()); - CHECK_EXPECTED(configure_params); - - auto network_group_list = device->configure(hef.value(), configure_params.value()); - CHECK_EXPECTED(network_group_list, "Failed configure device from hef"); + TRY(auto hef, Hef::create(params.hef_path.c_str()), "Failed reading hef file {}", params.hef_path); + TRY(const auto interface, device->get_default_streams_interface(), "Failed to get default streams interface"); + TRY(auto configure_params, get_configure_params(params, hef, interface)); + TRY(auto network_group_list, device->configure(hef, configure_params), "Failed configure device from hef"); if (use_batch_to_measure_opt(params)) { auto status = DownloadActionListCommand::set_batch_to_measure(*device, params.runtime_data.batch_to_measure); CHECK_SUCCESS_AS_EXPECTED(status); } - auto inference_result = activate_and_run_single_device(*device, network_group_list.value(), params); + auto inference_result = activate_and_run_single_device(*device, network_group_list, params); if (use_batch_to_measure_opt(params) && (0 == params.frames_count) && inference_result) { - auto min_frames_count = get_min_inferred_frames_count(inference_result.value()); - CHECK_EXPECTED(min_frames_count); - if (min_frames_count.value() < params.runtime_data.batch_to_measure) { + TRY(auto min_frames_count, get_min_inferred_frames_count(inference_result.value())); + if (min_frames_count < params.runtime_data.batch_to_measure) { LOGGER__WARNING("Number of frames sent ({}) is smaller than --batch-to-measure ({}), " - "hence timestamps will not be updated in runtime data", min_frames_count.value() , + "hence timestamps will not be updated in runtime data", min_frames_count, params.runtime_data.batch_to_measure); } } @@ -1207,11 +1136,11 @@ Expected run_command_hef_single_device(const inference_runner_param if (params.runtime_data.collect_runtime_data) { const auto runtime_data_output_path = format_runtime_data_output_path( params.runtime_data.runtime_data_output_path, params.hef_path); - DownloadActionListCommand::execute(*device, runtime_data_output_path, network_group_list.value(), + DownloadActionListCommand::execute(*device, runtime_data_output_path, network_group_list, params.hef_path); } - CHECK_EXPECTED(inference_result); + CHECK_EXPECTED(inference_result); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here return inference_result; } @@ -1224,13 +1153,10 @@ Expected activate_and_run_vdevice( { std::unique_ptr activated_network_group; if (!scheduler_is_used) { - auto activated_net_group_exp = activate_network_group(*network_groups[0]); - CHECK_EXPECTED(activated_net_group_exp, "Failed activate network_group"); - activated_network_group = activated_net_group_exp.release(); + TRY(activated_network_group, network_groups[0]->activate()); } - auto input_dataset = create_dataset(network_groups, params); - CHECK_EXPECTED(input_dataset, "Failed creating input dataset"); + TRY(const auto input_dataset, create_dataset(network_groups, params), "Failed creating input dataset"); hailo_power_measurement_types_t measurement_type = HAILO_POWER_MEASUREMENT_TYPES__MAX_ENUM; bool should_measure_power = false; @@ -1245,11 +1171,10 @@ Expected activate_and_run_vdevice( std::map> power_measurements; if (should_measure_power) { for (auto &device : physical_devices) { - auto long_power_measurement_exp = PowerMeasurementSubcommand::start_power_measurement(device, - HAILO_DVM_OPTIONS_AUTO, - measurement_type, params.power_measurement.sampling_period, params.power_measurement.averaging_factor); - CHECK_EXPECTED(long_power_measurement_exp, "Failed starting power measurement on device {}", device.get().get_dev_id()); - auto long_power_measurement_p = make_shared_nothrow(long_power_measurement_exp.release()); + TRY(auto long_power_measurement, PowerMeasurementSubcommand::start_power_measurement(device, + HAILO_DVM_OPTIONS_AUTO, measurement_type, params.power_measurement.sampling_period, params.power_measurement.averaging_factor), + "Failed starting power measurement on device {}", device.get().get_dev_id()); + auto long_power_measurement_p = make_shared_nothrow(std::move(long_power_measurement)); CHECK_NOT_NULL_AS_EXPECTED(long_power_measurement_p, HAILO_OUT_OF_HOST_MEMORY); power_measurements.emplace(device.get().get_dev_id(), std::move(long_power_measurement_p)); } @@ -1258,18 +1183,14 @@ Expected activate_and_run_vdevice( std::map> temp_measurements; if (params.measure_temp) { for (auto &device : physical_devices) { - auto temp_measure = TemperatureMeasurement::create_shared(device); - CHECK_EXPECTED(temp_measure); - auto status = temp_measure.value()->start_measurement(); + TRY(auto temp_measure, TemperatureMeasurement::create_shared(device)); + auto status = temp_measure->start_measurement(); CHECK_SUCCESS_AS_EXPECTED(status, "Failed starting temperature measurement on device {}", device.get().get_dev_id()); - temp_measurements.emplace(device.get().get_dev_id(), temp_measure.release()); + temp_measurements.emplace(device.get().get_dev_id(), temp_measure); } } - auto infer_result = run_inference(network_groups, input_dataset.value(), params); - CHECK_EXPECTED(infer_result, "Error failed running inference"); - - InferResult inference_result(infer_result.release()); + TRY(auto inference_result, run_inference(network_groups, input_dataset, params), "Error failed running inference"); inference_result.initialize_measurements(physical_devices); if (should_measure_power) { @@ -1311,12 +1232,9 @@ Expected activate_and_run_vdevice( Expected run_command_hef_vdevice(const inference_runner_params ¶ms) { - auto hef = Hef::create(params.hef_path.c_str()); - CHECK_EXPECTED(hef, "Failed reading hef file {}", params.hef_path); - - auto network_groups_infos = hef->get_network_groups_infos(); - CHECK_EXPECTED(network_groups_infos); - bool scheduler_is_used = (1 < network_groups_infos->size()) || params.vdevice_params.multi_process_service; + TRY(auto hef, Hef::create(params.hef_path.c_str()), "Failed reading hef file {}", params.hef_path); + TRY(auto network_groups_infos, hef.get_network_groups_infos()); + bool scheduler_is_used = (1 < network_groups_infos.size()) || params.vdevice_params.multi_process_service; hailo_vdevice_params_t vdevice_params = {}; auto status = hailo_init_vdevice_params(&vdevice_params); @@ -1326,12 +1244,8 @@ Expected run_command_hef_vdevice(const inference_runner_params &par } std::vector dev_ids; if (!params.vdevice_params.device_params.device_ids.empty()) { - auto dev_ids_exp = get_device_ids(params.vdevice_params.device_params); - CHECK_EXPECTED(dev_ids_exp); - - auto dev_ids_struct_exp = HailoRTCommon::to_device_ids_vector(dev_ids_exp.value()); - CHECK_EXPECTED(dev_ids_struct_exp); - dev_ids = dev_ids_struct_exp.release(); + TRY(auto dev_ids_strs, get_device_ids(params.vdevice_params.device_params)); + TRY(dev_ids, HailoRTCommon::to_device_ids_vector(dev_ids_strs)); vdevice_params.device_ids = dev_ids.data(); vdevice_params.device_count = static_cast(dev_ids.size()); @@ -1339,42 +1253,47 @@ Expected run_command_hef_vdevice(const inference_runner_params &par vdevice_params.scheduling_algorithm = (scheduler_is_used) ? HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN : HAILO_SCHEDULING_ALGORITHM_NONE; vdevice_params.group_id = params.vdevice_params.group_id.c_str(); vdevice_params.multi_process_service = params.vdevice_params.multi_process_service; - auto vdevice = VDevice::create(vdevice_params); - CHECK_EXPECTED(vdevice, "Failed creating vdevice"); + TRY(auto vdevice, VDevice::create(vdevice_params), "Failed creating vdevice"); std::vector> physical_devices; if (!params.vdevice_params.multi_process_service) { - auto expected_physical_devices = vdevice.value()->get_physical_devices(); - CHECK_EXPECTED(expected_physical_devices); - physical_devices = expected_physical_devices.value(); + TRY(physical_devices, vdevice->get_physical_devices()); } - auto interface = vdevice.value()->get_default_streams_interface(); - CHECK_EXPECTED(interface, "Failed to get default streams interface"); - - auto configure_params = get_configure_params(params, hef.value(), *interface); - CHECK_EXPECTED(configure_params); - - auto network_group_list = vdevice.value()->configure(hef.value(), configure_params.value()); - CHECK_EXPECTED(network_group_list, "Failed configure vdevice from hef"); + TRY(const auto interface, vdevice->get_default_streams_interface(), "Failed to get default streams interface"); + TRY(auto configure_params, get_configure_params(params, hef, interface)); + TRY(auto network_group_list, vdevice->configure(hef, configure_params), "Failed configure vdevice from hef"); for (auto &device : physical_devices) { + TRY(const auto identity, device.get().identify()); + CHECK_AS_EXPECTED((HailoRTCommon::is_power_measurement_supported(identity.device_architecture) || + !(params.power_measurement.measure_power)), HAILO_INVALID_OPERATION, + "HW arch {} does not support power measurement. Disable the power-measure option", + HailoRTCommon::get_device_arch_str(identity.device_architecture)); + CHECK_AS_EXPECTED((HailoRTCommon::is_current_measurement_supported(identity.device_architecture) || + !(params.power_measurement.measure_current)), HAILO_INVALID_OPERATION, + "HW arch {} does not support current measurement. Disable the current-measure option", + HailoRTCommon::get_device_arch_str(identity.device_architecture)); + CHECK_AS_EXPECTED((HailoRTCommon::is_temp_measurement_supported(identity.device_architecture) || + !(params.measure_temp)), HAILO_INVALID_OPERATION, + "HW arch {} does not support temperature measurement. Disable the temp-measure option", + HailoRTCommon::get_device_arch_str(identity.device_architecture)); + if (use_batch_to_measure_opt(params)) { status = DownloadActionListCommand::set_batch_to_measure(device.get(), params.runtime_data.batch_to_measure); CHECK_SUCCESS_AS_EXPECTED(status); } } - auto infer_result = activate_and_run_vdevice(physical_devices, scheduler_is_used, network_group_list.value(), params); - CHECK_EXPECTED(infer_result, "Error failed running inference"); + TRY(auto infer_result, activate_and_run_vdevice(physical_devices, scheduler_is_used, network_group_list, params), + "Error while running inference"); for (auto &device : physical_devices) { - if (use_batch_to_measure_opt(params) && (0 == params.frames_count) && infer_result) { - auto min_frames_count = get_min_inferred_frames_count(infer_result.value()); - CHECK_EXPECTED(min_frames_count); - if (min_frames_count.value() < params.runtime_data.batch_to_measure) { + if (use_batch_to_measure_opt(params) && (0 == params.frames_count)) { + TRY(const auto min_frames_count, get_min_inferred_frames_count(infer_result)); + if (min_frames_count < params.runtime_data.batch_to_measure) { LOGGER__WARNING("Number of frames sent ({}) is smaller than --batch-to-measure ({}), " - "hence timestamps will not be updated in runtime data", min_frames_count.value() , + "hence timestamps will not be updated in runtime data", min_frames_count, params.runtime_data.batch_to_measure); } } @@ -1383,7 +1302,7 @@ Expected run_command_hef_vdevice(const inference_runner_params &par auto output_path = (1 == physical_devices.size()) ? params.runtime_data.runtime_data_output_path : (std::string(device.get().get_dev_id()) + "_" + params.runtime_data.runtime_data_output_path); const auto runtime_data_output_path = format_runtime_data_output_path(output_path, params.hef_path); - DownloadActionListCommand::execute(device.get(), runtime_data_output_path, network_group_list.value(), + DownloadActionListCommand::execute(device.get(), runtime_data_output_path, network_group_list, params.hef_path); } } @@ -1402,18 +1321,16 @@ Expected use_vdevice(const hailo_vdevice_params ¶ms) return true; } - auto device_type = Device::get_device_type(params.device_params.device_ids[0]); - CHECK_EXPECTED(device_type); + TRY(const auto device_type, Device::get_device_type(params.device_params.device_ids[0])); - return device_type.value() != Device::Type::ETH; + return device_type != Device::Type::ETH; } Expected run_command_hef(const inference_runner_params ¶ms) { - auto use_vdevice_expected = use_vdevice(params.vdevice_params); - CHECK_EXPECTED(use_vdevice_expected); + TRY(auto use_vdevice, use_vdevice(params.vdevice_params)); - if (use_vdevice_expected.value()) { + if (use_vdevice) { return run_command_hef_vdevice(params); } else { @@ -1428,17 +1345,15 @@ static hailo_status run_command_hefs_dir(const inference_runner_params ¶ms, std::string hef_dir = params.hef_path; inference_runner_params curr_params = params; - const auto files = Filesystem::get_files_in_dir_flat(hef_dir); - CHECK_EXPECTED_AS_STATUS(files); + TRY(const auto files, Filesystem::get_files_in_dir_flat(hef_dir)); - for (const auto &full_path : files.value()) { + for (const auto &full_path : files) { if (Filesystem::has_suffix(full_path, ".hef")) { contains_hef = true; curr_params.hef_path = full_path; std::cout << std::string(80, '*') << std::endl << "Inferring " << full_path << ":"<< std::endl; - auto hef = Hef::create(full_path); - CHECK_EXPECTED_AS_STATUS(hef); - auto network_groups_names = hef->get_network_groups_names(); + TRY(auto hef, Hef::create(full_path)); + auto network_groups_names = hef.get_network_groups_names(); auto infer_stats = run_command_hef(curr_params); printer.print(network_groups_names , infer_stats); @@ -1458,23 +1373,20 @@ static hailo_status run_command_hefs_dir(const inference_runner_params ¶ms, hailo_status run_command(const inference_runner_params ¶ms) { - auto printer = InferStatsPrinter::create(params); - CHECK_EXPECTED_AS_STATUS(printer, "Failed to initialize infer stats printer"); + TRY(auto printer, InferStatsPrinter::create(params), "Failed to initialize infer stats printer"); if (!params.csv_output.empty()) { - printer->print_csv_header(); + printer.print_csv_header(); } - auto is_dir = Filesystem::is_directory(params.hef_path.c_str()); - CHECK_EXPECTED_AS_STATUS(is_dir, "Failed checking if path is directory"); + TRY(const auto is_dir, Filesystem::is_directory(params.hef_path.c_str()), "Failed checking if path is directory"); - if (is_dir.value()){ - return run_command_hefs_dir(params, printer.value()); + if (is_dir){ + return run_command_hefs_dir(params, printer); } else { auto infer_stats = run_command_hef(params); - auto hef = Hef::create(params.hef_path.c_str()); - CHECK_EXPECTED_AS_STATUS(hef); - auto network_groups_names = hef->get_network_groups_names(); - printer->print(network_groups_names, infer_stats); + TRY(auto hef, Hef::create(params.hef_path.c_str())); + auto network_groups_names = hef.get_network_groups_names(); + printer.print(network_groups_names, infer_stats); return infer_stats.status(); } } diff --git a/hailort/hailortcli/scan_command.cpp b/hailort/hailortcli/scan_command.cpp index 74e5d5f..814ce98 100644 --- a/hailort/hailortcli/scan_command.cpp +++ b/hailort/hailortcli/scan_command.cpp @@ -36,23 +36,20 @@ hailo_status ScanSubcommand::execute() return scan(); } else { - auto res = scan_ethernet(m_interface_ip_addr, m_interface_name); - CHECK_EXPECTED_AS_STATUS(res); + TRY(const auto res, scan_ethernet(m_interface_ip_addr, m_interface_name)); return HAILO_SUCCESS; } } hailo_status ScanSubcommand::scan() { - auto device_ids = Device::scan(); - CHECK_EXPECTED_AS_STATUS(device_ids); - - if (device_ids->size() == 0) { + TRY(const auto device_ids, Device::scan()); + if (device_ids.size() == 0) { std::cout << "Hailo devices not found" << std::endl; } else { std::cout << "Hailo Devices:" << std::endl; - for (const auto& device_id : device_ids.value()) { + for (const auto &device_id : device_ids) { std::cout << "[-] Device: " << device_id << std::endl; } } diff --git a/hailort/hailortcli/sensor_config_command.cpp b/hailort/hailortcli/sensor_config_command.cpp index 37232aa..0cb7137 100644 --- a/hailort/hailortcli/sensor_config_command.cpp +++ b/hailort/hailortcli/sensor_config_command.cpp @@ -108,9 +108,8 @@ SensorSectionsInfoSubcommand::SensorSectionsInfoSubcommand(CLI::App &parent_app) hailo_status SensorSectionsInfoSubcommand::execute_on_device(Device &device) { - auto sections_info = device.sensor_get_sections_info(); - CHECK_EXPECTED_AS_STATUS(sections_info); - return print_sections_info((SENSOR_CONFIG__section_info_t*)sections_info->data()); + TRY(auto sections_info, device.sensor_get_sections_info()); + return print_sections_info((SENSOR_CONFIG__section_info_t*)sections_info.data()); } hailo_status SensorSectionsInfoSubcommand::print_sections_info(SENSOR_CONFIG__section_info_t *operation_cfg) @@ -125,11 +124,11 @@ hailo_status SensorSectionsInfoSubcommand::print_sections_info(SENSOR_CONFIG__se } else { std::string reset_config = section_info->no_reset_offset ? "not valid" : "valid"; - auto sensor_type_expected = convert_sensor_type_to_string(section_info->sensor_type); - CHECK_EXPECTED_AS_STATUS(sensor_type_expected, "Failed convert sensor type to string"); + TRY( const auto sensor_type, convert_sensor_type_to_string(section_info->sensor_type), + "Failed convert sensor type to string"); std::cout << "Configuration Name: " << section_info->config_name << "\n"; - std::cout << "Sensor Type: " << sensor_type_expected.value() << "\n"; + std::cout << "Sensor Type: " << sensor_type << "\n"; std::cout << "Configuration lines number: " << (section_info->config_size / sizeof(SENSOR_CONFIG__operation_cfg_t)) << "\n"; std::cout << "Configuration size in bytes: " << section_info->config_size << "\n"; std::cout << "Reset configuration: " << reset_config << "\n"; diff --git a/hailort/hailortcli/udp_rate_limiter_command.cpp b/hailort/hailortcli/udp_rate_limiter_command.cpp index bbcb051..1323509 100644 --- a/hailort/hailortcli/udp_rate_limiter_command.cpp +++ b/hailort/hailortcli/udp_rate_limiter_command.cpp @@ -140,11 +140,10 @@ std::map UdpRateLimiterCommand::reset_commnad(const std: hailo_status UdpRateLimiterCommand::autoset_commnad(const std::vector &board_ports) { - const auto rates_from_hef = calc_rate_from_hef(m_hef_path, m_network_group_name, m_fps); - CHECK_EXPECTED_AS_STATUS(rates_from_hef); + TRY(const auto rates_from_hef, calc_rate_from_hef(m_hef_path, m_network_group_name, m_fps)); // On auto set, we use min rate for all input ports - auto min_rate_pair = *std::min_element(rates_from_hef.value().begin(), rates_from_hef.value().end(), + auto min_rate_pair = *std::min_element(rates_from_hef.begin(), rates_from_hef.end(), [](const auto& lhs, const auto& rhs) { return lhs.second < rhs.second; }); return set_command(board_ports, static_cast(min_rate_pair.second)); @@ -197,16 +196,11 @@ uint32_t UdpRateLimiterCommand::bit_rate_kbit_sec_to_bytes_sec(uint32_t rate_kbi Expected> UdpRateLimiterCommand::calc_rate_from_hef(const std::string &hef_path, const std::string &network_group_name, uint32_t fps) { - auto hef = Hef::create(hef_path.c_str()); - CHECK_EXPECTED(hef, "Failed reading hef file {}", hef_path.c_str()); + TRY(auto hef, Hef::create(hef_path.c_str()), "Failed reading hef file {}", hef_path.c_str()); + TRY(auto rate_calc, NetworkUdpRateCalculator::create(&(hef), network_group_name)); + TRY(auto calculated_rates, rate_calc.calculate_inputs_bandwith(fps)); - auto rate_calc = NetworkUdpRateCalculator::create(&(hef.value()), network_group_name); - CHECK_EXPECTED(rate_calc); - - auto calculated_rates = rate_calc->calculate_inputs_bandwith(fps); - CHECK_EXPECTED(calculated_rates); - - return calculated_rates.release(); + return calculated_rates; } std::vector UdpRateLimiterCommand::get_dports() diff --git a/hailort/hrpc/CMakeLists.txt b/hailort/hrpc/CMakeLists.txt new file mode 100644 index 0000000..ac354e8 --- /dev/null +++ b/hailort/hrpc/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(HRPC_IMPL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/os") +if(WIN32) + set(HRPC_OS_DIR "${HRPC_IMPL_DIR}/windows") +elseif(UNIX) + set(HRPC_OS_DIR "${HRPC_IMPL_DIR}/posix") +else() + message(FATAL_ERROR "Unexpeced host, stopping build") +endif() + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/rpc_connection.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/raw_connection.cpp + ${HRPC_IMPL_DIR}/pcie/raw_connection_internal.cpp + ${HRPC_OS_DIR}/raw_connection_internal.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/client.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/server.cpp +) + +set(HRPC_CPP_SOURCES ${SRC_FILES} PARENT_SCOPE) \ No newline at end of file diff --git a/hailort/hrpc/client.cpp b/hailort/hrpc/client.cpp new file mode 100644 index 0000000..5b3bc81 --- /dev/null +++ b/hailort/hrpc/client.cpp @@ -0,0 +1,125 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file client.hpp + * @brief RPC Client + **/ + +#include "client.hpp" + +using namespace hrpc; + +Expected> ResultEvent::create_shared() +{ + TRY(auto event, hailort::Event::create_shared(hailort::Event::State::not_signalled)); + auto ptr = make_shared_nothrow(event); + CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY); + return ptr; +} + +ResultEvent::ResultEvent(EventPtr event) : + m_event(event) +{ +} + +Buffer &&ResultEvent::release() +{ + return std::move(m_value); +} + +hailo_status ResultEvent::signal(Buffer &&value) +{ + m_value = std::move(value); + return m_event->signal(); +} + +hailo_status ResultEvent::wait(std::chrono::milliseconds timeout) +{ + return m_event->wait(timeout); +} + +Client::~Client() +{ + is_running = false; + (void)m_connection.close(); + if (m_thread.joinable()) { + m_thread.join(); + } +} + +hailo_status Client::connect() +{ + TRY(m_conn_context, ConnectionContext::create_shared(false)); + TRY(auto conn, RawConnection::create_shared(m_conn_context)); + auto status = conn->connect(); + CHECK_SUCCESS(status); + + m_connection = RpcConnection(conn); + m_thread = std::thread([this] { + auto status = message_loop(); + if ((status != HAILO_SUCCESS) && (status != HAILO_COMMUNICATION_CLOSED)) { // TODO: Use this to prevent future requests + LOGGER__ERROR("Error in message loop - {}", status); + } + }); + return HAILO_SUCCESS; +} + +hailo_status Client::message_loop() +{ + while (is_running) { + rpc_message_header_t header; + TRY_WITH_ACCEPTABLE_STATUS(HAILO_COMMUNICATION_CLOSED, auto message, m_connection.read_message(header)); + + assert(header.action_id < static_cast(HailoRpcActionID::MAX_VALUE)); + auto action_id_enum = static_cast(header.action_id); + if (m_custom_callbacks.find(action_id_enum) != m_custom_callbacks.end()) { + auto status = m_custom_callbacks[action_id_enum](MemoryView(message), m_connection); + CHECK_SUCCESS(status); + continue; + } + + std::unique_lock lock(m_message_mutex); + auto event = m_events[header.message_id]; + lock.unlock(); + auto status = event->signal(std::move(message)); + CHECK_SUCCESS(status); + } + + return HAILO_SUCCESS; +} + +Expected Client::execute_request(HailoRpcActionID action_id, const MemoryView &request, + std::function write_buffers_callback) +{ + std::unique_lock lock(m_message_mutex); + rpc_message_header_t header; + header.size = static_cast(request.size()); + header.message_id = m_messages_sent++; + header.action_id = static_cast(action_id); + + auto status = m_connection.write_message(header, request); + CHECK_SUCCESS_AS_EXPECTED(status); + if (write_buffers_callback) { + status = write_buffers_callback(m_connection); + CHECK_SUCCESS_AS_EXPECTED(status); + } + + TRY(auto event, ResultEvent::create_shared()); + m_events[header.message_id] = event; + + lock.unlock(); + status = event->wait(REQUEST_TIMEOUT); + CHECK_SUCCESS_AS_EXPECTED(status); + + m_events.erase(header.message_id); + return event->release(); +} + + +void Client::register_custom_reply(HailoRpcActionID action_id, + std::function callback) +{ + m_custom_callbacks[action_id] = callback; +} \ No newline at end of file diff --git a/hailort/hrpc/client.hpp b/hailort/hrpc/client.hpp new file mode 100644 index 0000000..ef53132 --- /dev/null +++ b/hailort/hrpc/client.hpp @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file client.hpp + * @brief RPC Client Header + **/ + +#ifndef _CLIENT_HPP_ +#define _CLIENT_HPP_ + +#include +#include +#include +#include + +#include "rpc_connection.hpp" +#include "hrpc_protocol/serializer.hpp" + + +namespace hrpc +{ + +#define REQUEST_TIMEOUT std::chrono::milliseconds(10000) + +class ResultEvent +{ +public: + static Expected> create_shared(); + ResultEvent(EventPtr event); + + Buffer &&release(); + hailo_status signal(Buffer &&value); + hailo_status wait(std::chrono::milliseconds timeout); + +private: + Buffer m_value; + EventPtr m_event; +}; + +class Client +{ +public: + Client() = default; + ~Client(); + + hailo_status connect(); + Expected execute_request(HailoRpcActionID action_id, const MemoryView &request, + std::function write_buffers_callback = nullptr); + void register_custom_reply(HailoRpcActionID action_id, std::function callback); + +protected: + hailo_status message_loop(); + + bool is_running = true; + std::shared_ptr m_conn_context; + RpcConnection m_connection; + std::thread m_thread; + std::unordered_map> m_events; + std::unordered_map> m_custom_callbacks; + uint32_t m_messages_sent = 0; + std::mutex m_message_mutex; +}; + +} // namespace hrpc + +#endif // _CLIENT_HPP_ \ No newline at end of file diff --git a/hailort/hrpc/os/pcie/raw_connection_internal.cpp b/hailort/hrpc/os/pcie/raw_connection_internal.cpp new file mode 100644 index 0000000..d2dd46a --- /dev/null +++ b/hailort/hrpc/os/pcie/raw_connection_internal.cpp @@ -0,0 +1,196 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file raw_connection_internal.cpp + * @brief PCIE Raw Connection + **/ + +#include "hrpc/os/pcie/raw_connection_internal.hpp" +#include "common/logger_macros.hpp" +#include "common/utils.hpp" +#include "hailo/hailort.h" +#include "vdma/driver/hailort_driver.hpp" + +// TODO: Remove this after we can choose ports in the driver +#define PCIE_PORT (1213355091) + +using namespace hrpc; + +Expected> PcieConnectionContext::create_shared(bool is_accepting) +{ + const auto max_size = PcieSession::max_transfer_size(); + TRY(auto write_buffer, Buffer::create(static_cast(max_size), BufferStorageParams::create_dma())); + TRY(auto read_buffer, Buffer::create(static_cast(max_size), BufferStorageParams::create_dma())); + + std::shared_ptr ptr = nullptr; + if (is_accepting) { + // Server side + TRY(auto driver, HailoRTDriver::create_pcie_ep()); + ptr = make_shared_nothrow(std::move(driver), is_accepting, + std::move(write_buffer), std::move(read_buffer)); + CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY); + return std::dynamic_pointer_cast(ptr); + } else { + // Client side + TRY(auto device_infos, HailoRTDriver::scan_devices()); + CHECK(device_infos.size() > 0, HAILO_NOT_FOUND, "No devices found"); + for (auto &device_info : device_infos) { + if (HailoRTDriver::AcceleratorType::SOC_ACCELERATOR == device_info.accelerator_type) { + TRY(auto driver, HailoRTDriver::create(device_info.device_id, device_info.dev_path)); + ptr = make_shared_nothrow(std::move(driver), is_accepting, + std::move(write_buffer), std::move(read_buffer)); + CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY); + return std::dynamic_pointer_cast(ptr); + } + } + } + LOGGER__ERROR("No suitable device found"); + return make_unexpected(HAILO_NOT_FOUND); +} + +hailo_status PcieConnectionContext::wait_for_available_connection() +{ + std::unique_lock lock(m_mutex); + bool was_successful = m_cv.wait_for(lock, std::chrono::milliseconds(HAILO_INFINITE), [this] () -> bool { + return (m_conn_count == 0); + }); + CHECK(was_successful, HAILO_TIMEOUT, "Got timeout in accept"); + + m_conn_count++; + return HAILO_SUCCESS; +} + +void PcieConnectionContext::mark_connection_closed() +{ + { + std::unique_lock lock(m_mutex); + m_conn_count--; + } + m_cv.notify_one(); +} + +Expected> PcieRawConnection::create_shared(std::shared_ptr context) +{ + auto ptr = make_shared_nothrow(context); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + return std::dynamic_pointer_cast(ptr); +} + +Expected> PcieRawConnection::accept() +{ + auto status = m_context->wait_for_available_connection(); + CHECK_SUCCESS(status); + + auto new_conn = make_shared_nothrow(m_context); + CHECK_NOT_NULL_AS_EXPECTED(new_conn, HAILO_OUT_OF_HOST_MEMORY); + + TRY(auto session, PcieSession::accept(m_context->driver(), PCIE_PORT)); + status = new_conn->set_session(std::move(session)); + CHECK_SUCCESS(status); + + return std::dynamic_pointer_cast(new_conn); +} + +hailo_status PcieRawConnection::set_session(PcieSession &&session) +{ + m_session = make_shared_nothrow(std::move(session)); + CHECK_NOT_NULL(m_session, HAILO_OUT_OF_HOST_MEMORY); + + return HAILO_SUCCESS; +} + +hailo_status PcieRawConnection::connect() +{ + TRY(auto session, PcieSession::connect(m_context->driver(), PCIE_PORT)); + auto status = set_session(std::move(session)); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +hailo_status PcieRawConnection::write(const uint8_t *buffer, size_t size) +{ + if (0 == size) { + return HAILO_SUCCESS; + } + + const auto alignment = OsUtils::get_dma_able_alignment(); + const auto max_size = PcieSession::max_transfer_size(); + bool is_aligned = ((reinterpret_cast(buffer) % alignment )== 0); + + size_t bytes_written = 0; + while (bytes_written < size) { + size_t amount_to_write = 0; + auto size_left = size - bytes_written; + if (is_aligned) { + amount_to_write = std::min(static_cast(size_left), static_cast(max_size)); + auto status = m_session->write(buffer + bytes_written, amount_to_write, m_timeout); + if (HAILO_STREAM_ABORT == status) { + return HAILO_COMMUNICATION_CLOSED; + } + CHECK_SUCCESS(status); + } else { + amount_to_write = std::min(static_cast(size_left), m_context->write_buffer().size()); + memcpy(m_context->write_buffer().data(), buffer + bytes_written, amount_to_write); + auto status = m_session->write(m_context->write_buffer().data(), amount_to_write, m_timeout); + if (HAILO_STREAM_ABORT == status) { + return HAILO_COMMUNICATION_CLOSED; + } + CHECK_SUCCESS(status); + } + + bytes_written += amount_to_write; + } + + return HAILO_SUCCESS; +} + +hailo_status PcieRawConnection::read(uint8_t *buffer, size_t size) +{ + if (0 == size) { + return HAILO_SUCCESS; + } + + const auto alignment = OsUtils::get_dma_able_alignment(); + const auto max_size = PcieSession::max_transfer_size(); + bool is_aligned = ((reinterpret_cast(buffer) % alignment) == 0); + + size_t bytes_read = 0; + while (bytes_read < size) { + size_t amount_to_read = 0; + auto size_left = size - bytes_read; + if (is_aligned) { + amount_to_read = std::min(static_cast(size_left), static_cast(max_size)); + auto status = m_session->read(buffer + bytes_read, amount_to_read, m_timeout); + if (HAILO_STREAM_ABORT == status) { + return HAILO_COMMUNICATION_CLOSED; + } + CHECK_SUCCESS(status); + } else { + amount_to_read = std::min(static_cast(size_left), m_context->read_buffer().size()); + auto status = m_session->read(m_context->read_buffer().data(), amount_to_read, m_timeout); + if (HAILO_STREAM_ABORT == status) { + return HAILO_COMMUNICATION_CLOSED; + } + CHECK_SUCCESS(status); + + memcpy(buffer + bytes_read, m_context->read_buffer().data(), amount_to_read); + } + + bytes_read += amount_to_read; + } + + return HAILO_SUCCESS; +} + +hailo_status PcieRawConnection::close() +{ + auto status = m_session->close(); + CHECK_SUCCESS(status); + + m_context->mark_connection_closed(); + + return HAILO_SUCCESS; +} \ No newline at end of file diff --git a/hailort/hrpc/os/pcie/raw_connection_internal.hpp b/hailort/hrpc/os/pcie/raw_connection_internal.hpp new file mode 100644 index 0000000..2753f9e --- /dev/null +++ b/hailort/hrpc/os/pcie/raw_connection_internal.hpp @@ -0,0 +1,78 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file raw_connection_internal.hpp + * @brief Raw Connection Header for pcie based comunication + **/ + +#ifndef _PCIE_RAW_CONNECTION_INTERNAL_HPP_ +#define _PCIE_RAW_CONNECTION_INTERNAL_HPP_ + +#include "hailo/expected.hpp" +#include "vdma/pcie_session.hpp" +#include "hrpc/raw_connection.hpp" + +#include +#include + +using namespace hailort; + +namespace hrpc +{ + +class PcieConnectionContext : public ConnectionContext +{ +public: + static Expected> create_shared(bool is_accepting); + + PcieConnectionContext(std::shared_ptr &&driver, bool is_accepting, + Buffer &&write_buffer, Buffer &&read_buffer) + : ConnectionContext(is_accepting), m_driver(std::move(driver)), + m_write_buffer(std::move(write_buffer)), m_read_buffer(std::move(read_buffer)), + m_conn_count(0) {} + + virtual ~PcieConnectionContext() = default; + + std::shared_ptr driver() { return m_driver; } + Buffer &write_buffer() { return m_write_buffer; } + Buffer &read_buffer() { return m_read_buffer; } + + hailo_status wait_for_available_connection(); + void mark_connection_closed(); + +private: + std::shared_ptr m_driver; + Buffer m_write_buffer; + Buffer m_read_buffer; + uint32_t m_conn_count; + std::mutex m_mutex; + std::condition_variable m_cv; +}; + +class PcieRawConnection : public RawConnection +{ +public: + static Expected> create_shared(std::shared_ptr context); + + PcieRawConnection() = default; + virtual ~PcieRawConnection() = default; + + virtual Expected> accept() override; + virtual hailo_status connect() override; + virtual hailo_status write(const uint8_t *buffer, size_t size) override; + virtual hailo_status read(uint8_t *buffer, size_t size) override; + virtual hailo_status close() override; + + explicit PcieRawConnection(std::shared_ptr context) : m_context(context) {} +private: + hailo_status set_session(PcieSession &&session); + + std::shared_ptr m_context; + std::shared_ptr m_session; +}; + +} // namespace hrpc + +#endif // _PCIE_RAW_CONNECTION_INTERNAL_HPP_ \ No newline at end of file diff --git a/hailort/hrpc/os/posix/raw_connection_internal.cpp b/hailort/hrpc/os/posix/raw_connection_internal.cpp new file mode 100644 index 0000000..752c3af --- /dev/null +++ b/hailort/hrpc/os/posix/raw_connection_internal.cpp @@ -0,0 +1,122 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file raw_connection_internal.cpp + * @brief Linux Sockets Raw Connection + **/ + +#include "hrpc/os/posix/raw_connection_internal.hpp" + +#include +#include +#include +#include +#include +#include +#include + +using namespace hrpc; + +Expected> OsConnectionContext::create_shared(bool is_accepting) +{ + auto ptr = make_shared_nothrow(is_accepting); + CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return std::dynamic_pointer_cast(ptr); +} + +Expected> OsRawConnection::create_shared(std::shared_ptr context) +{ + std::shared_ptr ptr; + if (context->is_accepting()) { + int fd = ::socket(AF_UNIX, SOCK_STREAM, 0); + CHECK_AS_EXPECTED(fd >= 0, HAILO_OPEN_FILE_FAILURE, "Socket creation error, errno = {}", errno); + + struct sockaddr_un server_addr; + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sun_family = AF_UNIX; + std::string addr = "/tmp/unix_socket"; + strncpy(server_addr.sun_path, addr.c_str(), addr.size()); + + unlink(addr.c_str()); + int result = ::bind(fd, (struct sockaddr*)&server_addr, sizeof(server_addr)); + CHECK_AS_EXPECTED(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Bind error, errno = {}", errno); + + result = ::listen(fd, 5); + CHECK_AS_EXPECTED(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Listen error, errno = {}", errno); + + ptr = make_shared_nothrow(fd, context); + } else { + + int fd = ::socket(AF_UNIX, SOCK_STREAM, 0); + CHECK_AS_EXPECTED(fd >= 0, HAILO_OPEN_FILE_FAILURE, "Socket creation error, errno = {}", errno); + ptr = make_shared_nothrow(fd, context); + } + + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + return ptr; +} + +Expected> OsRawConnection::accept() +{ + int fd = ::accept(m_fd, nullptr, nullptr); + CHECK_AS_EXPECTED(fd >= 0, HAILO_FILE_OPERATION_FAILURE, "Accept error, errno = {}", errno); + + std::shared_ptr ptr = make_shared_nothrow(fd, m_context); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +hailo_status OsRawConnection::connect() +{ + struct sockaddr_un server_addr; + std::string addr = "/tmp/unix_socket"; + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sun_family = AF_UNIX; + strncpy(server_addr.sun_path, addr.c_str(), addr.size()); + + int result = ::connect(m_fd, (struct sockaddr*)&server_addr, sizeof(server_addr)); + CHECK(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Connect error, errno = {}", errno); + + return HAILO_SUCCESS; +} + +hailo_status OsRawConnection::write(const uint8_t *buffer, size_t size) +{ + size_t bytes_written = 0; + while (bytes_written < size) { + ssize_t result = ::send(m_fd, buffer + bytes_written, size - bytes_written, MSG_NOSIGNAL); + CHECK(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Write error, errno = {}", errno); + bytes_written += result; + } + return HAILO_SUCCESS; +} + +hailo_status OsRawConnection::read(uint8_t *buffer, size_t size) +{ + size_t bytes_read = 0; + while (bytes_read < size) { + ssize_t result = ::read(m_fd, buffer + bytes_read, size - bytes_read); + if (0 == result) { + return HAILO_COMMUNICATION_CLOSED; // 0 means the communication is closed + } + CHECK(result >= 0, HAILO_FILE_OPERATION_FAILURE, "Read error, errno = {}", errno); + bytes_read += result; + } + return HAILO_SUCCESS; +} + +hailo_status OsRawConnection::close() +{ + int result = ::shutdown(m_fd, SHUT_RDWR); + CHECK(0 == result, HAILO_CLOSE_FAILURE, "Socket shutdown failed, errno = {}", errno); + + result = ::close(m_fd); + CHECK(0 == result, HAILO_CLOSE_FAILURE, "Socket close failed, errno = {}", errno); + + return HAILO_SUCCESS; +} \ No newline at end of file diff --git a/hailort/hrpc/os/posix/raw_connection_internal.hpp b/hailort/hrpc/os/posix/raw_connection_internal.hpp new file mode 100644 index 0000000..cbde53f --- /dev/null +++ b/hailort/hrpc/os/posix/raw_connection_internal.hpp @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file raw_connection_internal.hpp + * @brief Raw Connection Header for sockets based comunication + **/ + +#ifndef _POSIX_RAW_CONNECTION_INTERNAL_HPP_ +#define _POSIX_RAW_CONNECTION_INTERNAL_HPP_ + +#include "hailo/expected.hpp" +#include "hrpc/raw_connection.hpp" + +#include + +using namespace hailort; + +namespace hrpc +{ + +class OsConnectionContext : public ConnectionContext +{ +public: + static Expected> create_shared(bool is_accepting); + + OsConnectionContext(bool is_accepting) : ConnectionContext(is_accepting) {} + + virtual ~OsConnectionContext() = default; +}; + +class OsRawConnection : public RawConnection +{ +public: + static Expected> create_shared(std::shared_ptr context); + + virtual ~OsRawConnection() = default; + + virtual Expected> accept() override; + virtual hailo_status connect() override; + virtual hailo_status write(const uint8_t *buffer, size_t size) override; + virtual hailo_status read(uint8_t *buffer, size_t size) override; + virtual hailo_status close() override; + + OsRawConnection(int fd, std::shared_ptr context) : m_fd(fd), m_context(context) {} +private: + int m_fd; + std::shared_ptr m_context; +}; + +} // namespace hrpc + +#endif // _POSIX_RAW_CONNECTION_INTERNAL_HPP_ \ No newline at end of file diff --git a/hailort/hrpc/os/windows/raw_connection_internal.cpp b/hailort/hrpc/os/windows/raw_connection_internal.cpp new file mode 100644 index 0000000..cfb17a5 --- /dev/null +++ b/hailort/hrpc/os/windows/raw_connection_internal.cpp @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file raw_connection_internal.cpp + * @brief Windows Sockets Raw Connection + **/ + +#include "hrpc/os/windows/raw_connection_internal.hpp" + +#include "common/logger_macros.hpp" +#include "common/utils.hpp" +#include "hailo/hailort.h" + +using namespace hrpc; + +Expected> OsConnectionContext::create_shared(bool is_accepting) +{ + (void)is_accepting; + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +Expected> OsRawConnection::create_shared(std::shared_ptr context) +{ + (void)context; + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +Expected> OsRawConnection::accept() +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +hailo_status OsRawConnection::connect() +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status OsRawConnection::write(const uint8_t *buffer, size_t size) +{ + (void)buffer; + (void)size; + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status OsRawConnection::read(uint8_t *buffer, size_t size) +{ + (void)buffer; + (void)size; + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status OsRawConnection::close() +{ + return HAILO_NOT_IMPLEMENTED; +} \ No newline at end of file diff --git a/hailort/hrpc/os/windows/raw_connection_internal.hpp b/hailort/hrpc/os/windows/raw_connection_internal.hpp new file mode 100644 index 0000000..3b66717 --- /dev/null +++ b/hailort/hrpc/os/windows/raw_connection_internal.hpp @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file raw_connection_internal.hpp + * @brief Raw Connection Header for sockets based comunication + **/ + +#ifndef _WINDOWS_RAW_CONNECTION_INTERNAL_HPP_ +#define _WINDOWS_RAW_CONNECTION_INTERNAL_HPP_ + +#include "hailo/expected.hpp" +#include "hrpc/raw_connection.hpp" + +#include + +using namespace hailort; + +namespace hrpc +{ + +class OsConnectionContext : public ConnectionContext +{ +public: + static Expected> create_shared(bool is_accepting); +}; + +class OsRawConnection : public RawConnection +{ +public: + static Expected> create_shared(std::shared_ptr context); + + OsRawConnection() = default; + virtual ~OsRawConnection() = default; + + virtual Expected> accept() override; + virtual hailo_status connect() override; + virtual hailo_status write(const uint8_t *buffer, size_t size) override; + virtual hailo_status read(uint8_t *buffer, size_t size) override; + virtual hailo_status close() override; + + explicit OsRawConnection(std::shared_ptr /*context*/) {} +}; + +} // namespace hrpc + +#endif // _WINDOWS_RAW_CONNECTION_INTERNAL_HPP_ \ No newline at end of file diff --git a/hailort/hrpc/raw_connection.cpp b/hailort/hrpc/raw_connection.cpp new file mode 100644 index 0000000..cdc5caf --- /dev/null +++ b/hailort/hrpc/raw_connection.cpp @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file raw_connection.cpp + * @brief Raw Connection + **/ + +#include "hailo/vdevice.hpp" +#include "hrpc/raw_connection.hpp" +#include "hrpc/os/pcie/raw_connection_internal.hpp" + +#ifdef _WIN32 +#include "hrpc/os/windows/raw_connection_internal.hpp" +#else +#include "hrpc/os/posix/raw_connection_internal.hpp" +#endif + +#define HAILO_FORCE_SOCKET_COM_ENV_VAR "HAILO_FORCE_SOCKET_COM" + +using namespace hrpc; + + +Expected> ConnectionContext::create_shared(bool is_accepting) +{ + // The env var HAILO_FORCE_HRPC_CLIENT_ENV_VAR is supported for debug purposes + char *socket_com = std::getenv(HAILO_FORCE_SOCKET_COM_ENV_VAR); // TODO: Remove duplication + auto force_socket_com = (nullptr != socket_com) && ("1" == std::string(socket_com)); + + if (force_socket_com || VDevice::force_hrpc_client()) {// If forcing hrpc service, its because we work without EP driver -> use sockets + return OsConnectionContext::create_shared(is_accepting); + } else { + return PcieConnectionContext::create_shared(is_accepting); + } +} + +Expected> RawConnection::create_shared(std::shared_ptr context) +{ + // Create according to ConnectionContext type + auto os_connection_context = std::dynamic_pointer_cast(context); + if (os_connection_context != nullptr) { + return OsRawConnection::create_shared(os_connection_context); + } else { + return PcieRawConnection::create_shared(std::dynamic_pointer_cast(context)); + } +} \ No newline at end of file diff --git a/hailort/hrpc/raw_connection.hpp b/hailort/hrpc/raw_connection.hpp new file mode 100644 index 0000000..d7e1218 --- /dev/null +++ b/hailort/hrpc/raw_connection.hpp @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file raw_connection.hpp + * @brief Raw Connection Header + **/ + +#ifndef _RAW_CONNECTION_HPP_ +#define _RAW_CONNECTION_HPP_ + +#include "hailo/expected.hpp" +#include "vdma/pcie_session.hpp" + +#include + +using namespace hailort; + +namespace hrpc +{ + +class ConnectionContext +{ +public: + static Expected> create_shared(bool is_accepting); + + bool is_accepting() const { return m_is_accepting; } + + ConnectionContext(bool is_accepting) : m_is_accepting(is_accepting) {} + virtual ~ConnectionContext() = default; + +protected: + bool m_is_accepting; +}; + + +class RawConnection +{ +public: + static Expected> create_shared(std::shared_ptr context); + + RawConnection() = default; + virtual ~RawConnection() = default; + + virtual Expected> accept() = 0; + virtual hailo_status connect() = 0; + virtual hailo_status write(const uint8_t *buffer, size_t size) = 0; + virtual hailo_status read(uint8_t *buffer, size_t size) = 0; + virtual hailo_status close() = 0; + +protected: + std::chrono::milliseconds m_timeout = std::chrono::milliseconds(HAILO_INFINITE); +}; + +} // namespace hrpc + +#endif // _RAW_CONNECTION_HPP_ \ No newline at end of file diff --git a/hailort/hrpc/rpc_connection.cpp b/hailort/hrpc/rpc_connection.cpp new file mode 100644 index 0000000..69d22bc --- /dev/null +++ b/hailort/hrpc/rpc_connection.cpp @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file rpc_connection.cpp + * @brief RPC connection implementation + **/ + +#include "rpc_connection.hpp" + +namespace hrpc +{ + +hailo_status RpcConnection::write_message(const rpc_message_header_t &header, const MemoryView &buffer) { + auto header_with_magic = header; + header_with_magic.magic = RPC_MESSAGE_MAGIC; + auto status = m_raw->write(reinterpret_cast(&header_with_magic), sizeof(header_with_magic)); + if (HAILO_COMMUNICATION_CLOSED == status) { + return make_unexpected(status); + } + CHECK_SUCCESS(status); + + status = m_raw->write(buffer.data(), header.size); + if (HAILO_COMMUNICATION_CLOSED == status) { + return make_unexpected(status); + } + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +Expected RpcConnection::read_message(rpc_message_header_t &header) { + auto status = m_raw->read(reinterpret_cast(&header), sizeof(header)); + if (HAILO_COMMUNICATION_CLOSED == status) { + return make_unexpected(status); + } + CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_AS_EXPECTED(RPC_MESSAGE_MAGIC == header.magic, HAILO_INTERNAL_FAILURE, "Invalid magic! {} != {}", + header.magic, RPC_MESSAGE_MAGIC); + + TRY(auto buffer, Buffer::create(header.size, BufferStorageParams::create_dma())); + status = m_raw->read(buffer.data(), header.size); + if (HAILO_COMMUNICATION_CLOSED == status) { + return make_unexpected(status); + } + CHECK_SUCCESS_AS_EXPECTED(status); + + return buffer; +} + +hailo_status RpcConnection::write_buffer(const MemoryView &buffer) +{ + auto status = m_raw->write(buffer.data(), buffer.size()); + if (HAILO_COMMUNICATION_CLOSED == status) { + return make_unexpected(status); + } + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +hailo_status RpcConnection::read_buffer(MemoryView buffer) +{ + auto status = m_raw->read(buffer.data(), buffer.size()); + if (HAILO_COMMUNICATION_CLOSED == status) { + return make_unexpected(status); + } + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +hailo_status RpcConnection::close() +{ + if (m_raw) { + return m_raw->close(); + } + return HAILO_SUCCESS; +} + +} // namespace hrpc \ No newline at end of file diff --git a/hailort/hrpc/rpc_connection.hpp b/hailort/hrpc/rpc_connection.hpp new file mode 100644 index 0000000..d34e03c --- /dev/null +++ b/hailort/hrpc/rpc_connection.hpp @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file rpc_connection.hpp + * @brief RPC Connection Header + **/ + +#ifndef _RPC_CONNECTION_HPP_ +#define _RPC_CONNECTION_HPP_ + +#include "raw_connection.hpp" + +#include "hailo/buffer.hpp" +#include "common/utils.hpp" + +#define RPC_MESSAGE_MAGIC (0x8A554432) + + +namespace hrpc +{ + +#pragma pack(push, 1) +struct rpc_message_header_t +{ + uint32_t magic; // TODO: consider removing. check if hurts performance + uint32_t size; + uint32_t message_id; + uint32_t action_id; +}; +#pragma pack(pop) + +class RpcConnection +{ +public: + RpcConnection() = default; + explicit RpcConnection(std::shared_ptr raw) : m_raw(raw) {} + + hailo_status write_message(const rpc_message_header_t &header, const MemoryView &buffer); + Expected read_message(rpc_message_header_t &header); + + hailo_status write_buffer(const MemoryView &buffer); + hailo_status read_buffer(MemoryView buffer); + + hailo_status close(); + +private: + std::shared_ptr m_raw; +}; + +} // namespace hrpc + +#endif // _RPC_CONNECTION_HPP_ \ No newline at end of file diff --git a/hailort/hrpc/server.cpp b/hailort/hrpc/server.cpp new file mode 100644 index 0000000..304c41c --- /dev/null +++ b/hailort/hrpc/server.cpp @@ -0,0 +1,123 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file server.cpp + * @brief RPC Server + **/ + +#include "server.hpp" + +namespace hrpc +{ + +ServerContext::ServerContext(Server &server, RpcConnection connection) : + m_server(server), m_connection(connection) {} + +hailo_status ServerContext::trigger_callback(uint32_t callback_id, hailo_status callback_status, std::function write_buffers_callback) +{ + return m_server.trigger_callback(callback_id, m_connection, callback_status, write_buffers_callback); +} + +RpcConnection &ServerContext::connection() +{ + return m_connection; +} + +void Dispatcher::register_action(HailoRpcActionID action_id, + std::function(const MemoryView&, ServerContextPtr)> action) +{ + m_actions[action_id] = action; +} + +Expected Dispatcher::call_action(HailoRpcActionID action_id, const MemoryView &request, ServerContextPtr server_context) +{ + if (m_actions.find(action_id) != m_actions.end()) { + return m_actions[action_id](request, server_context); + } + LOGGER__ERROR("Failed to find RPC action {}", action_id); + return make_unexpected(HAILO_RPC_FAILED); +} + +hailo_status Server::serve() +{ + while (true) { + TRY(auto client_connection, create_client_connection()); + auto th = std::thread([this, client_connection]() { serve_client(client_connection); }); + th.detach(); + } + return HAILO_SUCCESS; +} + +void Server::set_dispatcher(Dispatcher dispatcher) +{ + m_dispatcher = dispatcher; +} + +Expected Server::create_client_connection() +{ + TRY(auto server_connection, RawConnection::create_shared(m_connection_context)); + TRY(auto conn, server_connection->accept()); + + return RpcConnection(conn); +} + +hailo_status Server::serve_client(RpcConnection client_connection) +{ + auto server_context = make_shared_nothrow(*this, client_connection); + CHECK_NOT_NULL(server_context, HAILO_OUT_OF_HOST_MEMORY); + while (true) { + rpc_message_header_t header; + auto request = client_connection.read_message(header); + if (HAILO_COMMUNICATION_CLOSED == request.status()) { + cleanup_client_resources(client_connection); + break; // Client EP is disconnected, exit this loop + } + CHECK_EXPECTED_AS_STATUS(request); + + assert(header.action_id < static_cast(HailoRpcActionID::MAX_VALUE)); + TRY(auto reply, m_dispatcher.call_action(static_cast(header.action_id), MemoryView(*request), server_context)); + { + std::unique_lock lock(m_write_mutex); + header.size = static_cast(reply.size()); + + auto status = client_connection.write_message(header, MemoryView(reply)); + if ((HAILO_COMMUNICATION_CLOSED == status) || (HAILO_FILE_OPERATION_FAILURE == status)) { + lock.unlock(); // We need to acquire this lock when releasing the client resources (trigger cb) + cleanup_client_resources(client_connection); + break; // Client EP is disconnected, exit this loop + } + CHECK_SUCCESS(status); + } + } + + return HAILO_SUCCESS; +} + +hailo_status Server::trigger_callback(uint32_t callback_id, RpcConnection connection, hailo_status callback_status, + std::function write_buffers_callback) +{ + TRY(auto reply, CallbackCalledSerializer::serialize_reply(callback_status, callback_id)); + + std::unique_lock lock(m_write_mutex); + rpc_message_header_t header; + header.action_id = static_cast(HailoRpcActionID::CALLBACK_CALLED); + header.message_id = callback_id; + header.size = static_cast(reply.size()); + + auto status = connection.write_message(header, MemoryView(reply)); + if ((HAILO_COMMUNICATION_CLOSED == status) || (HAILO_FILE_OPERATION_FAILURE == status)) { + return status; + } + CHECK_SUCCESS(status); + + if (write_buffers_callback) { + status = write_buffers_callback(connection); + CHECK_SUCCESS(status); + } + + return HAILO_SUCCESS; +} + +} // namespace hrpc diff --git a/hailort/hrpc/server.hpp b/hailort/hrpc/server.hpp new file mode 100644 index 0000000..45d5245 --- /dev/null +++ b/hailort/hrpc/server.hpp @@ -0,0 +1,78 @@ +#ifndef _SERVER_HPP_ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file server.hpp + * @brief RPC Server Header + **/ + +#define _SERVER_HPP_ + +#include +#include + +#include "rpc_connection.hpp" +#include "hailort_service/service_resource_manager.hpp" +#include "hrpc_protocol/serializer.hpp" + + +namespace hrpc +{ + +class Server; +class ServerContext +{ +public: + ServerContext(Server &server, RpcConnection connection); + hailo_status trigger_callback(uint32_t callback_id, hailo_status callback_status, + std::function write_buffers_callback = nullptr); + RpcConnection &connection(); + +private: + Server &m_server; + RpcConnection m_connection; +}; +using ServerContextPtr = std::shared_ptr; + +class Dispatcher +{ +public: + Dispatcher() = default; + + void register_action(HailoRpcActionID action_id, + std::function(const MemoryView&, ServerContextPtr)> action); + Expected call_action(HailoRpcActionID action_id, const MemoryView &request, ServerContextPtr server_context); + +private: + std::unordered_map(const MemoryView&, ServerContextPtr)>> m_actions; +}; + +class Server +{ +public: + Server(std::shared_ptr connection_context) : m_connection_context(connection_context) {}; + virtual ~Server() = default; + + hailo_status serve(); + + void set_dispatcher(Dispatcher dispatcher); + + friend class ServerContext; +protected: + std::shared_ptr m_connection_context; +private: + Expected create_client_connection(); + hailo_status serve_client(RpcConnection client_connection); + hailo_status trigger_callback(uint32_t callback_id, RpcConnection connection, hailo_status callback_status, + std::function write_buffers_callback = nullptr); + virtual hailo_status cleanup_client_resources(RpcConnection client_connection) = 0; + + Dispatcher m_dispatcher; + std::mutex m_write_mutex; +}; + +} // namespace hrpc + +#endif // _SERVER_HPP_ \ No newline at end of file diff --git a/hailort/hrpc_protocol/CMakeLists.txt b/hailort/hrpc_protocol/CMakeLists.txt new file mode 100644 index 0000000..d6b925a --- /dev/null +++ b/hailort/hrpc_protocol/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.0.0) + +protobuf_generate_cpp(PROTO_RPC_SRC PROTO_RPC_HEADER rpc.proto) +get_filename_component(PROTO_HEADER_DIRECTORY ${PROTO_RPC_HEADER} DIRECTORY) + +add_library(rpc_proto STATIC EXCLUDE_FROM_ALL ${PROTO_RPC_SRC} ${PROTO_RPC_HEADER}) +target_link_libraries(rpc_proto libprotobuf-lite) +set_target_properties(rpc_proto PROPERTIES CXX_STANDARD 14 GENERATED TRUE POSITION_INDEPENDENT_CODE ON) +if(CMAKE_HOST_WIN32) + # https://github.com/protocolbuffers/protobuf/tree/master/cmake#notes-on-compiler-warnings + target_compile_options(rpc_proto PRIVATE /wd4244) +endif() +get_filename_component(PROTO_HEADER_DIRECTORY ${PROTO_RPC_HEADER} DIRECTORY) +target_include_directories(rpc_proto + PUBLIC + $ + $ +) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/serializer.cpp +) + +set(HRPC_PROTOCOL_CPP_SOURCES ${SRC_FILES} PARENT_SCOPE) \ No newline at end of file diff --git a/hailort/hrpc_protocol/rpc.proto b/hailort/hrpc_protocol/rpc.proto new file mode 100644 index 0000000..d99bd5c --- /dev/null +++ b/hailort/hrpc_protocol/rpc.proto @@ -0,0 +1,207 @@ +syntax = "proto3"; + +option optimize_for = LITE_RUNTIME; + +message RpcRequest { + oneof request { + VDevice_Create_Request create_vdevice_request = 1; + VDevice_Destroy_Request destroy_vdevice_request = 2; + VDevice_CreateInferModel_Request create_infer_model_request = 3; + + InferModel_Destroy_Request destroy_infer_model_request = 4; + InferModel_CreateConfiguredInferModel_Request create_configured_infer_model_request = 5; + + ConfiguredInferModel_Destroy_Request destroy_configured_infer_model_request = 6; + ConfiguredInferModel_SetSchedulerTimeout_Request set_scheduler_timeout_request = 7; + ConfiguredInferModel_SetSchedulerThreshold_Request set_scheduler_threshold_request = 8; + ConfiguredInferModel_SetSchedulerPriority_Request set_scheduler_priority_request = 9; + ConfiguredInferModel_GetHwLatencyMeasurement_Request get_hw_latency_measurement_request = 10; + ConfiguredInferModel_Activate_Request activate_request = 11; + ConfiguredInferModel_Deactivate_Request deactivate_request = 12; + ConfiguredInferModel_Shutdown_Request shutdown_request = 13; + ConfiguredInferModel_AsyncInfer_Request async_infer_request = 14; + } +} + +message RpcReply { + oneof reply { + VDevice_Create_Reply create_vdevice_reply = 1; + VDevice_Destroy_Reply destroy_vdevice_reply = 2; + VDevice_CreateInferModel_Reply create_infer_model_reply = 3; + + InferModel_Destroy_Reply destroy_infer_model_reply = 4; + InferModel_CreateConfiguredInferModel_Reply create_configured_infer_model_reply = 5; + + ConfiguredInferModel_Destroy_Reply destroy_configured_infer_model_reply = 6; + ConfiguredInferModel_SetSchedulerTimeout_Reply set_scheduler_timeout_reply = 7; + ConfiguredInferModel_SetSchedulerThreshold_Reply set_scheduler_threshold_reply = 8; + ConfiguredInferModel_SetSchedulerPriority_Reply set_scheduler_priority_reply = 9; + ConfiguredInferModel_GetHwLatencyMeasurement_Reply get_hw_latency_measurement_reply = 10; + ConfiguredInferModel_Activate_Reply activate_reply = 11; + ConfiguredInferModel_Deactivate_Reply deactivate_reply = 12; + ConfiguredInferModel_Shutdown_Reply shutdown_reply = 13; + ConfiguredInferModel_AsyncInfer_Reply async_infer_reply = 14; + + CallbackCalled_Reply callback_called_reply = 15; + } +} + +message HailoObjectHandle { + uint32 id = 1; +} + +message HailoCallbackHandle { + uint32 id = 1; +} + +message VDeviceParamsProto { + uint32 scheduling_algorithm = 1; + string group_id = 2; +} + +message VDevice_Create_Request { + VDeviceParamsProto params = 1; +} + +message VDevice_Create_Reply { + uint32 status = 1; + HailoObjectHandle vdevice_handle = 2; +} + +message VDevice_Destroy_Request { + HailoObjectHandle vdevice_handle = 1; +} + +message VDevice_Destroy_Reply { + uint32 status = 1; +} + +message VDevice_CreateInferModel_Request { + HailoObjectHandle vdevice_handle = 1; + uint64 hef_size = 2; + // Protocol note: After this message, server expects to get HEF data (buffer of size 'hef_size') +} + +message VDevice_CreateInferModel_Reply { + uint32 status = 1; + HailoObjectHandle infer_model_handle = 2; +} + +message InferModel_Destroy_Request { + HailoObjectHandle infer_model_handle = 1; +} + +message InferModel_Destroy_Reply { + uint32 status = 1; +} + +message InferStreamParamsProto { + string name = 1; + uint32 format_order = 2; + uint32 format_type = 3; + float nms_score_threshold = 4; + float nms_iou_threshold = 5; + uint32 nms_max_proposals_per_class = 6; + uint32 nms_max_accumulated_mask_size = 7; +}; + +message InferModel_CreateConfiguredInferModel_Request { + HailoObjectHandle infer_model_handle = 1; + HailoObjectHandle vdevice_handle = 2; + repeated InferStreamParamsProto input_infer_streams = 3; + repeated InferStreamParamsProto output_infer_streams = 4; + uint32 batch_size = 5; + uint32 power_mode = 6; + uint32 latency_flag = 7; +} + +message InferModel_CreateConfiguredInferModel_Reply { + uint32 status = 1; + HailoObjectHandle configured_infer_model_handle = 2; + uint32 async_queue_size = 3; +} + +message ConfiguredInferModel_Destroy_Request { + HailoObjectHandle configured_infer_model_handle = 1; +} + +message ConfiguredInferModel_Destroy_Reply { + uint32 status = 1; +} + +message ConfiguredInferModel_SetSchedulerTimeout_Request { + HailoObjectHandle configured_infer_model_handle = 1; + uint32 timeout = 2; +} + +message ConfiguredInferModel_SetSchedulerTimeout_Reply { + uint32 status = 1; +} + +message ConfiguredInferModel_SetSchedulerThreshold_Request { + HailoObjectHandle configured_infer_model_handle = 1; + uint32 threshold = 2; +} + +message ConfiguredInferModel_SetSchedulerThreshold_Reply { + uint32 status = 1; +} + +message ConfiguredInferModel_SetSchedulerPriority_Request { + HailoObjectHandle configured_infer_model_handle = 1; + uint32 priority = 2; +} + +message ConfiguredInferModel_SetSchedulerPriority_Reply { + uint32 status = 1; +} + +message ConfiguredInferModel_GetHwLatencyMeasurement_Request { + HailoObjectHandle configured_infer_model_handle = 1; +} + +message ConfiguredInferModel_GetHwLatencyMeasurement_Reply { + uint32 status = 1; + uint32 avg_hw_latency = 2; +} + +message ConfiguredInferModel_Activate_Request { + HailoObjectHandle configured_infer_model_handle = 1; +} + +message ConfiguredInferModel_Activate_Reply { + uint32 status = 1; +} + +message ConfiguredInferModel_Deactivate_Request { + HailoObjectHandle configured_infer_model_handle = 1; +} + +message ConfiguredInferModel_Deactivate_Reply { + uint32 status = 1; +} + +message ConfiguredInferModel_Shutdown_Request { + HailoObjectHandle configured_infer_model_handle = 1; +} + +message ConfiguredInferModel_Shutdown_Reply { + uint32 status = 1; +} + +message ConfiguredInferModel_AsyncInfer_Request { + HailoObjectHandle configured_infer_model_handle = 1; + HailoObjectHandle infer_model_handle = 2; + HailoCallbackHandle callback_handle = 3; + // Protocol note: After this messgae, server expects to get the input buffers, one after the other, in order +} + +message ConfiguredInferModel_AsyncInfer_Reply { + uint32 status = 1; +} + +message CallbackCalled_Reply { + uint32 status = 1; + HailoCallbackHandle callback_handle = 2; + // Protocol note: After this messgae, and only if status is HAILO_SUCCESS, server expects to get the output buffers, one after the other, in order +} \ No newline at end of file diff --git a/hailort/hrpc_protocol/serializer.cpp b/hailort/hrpc_protocol/serializer.cpp new file mode 100644 index 0000000..5dad755 --- /dev/null +++ b/hailort/hrpc_protocol/serializer.cpp @@ -0,0 +1,834 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file serializer.cpp + * @brief HRPC Serialization implementation + **/ + +#include "serializer.hpp" +#include "hailo/hailort.h" +#include "hailo/hailort_common.hpp" +#include "hailo/hailort_defaults.hpp" +#include "common/utils.hpp" + +// https://github.com/protocolbuffers/protobuf/tree/master/cmake#notes-on-compiler-warnings +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4244 4267 4127) +#else +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include "rpc.pb.h" +#if defined(_MSC_VER) +#pragma warning(pop) +#else +#pragma GCC diagnostic pop +#endif + +namespace hailort +{ + +Expected CreateVDeviceSerializer::serialize_request(const hailo_vdevice_params_t ¶ms) +{ + VDevice_Create_Request request; + + auto proto_params = request.mutable_params(); + proto_params->set_scheduling_algorithm(params.scheduling_algorithm); + proto_params->set_group_id(params.group_id == nullptr ? "" : std::string(params.group_id)); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'CreateVDevice'"); + + return serialized_request; +} + +Expected CreateVDeviceSerializer::deserialize_request(const MemoryView &serialized_request) +{ + VDevice_Create_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'CreateVDevice'"); + + bool multi_process_service_flag = false; + hailo_vdevice_params_t res = { + 1, + nullptr, + static_cast(request.params().scheduling_algorithm()), + request.params().group_id().c_str(), + multi_process_service_flag + }; + + return res; +} + +Expected CreateVDeviceSerializer::serialize_reply(hailo_status status, rpc_object_handle_t vdevice_handle) +{ + VDevice_Create_Reply reply; + + reply.set_status(status); + auto proto_vdevice_handle = reply.mutable_vdevice_handle(); + proto_vdevice_handle->set_id(vdevice_handle); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'CreateVDevice'"); + + return serialized_reply; +} + +Expected> CreateVDeviceSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + VDevice_Create_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'CreateVDevice'"); + + return std::make_tuple(static_cast(reply.status()), reply.vdevice_handle().id()); +} + +Expected DestroyVDeviceSerializer::serialize_request(rpc_object_handle_t vdevice_handle) +{ + VDevice_Destroy_Request request; + + auto proto_vdevice_handle= request.mutable_vdevice_handle(); + proto_vdevice_handle->set_id(vdevice_handle); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'DestroyVDevice'"); + + return serialized_request; +} + +Expected DestroyVDeviceSerializer::deserialize_request(const MemoryView &serialized_request) +{ + VDevice_Destroy_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyVDevice'"); + + return request.vdevice_handle().id(); +} + +Expected DestroyVDeviceSerializer::serialize_reply(hailo_status status) +{ + VDevice_Destroy_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'DestroyVDevice'"); + + return serialized_reply; +} + +hailo_status DestroyVDeviceSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + VDevice_Destroy_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyVDevice'"); + + return static_cast(reply.status()); +} + +Expected CreateInferModelSerializer::serialize_request(rpc_object_handle_t vdevice_handle, uint64_t hef_size) +{ + VDevice_CreateInferModel_Request request; + + auto proto_vdevice_handle = request.mutable_vdevice_handle(); + proto_vdevice_handle->set_id(vdevice_handle); + + request.set_hef_size(hef_size); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'CreateVInferModel'"); + + return serialized_request; +} + +Expected> CreateInferModelSerializer::deserialize_request(const MemoryView &serialized_request) +{ + VDevice_CreateInferModel_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'CreateVInferModel'"); + + return std::make_tuple(request.vdevice_handle().id(), request.hef_size()); +} + +Expected CreateInferModelSerializer::serialize_reply(hailo_status status, rpc_object_handle_t infer_model_handle) +{ + VDevice_CreateInferModel_Reply reply; + + reply.set_status(status); + auto proto_infer_model_handle = reply.mutable_infer_model_handle(); + proto_infer_model_handle->set_id(infer_model_handle); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'CreateVInferModel'"); + + return serialized_reply; +} + +Expected> CreateInferModelSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + VDevice_CreateInferModel_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'CreateVInferModel'"); + + return std::make_tuple(static_cast(reply.status()), reply.infer_model_handle().id()); +} + +Expected DestroyInferModelSerializer::serialize_request(rpc_object_handle_t infer_model_handle) +{ + InferModel_Destroy_Request request; + + auto proto_infer_model_handle = request.mutable_infer_model_handle(); + proto_infer_model_handle->set_id(infer_model_handle); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'DestroyInferModel'"); + + return serialized_request; +} + +Expected DestroyInferModelSerializer::deserialize_request(const MemoryView &serialized_request) +{ + InferModel_Destroy_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyInferModel'"); + + return request.infer_model_handle().id(); +} + +Expected DestroyInferModelSerializer::serialize_reply(hailo_status status) +{ + InferModel_Destroy_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'DestroyInferModel'"); + + return serialized_reply; +} + +hailo_status DestroyInferModelSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + InferModel_Destroy_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyInferModel'"); + + return static_cast(reply.status()); +} + +Expected CreateConfiguredInferModelSerializer::serialize_request(rpc_create_configured_infer_model_request_params_t params) +{ + InferModel_CreateConfiguredInferModel_Request request; + + auto proto_infer_model_handle = request.mutable_infer_model_handle(); + proto_infer_model_handle->set_id(params.infer_model_handle); + + auto proto_vdevide_handle = request.mutable_vdevice_handle(); + proto_vdevide_handle->set_id(params.vdevice_handle); + + for (auto &input_stream_params : params.input_streams_params) { + auto proto_input_stream = request.add_input_infer_streams(); + proto_input_stream->set_name(input_stream_params.first); + proto_input_stream->set_format_order(input_stream_params.second.format_order); + proto_input_stream->set_format_type(input_stream_params.second.format_type); + proto_input_stream->set_nms_score_threshold(input_stream_params.second.nms_score_threshold); + proto_input_stream->set_nms_iou_threshold(input_stream_params.second.nms_iou_threshold); + proto_input_stream->set_nms_max_proposals_per_class(input_stream_params.second.nms_max_proposals_per_class); + proto_input_stream->set_nms_max_accumulated_mask_size(input_stream_params.second.nms_max_accumulated_mask_size); + } + + for (auto &output_stream_params : params.output_streams_params) { + auto proto_output_stream = request.add_output_infer_streams(); + proto_output_stream->set_name(output_stream_params.first); + proto_output_stream->set_format_order(output_stream_params.second.format_order); + proto_output_stream->set_format_type(output_stream_params.second.format_type); + proto_output_stream->set_nms_score_threshold(output_stream_params.second.nms_score_threshold); + proto_output_stream->set_nms_iou_threshold(output_stream_params.second.nms_iou_threshold); + proto_output_stream->set_nms_max_proposals_per_class(output_stream_params.second.nms_max_proposals_per_class); + proto_output_stream->set_nms_max_accumulated_mask_size(output_stream_params.second.nms_max_accumulated_mask_size); + } + + request.set_batch_size(static_cast(params.batch_size)); + request.set_power_mode(static_cast(params.power_mode)); + request.set_latency_flag(static_cast(params.latency_flag)); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'CreateConfiguredInferModel'"); + + return serialized_request; +} + +Expected CreateConfiguredInferModelSerializer::deserialize_request(const MemoryView &serialized_request) +{ + rpc_create_configured_infer_model_request_params_t request_params; + InferModel_CreateConfiguredInferModel_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'CreateConfiguredInferModel'"); + + request_params.infer_model_handle = request.infer_model_handle().id(); + request_params.vdevice_handle = request.vdevice_handle().id(); + + for (auto input_stream: request.input_infer_streams()) { + rpc_stream_params_t current_stream_params; + current_stream_params.format_order = input_stream.format_order(); + current_stream_params.format_type = input_stream.format_type(); + current_stream_params.nms_score_threshold = input_stream.nms_score_threshold(); + current_stream_params.nms_iou_threshold = input_stream.nms_iou_threshold(); + current_stream_params.nms_max_proposals_per_class = input_stream.nms_max_proposals_per_class(); + current_stream_params.nms_max_accumulated_mask_size = input_stream.nms_max_accumulated_mask_size(); + request_params.input_streams_params.emplace(input_stream.name(), current_stream_params); + } + + for (auto output_stream: request.output_infer_streams()) { + rpc_stream_params_t current_stream_params; + current_stream_params.format_order = output_stream.format_order(); + current_stream_params.format_type = output_stream.format_type(); + current_stream_params.nms_score_threshold = output_stream.nms_score_threshold(); + current_stream_params.nms_iou_threshold = output_stream.nms_iou_threshold(); + current_stream_params.nms_max_proposals_per_class = output_stream.nms_max_proposals_per_class(); + current_stream_params.nms_max_accumulated_mask_size = output_stream.nms_max_accumulated_mask_size(); + request_params.output_streams_params.emplace(output_stream.name(), current_stream_params); + } + + request_params.batch_size = static_cast(request.batch_size()); + request_params.power_mode = static_cast(request.power_mode()); + request_params.latency_flag = static_cast(request.latency_flag()); + + return request_params; +} + +Expected CreateConfiguredInferModelSerializer::serialize_reply(hailo_status status, rpc_object_handle_t configured_infer_handle, + uint32_t async_queue_size) +{ + InferModel_CreateConfiguredInferModel_Reply reply; + + reply.set_status(status); + auto proto_configured_infer_model_handle = reply.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_handle); + reply.set_async_queue_size(async_queue_size); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'CreateConfiguredInferModel'"); + + return serialized_reply; +} + +Expected> CreateConfiguredInferModelSerializer::deserialize_reply( + const MemoryView &serialized_reply) +{ + InferModel_CreateConfiguredInferModel_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'CreateConfiguredInferModel'"); + + return std::make_tuple(static_cast(reply.status()), reply.configured_infer_model_handle().id(), reply.async_queue_size()); +} + +Expected DestroyConfiguredInferModelSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle) +{ + ConfiguredInferModel_Destroy_Request request; + + auto proto_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_infer_model_handle->set_id(configured_infer_model_handle); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'DestroyConfiguredInferModel'"); + + return serialized_request; +} + +Expected DestroyConfiguredInferModelSerializer::deserialize_request(const MemoryView &serialized_request) +{ + ConfiguredInferModel_Destroy_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'DestroyConfiguredInferModel'"); + + return request.configured_infer_model_handle().id(); +} + +Expected DestroyConfiguredInferModelSerializer::serialize_reply(hailo_status status) +{ + ConfiguredInferModel_Destroy_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'DestroyConfiguredInferModel'"); + + return serialized_reply; +} + +hailo_status DestroyConfiguredInferModelSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_Destroy_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'CreateConfiguredInferModel'"); + CHECK_SUCCESS(static_cast(reply.status())); + + return HAILO_SUCCESS; +} + +Expected SetSchedulerTimeoutSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle, const std::chrono::milliseconds &timeout) +{ + ConfiguredInferModel_SetSchedulerTimeout_Request request; + + auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_model_handle); + request.set_timeout(static_cast(timeout.count())); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerTimeout'"); + + return serialized_request; +} + +Expected> SetSchedulerTimeoutSerializer::deserialize_request( + const MemoryView &serialized_request) +{ + ConfiguredInferModel_SetSchedulerTimeout_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerTimeout'"); + + return std::make_tuple(request.configured_infer_model_handle().id(), std::chrono::milliseconds(request.timeout())); +} + +Expected SetSchedulerTimeoutSerializer::serialize_reply(hailo_status status) +{ + ConfiguredInferModel_SetSchedulerTimeout_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerTimeout'"); + + return serialized_reply; +} + +hailo_status SetSchedulerTimeoutSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_SetSchedulerTimeout_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerTimeout'"); + + return static_cast(reply.status()); +} + +Expected SetSchedulerThresholdSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle, uint32_t threshold) +{ + ConfiguredInferModel_SetSchedulerThreshold_Request request; + + auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_model_handle); + request.set_threshold(threshold); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerThreshold'"); + + return serialized_request; +} + +Expected> SetSchedulerThresholdSerializer::deserialize_request( + const MemoryView &serialized_request) +{ + ConfiguredInferModel_SetSchedulerThreshold_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerThreshold'"); + + return std::make_tuple(request.configured_infer_model_handle().id(), request.threshold()); +} + +Expected SetSchedulerThresholdSerializer::serialize_reply(hailo_status status) +{ + ConfiguredInferModel_SetSchedulerThreshold_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerThreshold'"); + + return serialized_reply; +} + +hailo_status SetSchedulerThresholdSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_SetSchedulerThreshold_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerThreshold'"); + + return static_cast(reply.status()); +} + +Expected SetSchedulerPrioritySerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle, uint32_t priority) +{ + ConfiguredInferModel_SetSchedulerPriority_Request request; + + auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_model_handle); + request.set_priority(priority); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerPriority'"); + + return serialized_request; +} + +Expected> SetSchedulerPrioritySerializer::deserialize_request( + const MemoryView &serialized_request) +{ + ConfiguredInferModel_SetSchedulerPriority_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerPriority'"); + + return std::make_tuple(request.configured_infer_model_handle().id(), request.priority()); +} + +Expected SetSchedulerPrioritySerializer::serialize_reply(hailo_status status) +{ + ConfiguredInferModel_SetSchedulerPriority_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'SetSchedulerPriority'"); + + return serialized_reply; +} + +hailo_status SetSchedulerPrioritySerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_SetSchedulerPriority_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'SetSchedulerPriority'"); + + return static_cast(reply.status()); +} + +Expected GetHwLatencyMeasurementSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle) +{ + ConfiguredInferModel_GetHwLatencyMeasurement_Request request; + + auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_model_handle); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'GetHwLatencyMeasurement'"); + + return serialized_request; +} + +Expected GetHwLatencyMeasurementSerializer::deserialize_request(const MemoryView &serialized_request) +{ + ConfiguredInferModel_GetHwLatencyMeasurement_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'GetHwLatencyMeasurement'"); + + return request.configured_infer_model_handle().id(); +} + +Expected GetHwLatencyMeasurementSerializer::serialize_reply(hailo_status status, uint32_t avg_hw_latency) +{ + ConfiguredInferModel_GetHwLatencyMeasurement_Reply reply; + reply.set_status(status); + reply.set_avg_hw_latency(avg_hw_latency); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'GetHwLatencyMeasurement'"); + + return serialized_reply; +} + +Expected> GetHwLatencyMeasurementSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_GetHwLatencyMeasurement_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'GetHwLatencyMeasurement'"); + + return std::make_tuple(static_cast(reply.status()), std::chrono::nanoseconds(reply.avg_hw_latency())); +} + +Expected ActivateSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle) +{ + ConfiguredInferModel_Activate_Request request; + + auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_model_handle); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'Activate'"); + + return serialized_request; +} + +Expected ActivateSerializer::deserialize_request(const MemoryView &serialized_request) +{ + ConfiguredInferModel_Activate_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'Activate'"); + + return request.configured_infer_model_handle().id(); +} + +Expected ActivateSerializer::serialize_reply(hailo_status status) +{ + ConfiguredInferModel_Activate_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'Activate'"); + + return serialized_reply; +} + +hailo_status ActivateSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_Activate_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'Activate'"); + + return static_cast(reply.status()); +} + +Expected DeactivateSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle) +{ + ConfiguredInferModel_Deactivate_Request request; + + auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_model_handle); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'Deactivate'"); + + return serialized_request; +} + +Expected DeactivateSerializer::deserialize_request(const MemoryView &serialized_request) +{ + ConfiguredInferModel_Deactivate_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'Deactivate'"); + + return request.configured_infer_model_handle().id(); +} + +Expected DeactivateSerializer::serialize_reply(hailo_status status) +{ + ConfiguredInferModel_Deactivate_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'Deactivate'"); + + return serialized_reply; +} + +hailo_status DeactivateSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_Deactivate_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'Deactivate'"); + + return static_cast(reply.status()); +} + +Expected ShutdownSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle) +{ + ConfiguredInferModel_Shutdown_Request request; + + auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_model_handle); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'Shutdown'"); + + return serialized_request; +} + +Expected ShutdownSerializer::deserialize_request(const MemoryView &serialized_request) +{ + ConfiguredInferModel_Shutdown_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'Shutdown'"); + + return request.configured_infer_model_handle().id(); +} + +Expected ShutdownSerializer::serialize_reply(hailo_status status) +{ + ConfiguredInferModel_Shutdown_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'Shutdown'"); + + return serialized_reply; +} + +hailo_status ShutdownSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_Shutdown_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'Shutdown'"); + + return static_cast(reply.status()); +} + +Expected RunAsyncSerializer::serialize_request(rpc_object_handle_t configured_infer_model_handle, rpc_object_handle_t infer_model_handle, + rpc_object_handle_t callback_handle) +{ + ConfiguredInferModel_AsyncInfer_Request request; + + auto proto_configured_infer_model_handle = request.mutable_configured_infer_model_handle(); + proto_configured_infer_model_handle->set_id(configured_infer_model_handle); + + auto proto_infer_model_handle = request.mutable_infer_model_handle(); + proto_infer_model_handle->set_id(infer_model_handle); + + auto proto_cb_handle = request.mutable_callback_handle(); + proto_cb_handle->set_id(callback_handle); + + // TODO (HRT-13983) - check if we can use GetCachedSize + TRY(auto serialized_request, Buffer::create(request.ByteSizeLong(), BufferStorageParams::create_dma())); + CHECK_AS_EXPECTED(request.SerializeToArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to serialize 'RunAsync'"); + + return serialized_request; +} + +Expected> RunAsyncSerializer::deserialize_request( + const MemoryView &serialized_request) +{ + ConfiguredInferModel_AsyncInfer_Request request; + + CHECK_AS_EXPECTED(request.ParseFromArray(serialized_request.data(), static_cast(serialized_request.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'RunAsync'"); + + return std::make_tuple(request.configured_infer_model_handle().id(), request.infer_model_handle().id(), + request.callback_handle().id()); +} + +Expected RunAsyncSerializer::serialize_reply(hailo_status status) +{ + ConfiguredInferModel_AsyncInfer_Reply reply; + reply.set_status(status); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'RunAsync'"); + + return serialized_reply; +} + +hailo_status RunAsyncSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + ConfiguredInferModel_AsyncInfer_Reply reply; + + CHECK(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'RunAsync'"); + + return static_cast(reply.status()); +} + +Expected CallbackCalledSerializer::serialize_reply(hailo_status status, rpc_object_handle_t callback_handle) +{ + CallbackCalled_Reply reply; + + reply.set_status(status); + auto proto_callback_handle = reply.mutable_callback_handle(); + proto_callback_handle->set_id(callback_handle); + + TRY(auto serialized_reply, Buffer::create(reply.ByteSizeLong(), BufferStorageParams::create_dma())); + + CHECK_AS_EXPECTED(reply.SerializeToArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to serialize 'CallbackCalled'"); + + return serialized_reply; +} + +Expected> CallbackCalledSerializer::deserialize_reply(const MemoryView &serialized_reply) +{ + CallbackCalled_Reply reply; + + CHECK_AS_EXPECTED(reply.ParseFromArray(serialized_reply.data(), static_cast(serialized_reply.size())), + HAILO_RPC_FAILED, "Failed to de-serialize 'CallbackCalled'"); + + return std::make_tuple(static_cast(reply.status()), reply.callback_handle().id()); +} + +} /* namespace hailort */ diff --git a/hailort/hrpc_protocol/serializer.hpp b/hailort/hrpc_protocol/serializer.hpp new file mode 100644 index 0000000..a4d435c --- /dev/null +++ b/hailort/hrpc_protocol/serializer.hpp @@ -0,0 +1,253 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file serializer.hpp + * @brief HRPC protocol serialization + **/ + +#ifndef _HAILO_SERIALIZER_HPP_ +#define _HAILO_SERIALIZER_HPP_ + +#include "hailo/hailort.h" +#include "hailo/buffer.hpp" +#include "hailo/expected.hpp" + +#include +#include + +namespace hailort +{ + +#define INVALID_HANDLE_ID (UINT32_MAX) +#define INVALID_LATENCY_MEASUREMENT (UINT32_MAX) + +enum class HailoRpcActionID { + VDEVICE__CREATE, + VDEVICE__DESTROY, + VDEVICE__CREATE_INFER_MODEL, + + INFER_MODEL__DESTROY, + INFER_MODEL__CREATE_CONFIGURED_INFER_MODEL, + + CONFIGURED_INFER_MODEL__DESTROY, + CONFIGURED_INFER_MODEL__SET_SCHEDULER_TIMEOUT, + CONFIGURED_INFER_MODEL__SET_SCHEDULER_THRESHOLD, + CONFIGURED_INFER_MODEL__SET_SCHEDULER_PRIORITY, + CONFIGURED_INFER_MODEL__GET_HW_LATENCY_MEASUREMENT, + CONFIGURED_INFER_MODEL__ACTIVATE, + CONFIGURED_INFER_MODEL__DEACTIVATE, + CONFIGURED_INFER_MODEL__SHUTDOWN, + CONFIGURED_INFER_MODEL__RUN_ASYNC, + + CALLBACK_CALLED, + + MAX_VALUE, +}; + +using rpc_object_handle_t = uint32_t; +struct rpc_stream_params_t +{ + uint32_t format_order; + uint32_t format_type; + float32_t nms_score_threshold; + float32_t nms_iou_threshold; + uint32_t nms_max_proposals_per_class; + uint32_t nms_max_accumulated_mask_size; +}; +using rpc_stream_params_map_t = std::unordered_map; +struct rpc_create_configured_infer_model_request_params_t +{ + rpc_object_handle_t infer_model_handle; + rpc_object_handle_t vdevice_handle; + rpc_stream_params_map_t input_streams_params; + rpc_stream_params_map_t output_streams_params; + uint16_t batch_size; + hailo_power_mode_t power_mode; + hailo_latency_measurement_flags_t latency_flag; +}; + +class CreateVDeviceSerializer +{ +public: + CreateVDeviceSerializer() = delete; + + static Expected serialize_request(const hailo_vdevice_params_t ¶ms); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status, rpc_object_handle_t vdevice_handle = INVALID_HANDLE_ID); + static Expected> deserialize_reply(const MemoryView &serialized_reply); +}; + +class DestroyVDeviceSerializer +{ +public: + DestroyVDeviceSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t vdevice_handle); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class CreateInferModelSerializer +{ +public: + CreateInferModelSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t vdevice_handle, uint64_t hef_size); + static Expected> deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status, rpc_object_handle_t infer_model_handle = INVALID_HANDLE_ID); + static Expected> deserialize_reply(const MemoryView &serialized_reply); +}; + +class DestroyInferModelSerializer +{ +public: + DestroyInferModelSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t infer_model_handle); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class CreateConfiguredInferModelSerializer +{ +public: + CreateConfiguredInferModelSerializer() = delete; + + static Expected serialize_request(rpc_create_configured_infer_model_request_params_t params); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status, rpc_object_handle_t configured_infer_handle = INVALID_HANDLE_ID, + uint32_t async_queue_size = 0); + static Expected> deserialize_reply(const MemoryView &serialized_reply); +}; + +class DestroyConfiguredInferModelSerializer +{ +public: + DestroyConfiguredInferModelSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class SetSchedulerTimeoutSerializer +{ +public: + SetSchedulerTimeoutSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle, const std::chrono::milliseconds &timeout); + static Expected> deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class SetSchedulerThresholdSerializer +{ +public: + SetSchedulerThresholdSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle, uint32_t threshold); + static Expected> deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class SetSchedulerPrioritySerializer +{ +public: + SetSchedulerPrioritySerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle, uint32_t priority); + static Expected> deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class GetHwLatencyMeasurementSerializer +{ +public: + GetHwLatencyMeasurementSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status, uint32_t avg_hw_latency = INVALID_LATENCY_MEASUREMENT); + static Expected> deserialize_reply(const MemoryView &serialized_reply); +}; + +class ActivateSerializer +{ +public: + ActivateSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class DeactivateSerializer +{ +public: + DeactivateSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class ShutdownSerializer +{ +public: + ShutdownSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle); + static Expected deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class RunAsyncSerializer +{ +public: + RunAsyncSerializer() = delete; + + static Expected serialize_request(rpc_object_handle_t configured_infer_model_handle, rpc_object_handle_t infer_model_handle, + rpc_object_handle_t callback_handle); + static Expected> deserialize_request(const MemoryView &serialized_request); + + static Expected serialize_reply(hailo_status status); + static hailo_status deserialize_reply(const MemoryView &serialized_reply); +}; + +class CallbackCalledSerializer +{ +public: + CallbackCalledSerializer() = delete; + + static Expected serialize_reply(hailo_status status, rpc_object_handle_t callback_handle = INVALID_HANDLE_ID); + static Expected> deserialize_reply(const MemoryView &serialized_reply); +}; + + +} /* namespace hailort */ + +#endif /* _HAILO_SERIALIZER_HPP_ */ diff --git a/hailort/libhailort/CMakeLists.txt b/hailort/libhailort/CMakeLists.txt index c547641..33e183a 100644 --- a/hailort/libhailort/CMakeLists.txt +++ b/hailort/libhailort/CMakeLists.txt @@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.0.0) # set(CMAKE_C_CLANG_TIDY "clang-tidy;-checks=*") set(HAILORT_MAJOR_VERSION 4) -set(HAILORT_MINOR_VERSION 17) -set(HAILORT_REVISION_VERSION 1) +set(HAILORT_MINOR_VERSION 18) +set(HAILORT_REVISION_VERSION 0) # Add the cmake folder so the modules there are found set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) @@ -11,6 +11,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) # Generate hef-proto files using host protobuf protobuf_generate_cpp(PROTO_HEF_SRC PROTO_HEF_HEADER hef.proto) protobuf_generate_python(PROTO_HEF_PY hef.proto) # TODO (HRT-12504): Copy hef_pb2.py to tools directory +protobuf_generate_python(PROTO_HEF_PY tracer_profiler.proto) add_library(hef_proto ${PROTO_HEF_SRC} ${PROTO_HEF_HEADER} ${PROTO_HEF_PY}) target_link_libraries(hef_proto libprotobuf-lite) @@ -68,4 +69,6 @@ if(HAILO_BUILD_UT) add_subdirectory(tests) endif() add_subdirectory(bindings) -add_subdirectory(doc) +if(HAILO_BUILD_DOC) + add_subdirectory(doc) +endif() diff --git a/hailort/libhailort/bindings/gstreamer/CMakeLists.txt b/hailort/libhailort/bindings/gstreamer/CMakeLists.txt index d8d4480..5a60180 100644 --- a/hailort/libhailort/bindings/gstreamer/CMakeLists.txt +++ b/hailort/libhailort/bindings/gstreamer/CMakeLists.txt @@ -8,7 +8,7 @@ if(NOT CMAKE_HOST_UNIX) message(FATAL_ERROR "Only unix hosts are supported, stopping build") endif() -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) # GST_PLUGIN_DEFINE needs PACKAGE to be defined set(GST_HAILO_PACKAGE_NAME "hailo") @@ -27,6 +27,8 @@ add_library(gsthailo SHARED gst-hailo/sync_gst_hailosend.cpp gst-hailo/sync_gst_hailorecv.cpp gst-hailo/gsthailonet.cpp + gst-hailo/gsthailo_allocator.cpp + gst-hailo/gsthailo_dmabuf_allocator.cpp gst-hailo/gsthailodevicestats.cpp gst-hailo/common.cpp gst-hailo/network_group_handle.cpp diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp index bbee72b..2a95dd7 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp @@ -33,8 +33,11 @@ using namespace hailort; +#define ERROR(msg, ...) g_print("HailoNet Error: " msg, ##__VA_ARGS__) #define PLUGIN_AUTHOR "Hailo Technologies Ltd. (\"Hailo\")" +#define MAX_STRING_SIZE (PATH_MAX) + #define MAX_QUEUED_BUFFERS_IN_INPUT (16) #define MAX_QUEUED_BUFFERS_IN_OUTPUT (16) #define MAX_QUEUED_BUFFERS_IN_CORE (16) @@ -156,6 +159,31 @@ using namespace hailort; } while(0) #define CHECK_EXPECTED(obj, ...) _CHECK_EXPECTED(obj, "" __VA_ARGS__) +#define __HAILO_CONCAT(x, y) x ## y +#define _HAILO_CONCAT(x, y) __HAILO_CONCAT(x, y) + +#define _TRY(expected_var_name, var_decl, expr, ...) \ + auto expected_var_name = (expr); \ + CHECK_EXPECTED(expected_var_name, __VA_ARGS__); \ + var_decl = expected_var_name.release() + +/** + * The TRY macro is used to allow easier validation and access for variables returned as Expected. + * If the expression returns an Expected with status HAILO_SUCCESS, the macro will release the expected and assign + * the var_decl. + * Otherwise, the macro will cause current function to return the failed status. + * + * Usage example: + * + * Expected func() { + * TRY(auto var, return_5()); + * // Now var is int with value 5 + * + * // func will return Unexpected with status HAILO_INTERNAL_FAILURE + * TRY(auto var2, return_error(HAILO_INTERNAL_FAILURE), "Failed doing stuff {}", 5); + */ +#define TRY(var_decl, expr, ...) _TRY(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__) + #define RGB_FEATURES_SIZE (3) #define RGBA_FEATURES_SIZE (4) #define GRAY8_FEATURES_SIZE (1) @@ -194,7 +222,7 @@ public: return *this; } - const T &get() + const T &get() const { return m_value; } @@ -209,6 +237,38 @@ private: bool m_was_changed; }; +class HailoElemStringProperty final +{ +public: + HailoElemStringProperty(const std::string &default_val) : m_was_changed(false) { + memset(m_string, 0, sizeof(m_string)); + strncpy(m_string, default_val.c_str(), sizeof(m_string) - 1); + } + + ~HailoElemStringProperty() {} + + HailoElemStringProperty &operator=(const std::string &value) + { + m_was_changed = true; + strncpy(m_string, value.c_str(), sizeof(m_string) - 1); + return *this; + } + + const std::string get() const + { + return m_string; + } + + bool was_changed() + { + return m_was_changed; + } + +private: + char m_string[MAX_STRING_SIZE]; + bool m_was_changed; +}; + template<> HailoElemProperty::~HailoElemProperty(); diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_allocator.cpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_allocator.cpp new file mode 100644 index 0000000..08d455f --- /dev/null +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_allocator.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021-2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#include "gsthailo_allocator.hpp" + + +G_DEFINE_TYPE (GstHailoAllocator, gst_hailo_allocator, GST_TYPE_ALLOCATOR); + + +static GstMemory *gst_hailo_allocator_alloc(GstAllocator* allocator, gsize size, GstAllocationParams* /*params*/) { + GstHailoAllocator *hailo_allocator = GST_HAILO_ALLOCATOR(allocator); + auto buffer = Buffer::create(size, BufferStorageParams::create_dma()); + if (!buffer) { + ERROR("Creating buffer for allocator has failed, status = %d\n", buffer.status()); + return nullptr; + } + + GstMemory *memory = gst_memory_new_wrapped(static_cast(0), buffer->data(), + buffer->size(), 0, buffer->size(), nullptr, nullptr); + if (nullptr == memory) { + ERROR("Creating new GstMemory for allocator has failed!\n"); + return nullptr; + } + + hailo_allocator->buffers[memory] = std::move(buffer.release()); + return memory; +} + +static void gst_hailo_allocator_free(GstAllocator* allocator, GstMemory *mem) { + GstHailoAllocator *hailo_allocator = GST_HAILO_ALLOCATOR(allocator); + hailo_allocator->buffers.erase(mem); +} + +static void gst_hailo_allocator_class_init(GstHailoAllocatorClass* klass) { + GstAllocatorClass* allocator_class = GST_ALLOCATOR_CLASS(klass); + + allocator_class->alloc = gst_hailo_allocator_alloc; + allocator_class->free = gst_hailo_allocator_free; +} + +static void gst_hailo_allocator_init(GstHailoAllocator* allocator) { + allocator->buffers = std::unordered_map(); +} diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_allocator.hpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_allocator.hpp new file mode 100644 index 0000000..7495fbe --- /dev/null +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_allocator.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021-2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef _GST_HAILO_ALLOCATOR_HPP_ +#define _GST_HAILO_ALLOCATOR_HPP_ + +#include "common.hpp" + +using namespace hailort; + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_ALLOCATOR (gst_hailo_allocator_get_type()) +#define GST_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocator)) +#define GST_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocatorClass)) +#define GST_IS_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_HAILO_ALLOCATOR)) +#define GST_IS_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_HAILO_ALLOCATOR)) + +struct GstHailoAllocator +{ + GstAllocator parent; + std::unordered_map buffers; +}; + +struct GstHailoAllocatorClass +{ + GstAllocatorClass parent; +}; + +GType gst_hailo_allocator_get_type(void); + +G_END_DECLS + +#endif /* _GST_HAILO_ALLOCATOR_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_dmabuf_allocator.cpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_dmabuf_allocator.cpp new file mode 100644 index 0000000..cf9b21e --- /dev/null +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_dmabuf_allocator.cpp @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2021-2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gsthailo_dmabuf_allocator.hpp" + +#include +#include + +// TODO: HRT-13107 +#define DEVPATH "/dev/dma_heap/linux,cma" + +G_DEFINE_TYPE (GstHailoDmabufAllocator, gst_hailo_dmabuf_allocator, GST_TYPE_DMABUF_ALLOCATOR); + + +bool GstHailoDmaHeapControl::dma_heap_fd_open = false; +int GstHailoDmaHeapControl::dma_heap_fd = -1; + + +static GstMemory *gst_hailo_dmabuf_allocator_alloc(GstAllocator* allocator, gsize size, GstAllocationParams* /*params*/) { + GstHailoDmabufAllocator *hailo_allocator = GST_HAILO_DMABUF_ALLOCATOR(allocator); + + if (!GstHailoDmaHeapControl::dma_heap_fd_open) { + GstHailoDmaHeapControl::dma_heap_fd = open(DEVPATH, O_RDWR | O_CLOEXEC); + if (GstHailoDmaHeapControl::dma_heap_fd < 0) { + ERROR("open fd failed!\n"); + return nullptr; + } + GstHailoDmaHeapControl::dma_heap_fd_open = true; + } + + dma_heap_allocation_data heap_data; + heap_data = { + .len = size, + .fd = 0, + .fd_flags = O_RDWR | O_CLOEXEC, + .heap_flags = 0, + }; + + int ret = ioctl(GstHailoDmaHeapControl::dma_heap_fd, DMA_HEAP_IOCTL_ALLOC, &heap_data); + if (ret < 0) { + ERROR("ioctl DMA_HEAP_IOCTL_ALLOC failed! ret = %d\n", ret); + return nullptr; + } + + if (GST_IS_DMABUF_ALLOCATOR(hailo_allocator) == false) { + ERROR("hailo_allocator is not dmabuf!\n"); + return nullptr; + } + + GstMemory *memory = gst_dmabuf_allocator_alloc(allocator, heap_data.fd, size); + if (nullptr == memory) { + ERROR("Creating new GstMemory for allocator has failed!\n"); + return nullptr; + } + + hailo_allocator->dma_buffers[memory] = heap_data; + return memory; +} + +static void gst_hailo_dmabuf_allocator_free(GstAllocator* allocator, GstMemory *mem) { + GstHailoDmabufAllocator *hailo_allocator = GST_HAILO_DMABUF_ALLOCATOR(allocator); + close(hailo_allocator->dma_buffers[mem].fd); + hailo_allocator->dma_buffers.erase(mem); +} + +static void gst_hailo_dmabuf_allocator_class_init(GstHailoDmabufAllocatorClass* klass) { + GstAllocatorClass* allocator_class = GST_ALLOCATOR_CLASS(klass); + allocator_class->alloc = gst_hailo_dmabuf_allocator_alloc; + allocator_class->free = gst_hailo_dmabuf_allocator_free; +} + +static void gst_hailo_dmabuf_allocator_init(GstHailoDmabufAllocator* allocator) { + allocator->dma_buffers = std::unordered_map(); +} diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_dmabuf_allocator.hpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_dmabuf_allocator.hpp new file mode 100644 index 0000000..e45c777 --- /dev/null +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailo_dmabuf_allocator.hpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021-2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef _GST_HAILO_DMABUF_ALLOCATOR_HPP_ +#define _GST_HAILO_DMABUF_ALLOCATOR_HPP_ + +#include "common.hpp" +#include "hailo/hailort_dma-heap.h" + +#include + +using namespace hailort; + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_DMABUF_ALLOCATOR (gst_hailo_dmabuf_allocator_get_type()) +#define GST_HAILO_DMABUF_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_HAILO_DMABUF_ALLOCATOR, GstHailoDmabufAllocator)) +#define GST_HAILO_DMABUF_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_HAILO_DMABUF_ALLOCATOR, GstHailoDmabufAllocator)) +#define GST_IS_HAILO_DMABUF_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_HAILO_DMABUF_ALLOCATOR)) +#define GST_IS_HAILO_DMABUF_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_HAILO_DMABUF_ALLOCATOR)) + +#define GST_HAILO_USE_DMA_BUFFER_ENV_VAR "GST_HAILO_USE_DMA_BUFFER" + +class GstHailoDmaHeapControl { +public: + static bool dma_heap_fd_open; + static int dma_heap_fd; +}; + +struct GstHailoDmabufAllocator +{ + GstDmaBufAllocator parent; + std::unordered_map dma_buffers; +}; + +struct GstHailoDmabufAllocatorClass +{ + GstDmaBufAllocatorClass parent; +}; + +GType gst_hailo_dmabuf_allocator_get_type(void); + +G_END_DECLS + +#endif /* _GST_HAILO_DMABUF_ALLOCATOR_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp index e1508af..271440d 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp @@ -23,12 +23,10 @@ #include "hailo/hailort_common.hpp" #include "hailo/hailort_defaults.hpp" -#include #include #include #define WAIT_FOR_ASYNC_READY_TIMEOUT (std::chrono::milliseconds(10000)) -#define ERROR(msg, ...) g_print(msg, ##__VA_ARGS__) enum { @@ -63,7 +61,6 @@ enum static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); -G_DEFINE_TYPE (GstHailoAllocator, gst_hailo_allocator, GST_TYPE_ALLOCATOR); G_DEFINE_TYPE (GstHailoNet, gst_hailonet, GST_TYPE_ELEMENT); static std::atomic_uint32_t hailonet_count(0); @@ -74,41 +71,6 @@ static bool gst_hailo_should_use_dma_buffers() return (nullptr != env) && (0 == g_strcmp0(env, "1")); } -static GstMemory *gst_hailo_allocator_alloc(GstAllocator* allocator, gsize size, GstAllocationParams* /*params*/) { - GstHailoAllocator *hailo_allocator = GST_HAILO_ALLOCATOR(allocator); - auto buffer = Buffer::create(size, BufferStorageParams::create_dma()); - if (!buffer) { - ERROR("Creating buffer for allocator has failed, status = %d\n", buffer.status()); - return nullptr; - } - - GstMemory *memory = gst_memory_new_wrapped(static_cast(0), buffer->data(), - buffer->size(), 0, buffer->size(), nullptr, nullptr); - if (nullptr == memory) { - ERROR("Creating new GstMemory for allocator has failed!\n"); - return nullptr; - } - - hailo_allocator->buffers[memory] = std::move(buffer.release()); - return memory; -} - -static void gst_hailo_allocator_free(GstAllocator* allocator, GstMemory *mem) { - GstHailoAllocator *hailo_allocator = GST_HAILO_ALLOCATOR(allocator); - hailo_allocator->buffers.erase(mem); -} - -static void gst_hailo_allocator_class_init(GstHailoAllocatorClass* klass) { - GstAllocatorClass* allocator_class = GST_ALLOCATOR_CLASS(klass); - - allocator_class->alloc = gst_hailo_allocator_alloc; - allocator_class->free = gst_hailo_allocator_free; -} - -static void gst_hailo_allocator_init(GstHailoAllocator* allocator) { - allocator->buffers = std::unordered_map(); -} - static hailo_status gst_hailonet_deconfigure(GstHailoNet *self) { // This will wakeup any blocking calls to deuque @@ -122,6 +84,14 @@ static hailo_status gst_hailonet_deconfigure(GstHailoNet *self) return HAILO_SUCCESS; } +static void gst_hailonet_unref_input_caps(GstHailoNet *self) +{ + if (nullptr != self->input_caps) { + gst_caps_unref(self->input_caps); + self->input_caps = nullptr; + } +} + static hailo_status gst_hailonet_free(GstHailoNet *self) { std::unique_lock lock(self->infer_mutex); @@ -147,8 +117,24 @@ static hailo_status gst_hailonet_free(GstHailoNet *self) gst_queue_array_free(self->thread_queue); } - if (nullptr != self->input_caps) { - gst_caps_unref(self->input_caps); + while(!self->curr_event_queue.empty()) { + auto event = self->curr_event_queue.front(); + gst_event_unref(event); + self->curr_event_queue.pop(); + } + + for (auto &buffer_events_queue_pair : self->events_queue_per_buffer) { + while(!buffer_events_queue_pair.second.empty()) { + auto event = buffer_events_queue_pair.second.front(); + gst_event_unref(event); + buffer_events_queue_pair.second.pop(); + } + } + self->events_queue_per_buffer.clear(); + + { + std::unique_lock lock(self->input_caps_mutex); + gst_hailonet_unref_input_caps(self); } for (auto &name_pool_pair : self->output_buffer_pools) { @@ -156,14 +142,21 @@ static hailo_status gst_hailonet_free(GstHailoNet *self) CHECK(result, HAILO_INTERNAL_FAILURE, "Could not release buffer pool"); gst_object_unref(name_pool_pair.second); } + self->output_buffer_pools.clear(); + if (gst_hailo_should_use_dma_buffers()) { - gst_object_unref(self->dma_allocator); - } else { + if (GstHailoDmaHeapControl::dma_heap_fd_open) { + close(GstHailoDmaHeapControl::dma_heap_fd); + GstHailoDmaHeapControl::dma_heap_fd_open = false; + } + + if (nullptr != self->dmabuf_allocator) { + gst_object_unref(self->dmabuf_allocator); + } + } else if (nullptr != self->allocator) { gst_object_unref(self->allocator); } - self->props.free_strings(); - return HAILO_SUCCESS; } @@ -171,18 +164,15 @@ static hailo_status gst_hailonet_set_format_types(GstHailoNet *self, std::shared { if (self->props.m_input_format_type.was_changed()) { for (const auto &input_name : infer_model->get_input_names()) { - auto input = infer_model->input(input_name); - CHECK_EXPECTED_AS_STATUS(input); - - input->set_format_type(self->props.m_input_format_type.get()); + TRY(auto input, infer_model->input(input_name)); + input.set_format_type(self->props.m_input_format_type.get()); } } if (self->props.m_output_format_type.was_changed()) { for (const auto &output_name : infer_model->get_output_names()) { - auto output = infer_model->output(output_name); - CHECK_EXPECTED_AS_STATUS(output); + TRY(auto output, infer_model->output(output_name)); - output->set_format_type(self->props.m_output_format_type.get()); + output.set_format_type(self->props.m_output_format_type.get()); } } @@ -198,25 +188,24 @@ static hailo_status gst_hailonet_set_nms_params(GstHailoNet *self, std::shared_p }); for (const auto &output_name : infer_model->get_output_names()) { - auto output = infer_model->output(output_name); - CHECK_EXPECTED_AS_STATUS(output); + TRY(auto output, infer_model->output(output_name)); if (self->props.m_nms_score_threshold.was_changed()) { CHECK(has_nms_output, HAILO_INVALID_OPERATION, "NMS score threshold is set, but there is no NMS output in this model."); - if (output->is_nms()) { - output->set_nms_score_threshold(self->props.m_nms_score_threshold.get()); + if (output.is_nms()) { + output.set_nms_score_threshold(self->props.m_nms_score_threshold.get()); } } if (self->props.m_nms_iou_threshold.was_changed()) { CHECK(has_nms_output, HAILO_INVALID_OPERATION, "NMS IoU threshold is set, but there is no NMS output in this model."); - if (output->is_nms()) { - output->set_nms_iou_threshold(self->props.m_nms_iou_threshold.get()); + if (output.is_nms()) { + output.set_nms_iou_threshold(self->props.m_nms_iou_threshold.get()); } } if (self->props.m_nms_max_proposals_per_class.was_changed()) { CHECK(has_nms_output, HAILO_INVALID_OPERATION, "NMS max proposals per class is set, but there is no NMS output in this model."); - if (output->is_nms()) { - output->set_nms_max_proposals_per_class(self->props.m_nms_max_proposals_per_class.get()); + if (output.is_nms()) { + output.set_nms_max_proposals_per_class(self->props.m_nms_max_proposals_per_class.get()); } } } @@ -252,7 +241,7 @@ static Expected gst_hailonet_create_buffer_pool(GstHailoNet *sel self->props.m_outputs_max_pool_size.get()); if (gst_hailo_should_use_dma_buffers()) { - gst_buffer_pool_config_set_allocator(config, self->dma_allocator, nullptr); + gst_buffer_pool_config_set_allocator(config, GST_ALLOCATOR(self->dmabuf_allocator), nullptr); } else { gst_buffer_pool_config_set_allocator(config, GST_ALLOCATOR(self->allocator), nullptr); } @@ -266,6 +255,43 @@ static Expected gst_hailonet_create_buffer_pool(GstHailoNet *sel return pool; } +static void gst_hailonet_push_event_to_queue(GstHailoNet *self, GstEvent *event) +{ + std::unique_lock lock(self->input_queue_mutex); + self->curr_event_queue.push(event); +} + +static gboolean gst_hailonet_handle_queued_event(GstHailoNet *self, GstEvent *event) +{ + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_CAPS: + { + GstCaps *caps; + gst_event_parse_caps(event, &caps); + auto result = gst_pad_set_caps(self->srcpad, caps); + gst_event_unref(event); + return result; + } + default: + return gst_pad_push_event(self->srcpad, event); + } +} + +static void gst_hailonet_handle_buffer_events(GstHailoNet *self, GstBuffer *buffer) +{ + if (self->events_queue_per_buffer.find(buffer) == self->events_queue_per_buffer.end()) { + // The buffer does not have any events to send + return; + } + + while (!self->events_queue_per_buffer.at(buffer).empty()) { + GstEvent* event = self->events_queue_per_buffer.at(buffer).front(); + (void)gst_hailonet_handle_queued_event(self, event); + self->events_queue_per_buffer.at(buffer).pop(); + } + self->events_queue_per_buffer.erase(buffer); +} + static hailo_status gst_hailonet_configure(GstHailoNet *self) { if (self->is_configured) { @@ -288,9 +314,8 @@ static hailo_status gst_hailonet_configure(GstHailoNet *self) for (const auto &input_name : self->infer_model->get_input_names()) { if(self->props.m_no_transform.get()) { // In case transformation is disabled - format order will be the same as we get from the HW (stream info). - auto input_stream_infos = self->infer_model->hef().get_stream_info_by_name(input_name, HAILO_H2D_STREAM); - CHECK_EXPECTED_AS_STATUS(input_stream_infos); - self->infer_model->input(input_name)->set_format_order(input_stream_infos.value().format.order); + TRY(const auto input_stream_infos, self->infer_model->hef().get_stream_info_by_name(input_name, HAILO_H2D_STREAM)); + self->infer_model->input(input_name)->set_format_order(input_stream_infos.format.order); } else if (self->infer_model->input(input_name)->format().order == HAILO_FORMAT_ORDER_NHWC) { self->infer_model->input(input_name)->set_format_order(HAILO_FORMAT_ORDER_RGB4); } @@ -299,16 +324,14 @@ static hailo_status gst_hailonet_configure(GstHailoNet *self) if (self->props.m_no_transform.get()) { for (const auto &output_name : self->infer_model->get_output_names()) { // In case transformation is disabled - format order will be the same as we get from the HW (stream info). - auto output_stream_infos = self->infer_model->hef().get_stream_info_by_name(output_name, HAILO_D2H_STREAM); - CHECK_EXPECTED_AS_STATUS(output_stream_infos); - self->infer_model->output(output_name)->set_format_order(output_stream_infos.value().format.order); + TRY(const auto output_stream_infos, self->infer_model->hef().get_stream_info_by_name(output_name, HAILO_D2H_STREAM)); + self->infer_model->output(output_name)->set_format_order(output_stream_infos.format.order); } } - auto configured_infer_model = self->infer_model->configure(); - CHECK_EXPECTED_AS_STATUS(configured_infer_model); + TRY(auto configured_infer_model, self->infer_model->configure()); - auto ptr = make_shared_nothrow(configured_infer_model.release()); + auto ptr = make_shared_nothrow(std::move(configured_infer_model)); CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY); self->configured_infer_model = ptr; @@ -319,19 +342,33 @@ static hailo_status gst_hailonet_configure(GstHailoNet *self) return HAILO_SUCCESS; } +static void gst_hailonet_init_allocator(GstHailoNet *self) +{ + gchar *parent_name = gst_object_get_name(GST_OBJECT(self)); + gchar *name = g_strconcat(parent_name, ":hailo_allocator", NULL); + g_free(parent_name); + + if (gst_hailo_should_use_dma_buffers()) { + self->dmabuf_allocator = GST_HAILO_DMABUF_ALLOCATOR(g_object_new(GST_TYPE_HAILO_DMABUF_ALLOCATOR, "name", name, NULL)); + gst_object_ref_sink(self->dmabuf_allocator); + } else { + self->allocator = GST_HAILO_ALLOCATOR(g_object_new(GST_TYPE_HAILO_ALLOCATOR, "name", name, NULL)); + gst_object_ref_sink(self->allocator); + } + + g_free(name); +} + static hailo_status gst_hailonet_allocate_infer_resources(GstHailoNet *self) { - auto bindings = self->configured_infer_model->create_bindings(); - CHECK_EXPECTED_AS_STATUS(bindings); - self->infer_bindings = std::move(bindings.release()); + TRY(self->infer_bindings, self->configured_infer_model->create_bindings()); self->output_buffer_pools = std::unordered_map(); self->output_vstream_infos = std::unordered_map(); - auto async_queue_size = self->configured_infer_model->get_async_queue_size(); - CHECK_EXPECTED_AS_STATUS(async_queue_size); - self->input_queue = gst_queue_array_new(static_cast(async_queue_size.value())); - self->thread_queue = gst_queue_array_new(static_cast(async_queue_size.value())); + TRY(const auto async_queue_size, self->configured_infer_model->get_async_queue_size()); + self->input_queue = gst_queue_array_new(static_cast(async_queue_size)); + self->thread_queue = gst_queue_array_new(static_cast(async_queue_size)); self->is_thread_running = true; self->thread = std::thread([self] () { while (self->is_thread_running) { @@ -351,7 +388,7 @@ static hailo_status gst_hailonet_allocate_infer_resources(GstHailoNet *self) self->thread_cv.notify_all(); if (GST_IS_PAD(self->srcpad)) { // Checking because we fail here when exiting the application GstFlowReturn ret = gst_pad_push(self->srcpad, buffer); - if ((GST_FLOW_OK != ret) && (GST_FLOW_FLUSHING != ret) && (!self->has_got_eos)) { + if ((GST_FLOW_OK != ret) && (GST_FLOW_FLUSHING != ret) && ((GST_FLOW_EOS != ret)) && (!self->has_got_eos)) { ERROR("gst_pad_push failed with status = %d\n", ret); break; } @@ -359,23 +396,58 @@ static hailo_status gst_hailonet_allocate_infer_resources(GstHailoNet *self) } }); + gst_hailonet_init_allocator(self); for (auto &output : self->infer_model->outputs()) { - auto buffer_pool = gst_hailonet_create_buffer_pool(self, output.get_frame_size()); - CHECK_EXPECTED_AS_STATUS(buffer_pool); - - self->output_buffer_pools[output.name()] = buffer_pool.release(); + TRY(self->output_buffer_pools[output.name()], gst_hailonet_create_buffer_pool(self, output.get_frame_size())); } - auto vstream_infos = self->infer_model->hef().get_output_vstream_infos(); - CHECK_EXPECTED_AS_STATUS(vstream_infos); - - for (const auto &vstream_info : vstream_infos.value()) { + TRY(const auto vstream_infos, self->infer_model->hef().get_output_vstream_infos()); + for (const auto &vstream_info : vstream_infos) { self->output_vstream_infos[vstream_info.name] = vstream_info; } return HAILO_SUCCESS; } +static GstPadProbeReturn gst_hailonet_sink_probe(GstPad */*pad*/, GstPadProbeInfo */*info*/, gpointer user_data) +{ + GstHailoNet *self = static_cast(user_data); + std::unique_lock lock(self->sink_probe_change_state_mutex); + + if (self->did_critical_failure_happen) { + return GST_PAD_PROBE_REMOVE; + } + + auto status = gst_hailonet_configure(self); + if (HAILO_SUCCESS != status) { + return GST_PAD_PROBE_REMOVE; + } + + status = gst_hailonet_allocate_infer_resources(self); + if (HAILO_SUCCESS != status) { + return GST_PAD_PROBE_REMOVE; + } + + if (HAILO_SCHEDULING_ALGORITHM_NONE != self->props.m_scheduling_algorithm.get()) { + self->props.m_is_active = true; + return GST_PAD_PROBE_REMOVE; + } + + if ((1 == hailonet_count) && (!self->props.m_is_active.was_changed())) { + self->props.m_is_active = true; + } + + if (self->props.m_is_active.get()) { + status = self->configured_infer_model->activate(); + if (HAILO_SUCCESS != status) { + return GST_PAD_PROBE_REMOVE; + } + } + + self->has_called_activate = true; + return GST_PAD_PROBE_REMOVE; +} + static GstStateChangeReturn gst_hailonet_change_state(GstElement *element, GstStateChange transition) { GstStateChangeReturn ret = GST_ELEMENT_CLASS(gst_hailonet_parent_class)->change_state(element, transition); @@ -409,6 +481,8 @@ static GstStateChangeReturn gst_hailonet_change_state(GstElement *element, GstSt if (HAILO_SUCCESS != status) { return GST_STATE_CHANGE_FAILURE; } + + gst_pad_add_probe(self->sinkpad, GST_PAD_PROBE_TYPE_BUFFER, static_cast(gst_hailonet_sink_probe), self, nullptr); break; } default: @@ -456,10 +530,7 @@ static void gst_hailonet_set_property(GObject *object, guint property_id, const g_warning("The network was already configured so changing the HEF path will not take place!"); break; } - if (nullptr != self->props.m_hef_path.get()) { - g_free(self->props.m_hef_path.get()); - } - self->props.m_hef_path = g_strdup(g_value_get_string(value)); + self->props.m_hef_path = g_value_get_string(value); break; case PROP_BATCH_SIZE: if (self->is_configured) { @@ -478,15 +549,12 @@ static void gst_hailonet_set_property(GObject *object, guint property_id, const g_warning("The network was already configured so changing the device ID will not take place!"); break; } - if (nullptr != self->props.m_device_id.get()) { - g_free(self->props.m_device_id.get()); - } - self->props.m_device_id = g_strdup(g_value_get_string(value)); + self->props.m_device_id = g_value_get_string(value); break; case PROP_DEVICE_COUNT: - if (nullptr != self->props.m_device_id.get()) { + if (!self->props.m_device_id.get().empty()) { g_error("device-id and device-count excludes eachother. received device-id=%s, device-count=%d", - self->props.m_device_id.get(), g_value_get_uint(value)); + self->props.m_device_id.get().c_str(), g_value_get_uint(value)); break; } if (self->is_configured) { @@ -500,10 +568,7 @@ static void gst_hailonet_set_property(GObject *object, guint property_id, const g_warning("The network was already configured so changing the vdevice group ID will not take place!"); break; } - if (nullptr != self->props.m_vdevice_group_id.get()) { - g_free(self->props.m_vdevice_group_id.get()); - } - self->props.m_vdevice_group_id = g_strdup(g_value_get_string(value)); + self->props.m_vdevice_group_id = g_value_get_string(value); break; case PROP_IS_ACTIVE: (void)gst_hailonet_toggle_activation(self, self->props.m_is_active.get(), g_value_get_boolean(value)); @@ -635,19 +700,19 @@ static void gst_hailonet_get_property(GObject *object, guint property_id, GValue GstHailoNet *self = GST_HAILONET(object); switch (property_id) { case PROP_HEF_PATH: - g_value_set_string(value, self->props.m_hef_path.get()); + g_value_set_string(value, self->props.m_hef_path.get().c_str()); break; case PROP_BATCH_SIZE: g_value_set_uint(value, self->props.m_batch_size.get()); break; case PROP_DEVICE_ID: - g_value_set_string(value, self->props.m_device_id.get()); + g_value_set_string(value, self->props.m_device_id.get().c_str()); break; case PROP_DEVICE_COUNT: g_value_set_uint(value, self->props.m_device_count.get()); break; case PROP_VDEVICE_GROUP_ID: - g_value_set_string(value, self->props.m_vdevice_group_id.get()); + g_value_set_string(value, self->props.m_vdevice_group_id.get().c_str()); break; case PROP_IS_ACTIVE: g_value_set_boolean(value, self->props.m_is_active.get()); @@ -911,11 +976,9 @@ static Expected> gst_hailone static hailo_status gst_hailonet_fill_multiple_input_bindings_dma_buffers(GstHailoNet *self, GstBuffer *buffer) { - auto input_buffers = gst_hailonet_read_input_dma_buffers_from_meta(self, buffer); - CHECK_EXPECTED_AS_STATUS(input_buffers); - for (const auto &name : self->infer_model->get_input_names()) - { - auto status = self->infer_bindings.input(name)->set_dma_buffer(input_buffers.value().at(name)); + TRY(auto input_buffers, gst_hailonet_read_input_dma_buffers_from_meta(self, buffer)); + for (const auto &name : self->infer_model->get_input_names()) { + auto status = self->infer_bindings.input(name)->set_dma_buffer(input_buffers.at(name)); CHECK_SUCCESS(status); } @@ -950,10 +1013,9 @@ static Expected> gst_hailonet_read_inp static hailo_status gst_hailonet_fill_multiple_input_bindings(GstHailoNet *self, GstBuffer *buffer) { - auto input_buffers = gst_hailonet_read_input_buffers_from_meta(self, buffer); - CHECK_EXPECTED_AS_STATUS(input_buffers); + TRY(auto input_buffers, gst_hailonet_read_input_buffers_from_meta(self, buffer)); for (const auto &name : self->infer_model->get_input_names()) { - auto status = self->infer_bindings.input(name)->set_buffer(MemoryView(input_buffers.value().at(name), + auto status = self->infer_bindings.input(name)->set_buffer(MemoryView(input_buffers.at(name), self->infer_model->input(name)->get_frame_size())); CHECK_SUCCESS(status); } @@ -961,9 +1023,20 @@ static hailo_status gst_hailonet_fill_multiple_input_bindings(GstHailoNet *self, return HAILO_SUCCESS; } +static void store_buffer_events(GstHailoNet *self, GstBuffer *buffer) +{ + self->events_queue_per_buffer[buffer] = std::queue(); + while (!self->curr_event_queue.empty()) { + GstEvent *event = self->curr_event_queue.front(); + self->events_queue_per_buffer[buffer].push(event); + self->curr_event_queue.pop(); + } +} + static hailo_status gst_hailonet_push_buffer_to_input_queue(GstHailoNet *self, GstBuffer *buffer) { std::unique_lock lock(self->input_queue_mutex); + store_buffer_events(self, buffer); gst_queue_array_push_tail(self->input_queue, buffer); return HAILO_SUCCESS; @@ -1022,11 +1095,12 @@ static hailo_status gst_hailonet_call_run_async(GstHailoNet *self, const std::un self->ongoing_frames++; } - auto job = self->configured_infer_model->run_async(self->infer_bindings, [self, tensors] (const AsyncInferCompletionInfo &/*completion_info*/) { + TRY(auto job, self->configured_infer_model->run_async(self->infer_bindings, [self, tensors] (const AsyncInferCompletionInfo &/*completion_info*/) { GstBuffer *buffer = nullptr; { std::unique_lock lock(self->input_queue_mutex); buffer = static_cast(gst_queue_array_pop_head(self->input_queue)); + gst_hailonet_handle_buffer_events(self, buffer); } for (auto &output : self->infer_model->outputs()) { @@ -1047,9 +1121,8 @@ static hailo_status gst_hailonet_call_run_async(GstHailoNet *self, const std::un self->flush_cv.notify_all(); gst_hailonet_push_buffer_to_thread(self, buffer); - }); - CHECK_EXPECTED_AS_STATUS(job); - job->detach(); + })); + job.detach(); return HAILO_SUCCESS; } @@ -1071,7 +1144,7 @@ static hailo_status gst_hailonet_async_infer_multi_input(GstHailoNet *self, GstB if (HAILO_STREAM_ABORT == tensors.status()) { return HAILO_SUCCESS; } - CHECK_EXPECTED_AS_STATUS(tensors); + CHECK_EXPECTED_AS_STATUS(tensors); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here status = gst_hailonet_call_run_async(self, tensors.value()); CHECK_SUCCESS(status); @@ -1090,7 +1163,7 @@ static hailo_status gst_hailonet_async_infer_single_input(GstHailoNet *self, Gst if (HAILO_STREAM_ABORT == tensors.status()) { return HAILO_SUCCESS; } - CHECK_EXPECTED_AS_STATUS(tensors); + CHECK_EXPECTED_AS_STATUS(tensors); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here status = gst_hailonet_call_run_async(self, tensors.value()); CHECK_SUCCESS(status); @@ -1125,6 +1198,10 @@ static GstFlowReturn gst_hailonet_chain(GstPad * /*pad*/, GstObject * parent, Gs GstHailoNet *self = GST_HAILONET(parent); std::unique_lock lock(self->infer_mutex); + if (self->did_critical_failure_happen) { + return GST_FLOW_ERROR; + } + if (self->props.m_pass_through.get() || !self->props.m_is_active.get() || !self->is_configured) { gst_hailonet_push_buffer_to_thread(self, buffer); return GST_FLOW_OK; @@ -1134,11 +1211,11 @@ static GstFlowReturn gst_hailonet_chain(GstPad * /*pad*/, GstObject * parent, Gs if (self->props.m_should_force_writable.get()) { buffer = gst_buffer_make_writable(buffer); if (nullptr == buffer) { - ERROR("Failed to make buffer writable!"); + ERROR("Failed to make buffer writable!\n"); return GST_FLOW_ERROR; } } else { - ERROR("Input buffer is not writable! Use force-writable property to force the buffer to be writable"); + ERROR("Input buffer is not writable! Use force-writable property to force the buffer to be writable\n"); return GST_FLOW_ERROR; } } @@ -1168,17 +1245,14 @@ static hailo_status gst_hailonet_init_infer_model(GstHailoNet * self) hailo_device_id_t device_id = {0}; if (self->props.m_device_id.was_changed()) { - auto expected_device_id = HailoRTCommon::to_device_id(self->props.m_device_id.get()); - CHECK_EXPECTED_AS_STATUS(expected_device_id); - device_id = std::move(expected_device_id.release()); - + TRY(device_id, HailoRTCommon::to_device_id(self->props.m_device_id.get())); vdevice_params.device_ids = &device_id; } if (self->props.m_device_count.was_changed()) { vdevice_params.device_count = self->props.m_device_count.get(); } if (self->props.m_vdevice_group_id.was_changed()) { - vdevice_params.group_id = self->props.m_vdevice_group_id.get(); + vdevice_params.group_id = self->props.m_vdevice_group_id.get().c_str(); } else if (self->props.m_vdevice_key.was_changed()) { auto key_str = std::to_string(self->props.m_vdevice_key.get()); vdevice_params.group_id = key_str.c_str(); @@ -1192,13 +1266,8 @@ static hailo_status gst_hailonet_init_infer_model(GstHailoNet * self) "To use multi-process-service please set scheduling-algorithm to a value other than 'none'"); } - auto vdevice = VDevice::create(vdevice_params); - CHECK_EXPECTED_AS_STATUS(vdevice); - self->vdevice = std::move(vdevice.release()); - - auto infer_model = self->vdevice->create_infer_model(self->props.m_hef_path.get()); - CHECK_EXPECTED_AS_STATUS(infer_model); - self->infer_model = infer_model.release(); + TRY(self->vdevice, VDevice::create(vdevice_params)); + TRY(self->infer_model, self->vdevice->create_infer_model(self->props.m_hef_path.get())); if(!(self->props.m_input_from_meta.get())){ CHECK(self->infer_model->inputs().size() == 1, HAILO_INVALID_OPERATION, @@ -1275,6 +1344,10 @@ static GstCaps *gst_hailonet_get_caps(GstHailoNet *self) return nullptr; } + if (nullptr != self->input_caps) { + return gst_caps_copy(self->input_caps); + } + if (nullptr == self->vdevice) { auto status = gst_hailonet_init_infer_model(self); if (HAILO_SUCCESS != status) { @@ -1283,9 +1356,10 @@ static GstCaps *gst_hailonet_get_caps(GstHailoNet *self) } } - // TODO (HRT-12491): check caps based on incoming metadata if (self->props.m_input_from_meta.get()) { GstCaps *new_caps = gst_caps_new_any(); + std::unique_lock lock(self->input_caps_mutex); + gst_hailonet_unref_input_caps(self); self->input_caps = new_caps; return gst_caps_copy(new_caps); } @@ -1312,6 +1386,8 @@ static GstCaps *gst_hailonet_get_caps(GstHailoNet *self) return nullptr; } + std::unique_lock lock(self->input_caps_mutex); + gst_hailonet_unref_input_caps(self); self->input_caps = new_caps; return gst_caps_copy(new_caps); } @@ -1341,81 +1417,19 @@ static gboolean gst_hailonet_handle_sink_query(GstPad * pad, GstObject * parent, } } -static gboolean gst_hailonet_handle_caps_event(GstHailoNet *self, GstCaps */*caps*/) -{ - if (nullptr == self->input_caps) { - return FALSE; - } - - GstCaps *caps_result = gst_pad_peer_query_caps(self->srcpad, self->input_caps); - if (gst_caps_is_empty(caps_result)) { - return FALSE; - } - - if (gst_caps_is_any(caps_result)) { - gst_caps_unref(caps_result); - return TRUE; - } - - GstCaps *outcaps = gst_caps_fixate(caps_result); - gboolean res = gst_pad_set_caps(self->srcpad, outcaps); - gst_caps_unref(outcaps); - return res; -} - static gboolean gst_hailonet_sink_event(GstPad *pad, GstObject *parent, GstEvent *event) { GstHailoNet *self = GST_HAILONET(parent); - switch (GST_EVENT_TYPE(event)) { - case GST_EVENT_CAPS: - { - GstCaps *caps; - gst_event_parse_caps(event, &caps); - auto result = gst_hailonet_handle_caps_event(self, caps); - gst_event_unref(event); - return result; - } - case GST_EVENT_EOS: + if (GST_EVENT_TYPE(event) == GST_EVENT_EOS) { self->has_got_eos = true; return gst_pad_push_event(self->srcpad, event); - default: - return gst_pad_event_default(pad, parent, event); - } -} - -static GstPadProbeReturn gst_hailonet_sink_probe(GstPad */*pad*/, GstPadProbeInfo */*info*/, gpointer user_data) -{ - GstHailoNet *self = static_cast(user_data); - std::unique_lock lock(self->sink_probe_change_state_mutex); - - auto status = gst_hailonet_configure(self); - if (HAILO_SUCCESS != status) { - return GST_PAD_PROBE_DROP; - } - - status = gst_hailonet_allocate_infer_resources(self); - if (HAILO_SUCCESS != status) { - return GST_PAD_PROBE_DROP; - } - - if (HAILO_SCHEDULING_ALGORITHM_NONE != self->props.m_scheduling_algorithm.get()) { - self->props.m_is_active = true; - return GST_PAD_PROBE_REMOVE; - } - - if ((1 == hailonet_count) && (!self->props.m_is_active.was_changed())) { - self->props.m_is_active = true; } - - if (self->props.m_is_active.get()) { - status = self->configured_infer_model->activate(); - if (HAILO_SUCCESS != status) { - return GST_PAD_PROBE_DROP; - } + if (GST_EVENT_IS_STICKY(event)) { + gst_hailonet_push_event_to_queue(self, event); + return TRUE; + } else { + return gst_pad_event_default(pad, parent, event); } - - self->has_called_activate = true; - return GST_PAD_PROBE_REMOVE; } static void gst_hailonet_flush_callback(GstHailoNet *self, gpointer /*data*/) @@ -1454,18 +1468,8 @@ static void gst_hailonet_init(GstHailoNet *self) self->has_called_activate = false; self->ongoing_frames = 0; self->did_critical_failure_happen = false; - - gchar *parent_name = gst_object_get_name(GST_OBJECT(self)); - gchar *name = g_strconcat(parent_name, ":hailo_allocator", NULL); - g_free(parent_name); - - if (gst_hailo_should_use_dma_buffers()) { - self->dma_allocator = gst_dmabuf_allocator_new(); - } else { - self->allocator = GST_HAILO_ALLOCATOR(g_object_new(GST_TYPE_HAILO_ALLOCATOR, "name", name, NULL)); - gst_object_ref_sink(self->allocator); - g_free(name); - } + self->events_queue_per_buffer = std::unordered_map>(); + self->curr_event_queue = std::queue(); g_signal_connect(self, "flush", G_CALLBACK(gst_hailonet_flush_callback), nullptr); diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp index 26244eb..a47cd4c 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp @@ -30,6 +30,8 @@ #include "hailo/infer_model.hpp" #include "common.hpp" +#include "gsthailo_allocator.hpp" +#include "gsthailo_dmabuf_allocator.hpp" #include #include @@ -40,35 +42,14 @@ using namespace hailort; G_BEGIN_DECLS -#define GST_TYPE_HAILO_ALLOCATOR (gst_hailo_allocator_get_type()) -#define GST_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocator)) -#define GST_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocatorClass)) -#define GST_IS_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_HAILO_ALLOCATOR)) -#define GST_IS_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_HAILO_ALLOCATOR)) - #define MIN_OUTPUTS_POOL_SIZE (MAX_GSTREAMER_BATCH_SIZE) #define MAX_OUTPUTS_POOL_SIZE (MAX_GSTREAMER_BATCH_SIZE * 4) -#define GST_HAILO_USE_DMA_BUFFER_ENV_VAR "GST_HAILO_USE_DMA_BUFFER" - -struct GstHailoAllocator -{ - GstAllocator parent; - std::unordered_map buffers; -}; - -struct GstHailoAllocatorClass -{ - GstAllocatorClass parent; -}; - -GType gst_hailo_allocator_get_type(void); - struct HailoNetProperties final { public: - HailoNetProperties() : m_hef_path(nullptr), m_batch_size(HAILO_DEFAULT_BATCH_SIZE), - m_device_id(nullptr), m_device_count(0), m_vdevice_group_id(nullptr), m_is_active(false), m_pass_through(false), + HailoNetProperties() : m_hef_path(""), m_batch_size(HAILO_DEFAULT_BATCH_SIZE), + m_device_id(""), m_device_count(0), m_vdevice_group_id(""), m_is_active(false), m_pass_through(false), m_outputs_min_pool_size(MIN_OUTPUTS_POOL_SIZE), m_outputs_max_pool_size(MAX_OUTPUTS_POOL_SIZE), m_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN), m_scheduler_timeout_ms(HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS), m_scheduler_threshold(HAILO_DEFAULT_SCHEDULER_THRESHOLD), m_scheduler_priority(HAILO_SCHEDULER_PRIORITY_NORMAL), @@ -78,24 +59,11 @@ public: m_vdevice_key(DEFAULT_VDEVICE_KEY) {} - void free_strings() - { - if (m_hef_path.was_changed()) { - g_free(m_hef_path.get()); - } - if (m_device_id.was_changed()) { - g_free(m_device_id.get()); - } - if (m_vdevice_group_id.was_changed()) { - g_free(m_vdevice_group_id.get()); - } - } - - HailoElemProperty m_hef_path; + HailoElemStringProperty m_hef_path; HailoElemProperty m_batch_size; - HailoElemProperty m_device_id; + HailoElemStringProperty m_device_id; HailoElemProperty m_device_count; - HailoElemProperty m_vdevice_group_id; + HailoElemStringProperty m_vdevice_group_id; HailoElemProperty m_is_active; HailoElemProperty m_pass_through; HailoElemProperty m_outputs_min_pool_size; @@ -119,42 +87,47 @@ public: }; typedef struct _GstHailoNet { - GstElement element; - GstPad *sinkpad; - GstPad *srcpad; - GstQueueArray *input_queue; - GstQueueArray *thread_queue; - std::atomic_uint32_t buffers_in_thread_queue; - std::thread thread; - HailoNetProperties props; - GstCaps *input_caps; - std::atomic_bool is_thread_running; - std::atomic_bool has_got_eos; - std::mutex sink_probe_change_state_mutex; - bool did_critical_failure_happen; - - std::unique_ptr vdevice; - std::shared_ptr infer_model; - std::shared_ptr configured_infer_model; - ConfiguredInferModel::Bindings infer_bindings; - bool is_configured; - std::mutex infer_mutex; - - bool has_called_activate; - std::atomic_uint32_t ongoing_frames; - std::condition_variable flush_cv; - std::mutex flush_mutex; - - GstVideoInfo input_frame_info; - - GstHailoAllocator *allocator; - GstAllocator *dma_allocator; - std::unordered_map output_buffer_pools; - std::unordered_map output_vstream_infos; - - std::mutex input_queue_mutex; - std::mutex thread_queue_mutex; - std::condition_variable thread_cv; + GstElement element; + GstPad *sinkpad; + GstPad *srcpad; + + std::unordered_map> events_queue_per_buffer; + std::queue curr_event_queue; + GstQueueArray *input_queue; + + GstQueueArray *thread_queue; + std::atomic_uint32_t buffers_in_thread_queue; + std::thread thread; + HailoNetProperties props; + GstCaps *input_caps; + std::atomic_bool is_thread_running; + std::atomic_bool has_got_eos; + std::mutex sink_probe_change_state_mutex; + bool did_critical_failure_happen; + + std::unique_ptr vdevice; + std::shared_ptr infer_model; + std::shared_ptr configured_infer_model; + ConfiguredInferModel::Bindings infer_bindings; + bool is_configured; + std::mutex infer_mutex; + + bool has_called_activate; + std::atomic_uint32_t ongoing_frames; + std::condition_variable flush_cv; + std::mutex flush_mutex; + std::mutex input_caps_mutex; + + GstVideoInfo input_frame_info; + + GstHailoAllocator *allocator; + GstHailoDmabufAllocator *dmabuf_allocator; + std::unordered_map output_buffer_pools; + std::unordered_map output_vstream_infos; + + std::mutex input_queue_mutex; + std::mutex thread_queue_mutex; + std::condition_variable thread_cv; } GstHailoNet; typedef struct _GstHailoNetClass { diff --git a/hailort/libhailort/bindings/python/CMakeLists.txt b/hailort/libhailort/bindings/python/CMakeLists.txt index 88f3b8d..1455674 100644 --- a/hailort/libhailort/bindings/python/CMakeLists.txt +++ b/hailort/libhailort/bindings/python/CMakeLists.txt @@ -1,4 +1,3 @@ cmake_minimum_required(VERSION 3.11.0) -include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/pybind11.cmake) add_subdirectory(src) diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py b/hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py index ed8ac13..26cacf4 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py @@ -14,6 +14,7 @@ def join_drivers_path(path): return os.path.join(_ROOT, 'hailo_platform', 'drivers', path) +import hailo_platform.pyhailort._pyhailort as _pyhailort from hailo_platform.tools.udp_rate_limiter import UDPRateLimiter from hailo_platform.pyhailort.hw_object import PcieDevice, EthernetDevice from hailo_platform.pyhailort.pyhailort import (HEF, ConfigureParams, @@ -26,7 +27,7 @@ from hailo_platform.pyhailort.pyhailort import (HEF, ConfigureParams, InputVStreams, OutputVStreams, InferVStreams, HailoStreamDirection, HailoFormatFlags, HailoCpuId, Device, VDevice, DvmTypes, PowerMeasurementTypes, SamplingPeriod, AveragingFactor, MeasurementBufferIndex, - HailoRTException, HailoSchedulingAlgorithm, HailoRTStreamAbortedByUser) + HailoRTException, HailoSchedulingAlgorithm, HailoRTStreamAbortedByUser, AsyncInferJob) def _verify_pyhailort_lib_exists(): python_version = "".join(str(i) for i in sys.version_info[:2]) @@ -41,25 +42,16 @@ def _verify_pyhailort_lib_exists(): _verify_pyhailort_lib_exists() -def get_version(package_name): - # See: https://packaging.python.org/guides/single-sourcing-package-version/ (Option 5) - # We assume that the installed package is actually the same one we import. This assumption may - # break in some edge cases e.g. if the user modifies sys.path manually. - - # hailo_platform package has been renamed to hailort, but the import is still hailo_platform - if package_name == "hailo_platform": - package_name = "hailort" - try: - import pkg_resources - return pkg_resources.get_distribution(package_name).version - except: - return 'unknown' +__version__ = "4.18.0" +if _pyhailort.__version__ != __version__: + raise ImportError( + f"_pyhailort version ({_pyhailort.__version__}) does not match pyhailort version ({__version__})" + ) -__version__ = get_version('hailo_platform') __all__ = ['EthernetDevice', 'DvmTypes', 'PowerMeasurementTypes', 'SamplingPeriod', 'AveragingFactor', 'MeasurementBufferIndex', 'UDPRateLimiter', 'PcieDevice', 'HEF', 'ConfigureParams', 'FormatType', 'FormatOrder', 'MipiDataTypeRx', 'MipiPixelsPerClock', 'MipiClockSelection', 'MipiIspImageInOrder', 'MipiIspImageOutDataType', 'join_drivers_path', 'IspLightFrequency', 'HailoPowerMode', 'Endianness', 'HailoStreamInterface', 'InputVStreamParams', 'OutputVStreamParams', 'InputVStreams', 'OutputVStreams', 'InferVStreams', 'HailoStreamDirection', 'HailoFormatFlags', 'HailoCpuId', - 'Device', 'VDevice', 'HailoRTException', 'HailoSchedulingAlgorithm', 'HailoRTStreamAbortedByUser'] + 'Device', 'VDevice', 'HailoRTException', 'HailoSchedulingAlgorithm', 'HailoRTStreamAbortedByUser', 'AsyncInferJob'] diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py b/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py index 4b71b92..858dbde 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py @@ -1,22 +1,20 @@ from enum import Enum, IntEnum import signal import struct -import pkg_resources -# hailo_platform package has been renamed to hailort, but the import is still hailo_platform -__version__ = pkg_resources.get_distribution("hailort").version import sys +from collections import deque +from dataclasses import dataclass from argparse import ArgumentTypeError +from datetime import timedelta import numpy import time -from hailo_platform.common.logger.logger import default_logger import gc import os +from hailo_platform.common.logger.logger import default_logger import hailo_platform.pyhailort._pyhailort as _pyhailort -if _pyhailort.__version__ != __version__: - raise ImportError("_pyhailort version ({}) does not match pyhailort version ({})".format(_pyhailort.__version__, __version__)) from hailo_platform.pyhailort._pyhailort import (TemperatureInfo, # noqa F401 DvmTypes, PowerMeasurementTypes, # noqa F401 @@ -47,6 +45,7 @@ class HailoSocket(object): MIN_UDP_PADDED_PAYLOAD_SIZE = HailoSocketDefs.MIN_UDP_PADDED_PAYLOAD_SIZE() MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP = HailoSocketDefs.MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP() + class HailoSchedulingAlgorithm(_pyhailort.SchedulingAlgorithm): pass @@ -112,43 +111,49 @@ class ExceptionWrapper(object): else: raise - def _raise_indicative_status_exception(self, libhailort_exception): - error_code = int(libhailort_exception.args[0]) + @staticmethod + def create_exception_from_status(error_code): string_error_code = get_status_message(error_code) if string_error_code == "HAILO_ETH_RECV_FAILURE": - raise UdpRecvError("Failed to receive data") from libhailort_exception + return UdpRecvError("Failed to receive data") if string_error_code == "HAILO_UNSUPPORTED_CONTROL_PROTOCOL_VERSION": - raise InvalidProtocolVersionException("HailoRT has failed because an invalid protocol version was received from device") from libhailort_exception + return InvalidProtocolVersionException("HailoRT has failed because an invalid protocol version was received from device") if string_error_code == "HAILO_FW_CONTROL_FAILURE": - raise HailoRTFirmwareControlFailedException("libhailort control operation failed") from libhailort_exception + return HailoRTFirmwareControlFailedException("libhailort control operation failed") if string_error_code == "HAILO_UNSUPPORTED_OPCODE": - raise HailoRTUnsupportedOpcodeException("HailoRT has failed because an unsupported opcode was sent to device") from libhailort_exception + return HailoRTUnsupportedOpcodeException("HailoRT has failed because an unsupported opcode was sent to device") if string_error_code == "HAILO_INVALID_FRAME": - raise HailoRTInvalidFrameException("An invalid frame was received") from libhailort_exception + return HailoRTInvalidFrameException("An invalid frame was received") if string_error_code == "HAILO_TIMEOUT": - raise HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred") from libhailort_exception + return HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred") if string_error_code == "HAILO_STREAM_ABORT": - raise HailoRTStreamAborted("Stream was aborted") from libhailort_exception + return HailoRTStreamAborted("Stream was aborted") if string_error_code == "HAILO_INVALID_OPERATION": - raise HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information") from libhailort_exception + return HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information") if string_error_code == "HAILO_INVALID_ARGUMENT": - raise HailoRTInvalidArgumentException("Invalid argument. See hailort.log for more information") from libhailort_exception + return HailoRTInvalidArgumentException("Invalid argument. See hailort.log for more information") if string_error_code == "HAILO_NOT_FOUND": - raise HailoRTNotFoundException("Item not found. See hailort.log for more information") from libhailort_exception + return HailoRTNotFoundException("Item not found. See hailort.log for more information") if string_error_code == "HAILO_INVALID_HEF": - raise HailoRTInvalidHEFException("Invalid HEF. See hailort.log for more information") from libhailort_exception + return HailoRTInvalidHEFException("Invalid HEF. See hailort.log for more information") if string_error_code == "HAILO_ETH_FAILURE": - raise HailoRTEthException("Ethernet failure. See hailort.log for more information") from libhailort_exception + return HailoRTEthException("Ethernet failure. See hailort.log for more information") if string_error_code == "HAILO_DRIVER_FAIL": - raise HailoRTPCIeDriverException("PCIe driver failure. run 'dmesg | grep hailo' for more information") from libhailort_exception + return HailoRTPCIeDriverException("PCIe driver failure. run 'dmesg | grep hailo' for more information") if string_error_code == "HAILO_NETWORK_GROUP_NOT_ACTIVATED": - raise HailoRTNetworkGroupNotActivatedException("Network group is not activated") from libhailort_exception + return HailoRTNetworkGroupNotActivatedException("Network group is not activated") else: - raise HailoRTException("libhailort failed with error: {} ({})".format(error_code, string_error_code)) from libhailort_exception + return HailoRTException("libhailort failed with error: {} ({})".format(error_code, string_error_code)) + + + def _raise_indicative_status_exception(self, libhailort_exception): + error_code = int(libhailort_exception.args[0]) + raise self.create_exception_from_status(error_code) from libhailort_exception + def get_status_message(status_code): status_str = _pyhailort.get_status_message(status_code) @@ -720,7 +725,7 @@ class ConfiguredNetwork(object): def set_scheduler_timeout(self, timeout_ms, network_name=None): """Sets the maximum time period that may pass before receiving run time from the scheduler. This will occur providing at least one send request has been sent, there is no minimum requirement for send - requests, (e.g. threshold - see set_scheduler_threshold()). + requests, (e.g. threshold - see :func:`ConfiguredNetwork.set_scheduler_threshold`). Args: timeout_ms (int): Timeout in milliseconds. @@ -730,7 +735,7 @@ class ConfiguredNetwork(object): def set_scheduler_threshold(self, threshold): """Sets the minimum number of send requests required before the network is considered ready to get run time from the scheduler. - If at least one send request has been sent, but the threshold is not reached within a set time period (e.g. timeout - see hailo_set_scheduler_timeout()), + If at least one send request has been sent, but the threshold is not reached within a set time period (e.g. timeout - see :func:`ConfiguredNetwork.set_scheduler_timeout`), the scheduler will consider the network ready regardless. Args: @@ -748,6 +753,15 @@ class ConfiguredNetwork(object): """ return self._configured_network.set_scheduler_priority(priority) + def init_cache(self, read_offset, write_offset_delta): + return self._configured_network.init_cache(read_offset, write_offset_delta) + + def get_cache_info(self): + return self._configured_network.get_cache_info() + + def update_cache_offset(self, offset_delta_bytes): + return self._configured_network.update_cache_offset(offset_delta_bytes) + class EmptyContextManager(object): """An empty context manager that returns instead of activated network group when scheduler is enabled`.""" @@ -914,11 +928,11 @@ class InferVStreams(object): time_before_infer_calcs = time.perf_counter() if not isinstance(input_data, dict): - input_stream_infos = self._configured_net_group.get_input_stream_infos() - if len(input_stream_infos) != 1: + input_vstream_infos = self._configured_net_group.get_input_vstream_infos() + if len(input_vstream_infos) != 1: raise Exception("when there is more than one input, the input_data should be of type dict," - " mapping between each input_name, and his input_data tensor. number of inputs: {}".format(len(input_stream_infos))) - input_data = {input_stream_infos[0].name : input_data} + " mapping between each input_name, and his input_data tensor. number of inputs: {}".format(len(input_vstream_infos))) + input_data = {input_vstream_infos[0].name : input_data} batch_size = InferVStreams._get_number_of_frames(input_data) output_buffers, output_buffers_info = self._make_output_buffers_and_infos(input_data, batch_size) @@ -939,11 +953,11 @@ class InferVStreams(object): if output_buffers_info[name].output_order == FormatOrder.HAILO_NMS_WITH_BYTE_MASK: nms_shape = output_buffers_info[name].vstream_info.nms_shape output_dtype = output_buffers_info[name].output_dtype - input_stream_infos = self._configured_net_group.get_input_stream_infos() - if len(input_stream_infos) != 1: - raise Exception("Output format HAILO_NMS_WITH_BYTE_MASK should have 1 input. Number of inputs: {}".format(len(input_stream_infos))) - input_height = input_stream_infos[0].shape[0] - input_width = input_stream_infos[0].shape[1] + input_vstream_infos = self._configured_net_group.get_input_vstream_infos() + if len(input_vstream_infos) != 1: + raise Exception("Output format HAILO_NMS_WITH_BYTE_MASK should have 1 input. Number of inputs: {}".format(len(input_vstream_infos))) + input_height = input_vstream_infos[0].shape[0] + input_width = input_vstream_infos[0].shape[1] output_buffers[name] = HailoRTTransformUtils._output_raw_buffer_to_nms_with_byte_mask_format(result_array, nms_shape.number_of_classes, batch_size, input_height, input_width, nms_shape.max_bboxes_per_class, output_dtype, self._tf_nms_format) @@ -1499,8 +1513,8 @@ class HailoFormatFlags(_pyhailort.FormatFlags): SUPPORTED_PROTOCOL_VERSION = 2 SUPPORTED_FW_MAJOR = 4 -SUPPORTED_FW_MINOR = 17 -SUPPORTED_FW_REVISION = 1 +SUPPORTED_FW_MINOR = 18 +SUPPORTED_FW_REVISION = 0 MEGA_MULTIPLIER = 1000.0 * 1000.0 @@ -1511,6 +1525,8 @@ class DeviceArchitectureTypes(IntEnum): HAILO8L = 2 HAILO15H = 3 PLUTO = 4 + HAILO15M = 5 + HAILO10H = 6 def __str__(self): return self.name @@ -1566,8 +1582,11 @@ class BoardInformation(object): if ((device_arch == DeviceArchitectureTypes.HAILO8) or (device_arch == DeviceArchitectureTypes.HAILO8L)): return 'hailo8' - elif device_arch == DeviceArchitectureTypes.HAILO15H: + elif ((device_arch == DeviceArchitectureTypes.HAILO15H) or + (device_arch == DeviceArchitectureTypes.HAILO15M)): return 'hailo15' + elif (device_arch == DeviceArchitectureTypes.HAILO10H): + return 'hailo10' else: raise HailoRTException("Unsupported device architecture.") @@ -2442,6 +2461,18 @@ class Control: with ExceptionWrapper(): return self._device.remove_notification_callback(notification_id) + def _init_cache_info(self, cache_info): + with ExceptionWrapper(): + return self._device._init_cache_info(cache_info) + + def _get_cache_info(self): + with ExceptionWrapper(): + return self._device._get_cache_info() + + def _update_cache_read_offset(self, read_offset_delta): + with ExceptionWrapper(): + return self._device._update_cache_read_offset(read_offset_delta) + def _get_device_handle(self): return self._device @@ -2573,6 +2604,835 @@ class Device: return self._loaded_network_groups[0] +class AsyncInferCompletionInfo: + """ + Holds information about the async infer job + """ + + def __init__(self, exception): + """ + Args: + exception (:obj:`HailoRTException`): an exception corresponding to the error that happened inside the async infer job. + """ + self._exception = exception + + @property + def exception(self): + """ + Returns the exception that was set on this Infer job. if the job finished succesfully, returns None. + """ + return self._exception + + +class InferModel: + """ + Contains all of the necessary information for configuring the network for inference. + This class is used to set up the model for inference and includes methods for setting and getting the model's parameters. + By calling the configure function, the user can create a :obj:`ConfiguredInferModel` object, which is used to run inference. + """ + class InferStream: + """ + Represents the parameters of a stream. + In default, the stream's parameters are set to the default values of the model. + The user can change the stream's parameters by calling the setter functions. + """ + def __init__(self, infer_stream): + #""" + #Args: + # infer_stream (_pyhailort.InferStream): The C++ InferStream object. + #""" + self._infer_stream = infer_stream + + @property + def name(self): + """ + Returns: + name (str): the name of the edge. + """ + with ExceptionWrapper(): + return self._infer_stream.name() + + @property + def shape(self): + """ + Returns: + shape (list[int]): the shape of the edge. + """ + with ExceptionWrapper(): + return self._infer_stream.shape() + + @property + def format(self): + """ + Returns: + format (_pyhailort.hailo_format_t): the format of the edge. + """ + with ExceptionWrapper(): + return self._infer_stream.format() + + def set_format_type(self, type): + """ + Set the format type of the stream. + + Args: + type (_pyhailort.hailo_format_type_t): the format type + """ + with ExceptionWrapper(): + self._infer_stream.set_format_type(type) + + def set_format_order(self, order): + """ + Set the format order of the stream. + + Args: + order (_pyhailort.hailo_format_order_t): the format order + """ + with ExceptionWrapper(): + self._infer_stream.set_format_order(order) + + @property + def quant_infos(self): + """ + Returns: + quant_infos (list[_pyhailort.hailo_quant_info_t]): List of the quantization information of the edge. + """ + with ExceptionWrapper(): + return self._infer_stream.get_quant_infos() + + @property + def is_nms(self): + """ + Returns: + is_nms (bool): whether the stream is NMS. + """ + with ExceptionWrapper(): + return self._infer_stream.is_nms() + + def set_nms_score_threshold(self, threshold): + """ + Set NMS score threshold, used for filtering out candidates. Any box with score ${PROJECT_SOURCE_DIR}/platform_internals/hailo_platform_internals/pyhailort/ - ) - add_dependencies(pyhailort_internal_venv _pyhailort_internal) -endif() - # TODO (HRT-8637): change this hard-coded path set(HAILO_PYHAILORT_TARGET_DIR ${CMAKE_CURRENT_LIST_DIR}/../platform/hailo_platform/pyhailort/) -# copy files to a path the venv and whl will look for -message(STATUS "Copying _pyhailort artifacts into " ${HAILO_PYHAILORT_TARGET_DIR}) add_custom_target(pyhailort_venv ALL COMMAND ${CMAKE_COMMAND} -E copy $ ${HAILO_PYHAILORT_TARGET_DIR} ) @@ -86,4 +109,5 @@ add_dependencies(pyhailort_venv _pyhailort) install(TARGETS _pyhailort LIBRARY DESTINATION ${HAILO_PYHAILORT_TARGET_DIR} CONFIGURATIONS Release -) \ No newline at end of file +) + diff --git a/hailort/libhailort/bindings/python/src/bindings_common.hpp b/hailort/libhailort/bindings/python/src/bindings_common.hpp index 2367af4..1dcf5c1 100644 --- a/hailort/libhailort/bindings/python/src/bindings_common.hpp +++ b/hailort/libhailort/bindings/python/src/bindings_common.hpp @@ -38,18 +38,17 @@ public: } } - static std::vector get_pybind_shape(const hailo_vstream_info_t& vstream_info, const hailo_format_t &user_format) + static std::vector get_pybind_shape( + const hailo_3d_image_shape_t &shape, + const hailo_nms_shape_t &nms_shape, + const hailo_format_t &user_format) { - // We are using user_format instead of hw format inside the vstream_info - const auto shape = vstream_info.shape; - // TODO: support no transformations (i.e. use stream_info.hw_shape) (SDK-16811) switch (user_format.order) { case HAILO_FORMAT_ORDER_HAILO_NMS: - return { HailoRTCommon::get_nms_host_shape_size(vstream_info.nms_shape) }; - case HAILO_FORMAT_ORDER_HAILO_NMS_WITH_BYTE_MASK: { - return {HailoRTCommon::get_nms_host_frame_size(vstream_info.nms_shape, user_format) / HailoRTCommon::get_format_data_bytes(user_format)}; - } + return { HailoRTCommon::get_nms_host_shape_size(nms_shape) }; + case HAILO_FORMAT_ORDER_HAILO_NMS_WITH_BYTE_MASK: + return {HailoRTCommon::get_nms_host_frame_size(nms_shape, user_format) / HailoRTCommon::get_format_data_bytes(user_format)}; case HAILO_FORMAT_ORDER_NC: return {shape.features}; case HAILO_FORMAT_ORDER_NHW: diff --git a/hailort/libhailort/bindings/python/src/device_api.cpp b/hailort/libhailort/bindings/python/src/device_api.cpp index 529322e..5399c1c 100644 --- a/hailort/libhailort/bindings/python/src/device_api.cpp +++ b/hailort/libhailort/bindings/python/src/device_api.cpp @@ -395,7 +395,7 @@ hailo_chip_temperature_info_t DeviceWrapper::get_chip_temperature() void DeviceWrapper::set_notification_callback(const std::function &callback, hailo_notification_id_t notification_id, py::object opaque) { - // we capture opaque and move it because when opaque goes out of score it will be deleted, + // capture the opaque and move it, this is because when opaque goes out of scope it will be automatically deleted, // so capturing it ensures that it will not be deleted hailo_status status = device().set_notification_callback( [callback, op = std::move(opaque)] (Device &device, const hailo_notification_t ¬ification, void* opaque) { @@ -459,7 +459,27 @@ void DeviceWrapper::set_sleep_state(hailo_sleep_state_t sleep_state) VALIDATE_STATUS(status); } -void DeviceWrapper::add_to_python_module(py::module &m) +void DeviceWrapper::_init_cache_info(const hailo_cache_info_t &cache_info) +{ + auto status = device().init_cache_info(cache_info); + VALIDATE_STATUS(status); +} + +hailo_cache_info_t DeviceWrapper::_get_cache_info() +{ + auto cache_info = device().get_cache_info(); + VALIDATE_EXPECTED(cache_info); + + return cache_info.release(); +} + +void DeviceWrapper::_update_cache_read_offset(int32_t read_offset_delta) +{ + auto status = device().update_cache_read_offset(read_offset_delta); + VALIDATE_STATUS(status); +} + +void DeviceWrapper::bind(py::module &m) { py::class_(m, "Device") .def("is_valid", &DeviceWrapper::is_valid) @@ -526,6 +546,9 @@ void DeviceWrapper::add_to_python_module(py::module &m) .def("set_notification_callback", &DeviceWrapper::set_notification_callback) .def("remove_notification_callback", &DeviceWrapper::remove_notification_callback) .def("set_sleep_state", &DeviceWrapper::set_sleep_state) + .def("_init_cache_info", &DeviceWrapper::_init_cache_info) + .def("_get_cache_info", &DeviceWrapper::_get_cache_info) + .def("_update_cache_read_offset", &DeviceWrapper::_update_cache_read_offset) ; } diff --git a/hailort/libhailort/bindings/python/src/device_api.hpp b/hailort/libhailort/bindings/python/src/device_api.hpp index 5024f28..38c2fc1 100644 --- a/hailort/libhailort/bindings/python/src/device_api.hpp +++ b/hailort/libhailort/bindings/python/src/device_api.hpp @@ -133,8 +133,11 @@ public: py::bytes direct_read_memory(uint32_t address, uint32_t size); const char *get_dev_id() const; void set_sleep_state(hailo_sleep_state_t sleep_state); + void _init_cache_info(const hailo_cache_info_t &cache_info); + hailo_cache_info_t _get_cache_info(); + void _update_cache_read_offset(int32_t read_offset_delta); - static void add_to_python_module(py::module &m); + static void bind(py::module &m); private: DeviceWrapper(std::unique_ptr &&device) diff --git a/hailort/libhailort/bindings/python/src/hef_api.cpp b/hailort/libhailort/bindings/python/src/hef_api.cpp index 9d5fa2d..8b5d248 100644 --- a/hailort/libhailort/bindings/python/src/hef_api.cpp +++ b/hailort/libhailort/bindings/python/src/hef_api.cpp @@ -37,9 +37,12 @@ HefWrapper::HefWrapper(const MemoryView &hef_buffer) } } -HefWrapper HefWrapper::create_from_buffer(py::bytes data) +HefWrapper HefWrapper::create_from_buffer(const py::bytes &data) { - return HefWrapper(MemoryView((uint8_t*)std::string(data).c_str(), std::string(data).size())); + // TODO: HRT-13713 - When adding support to read the hef from pre-allocated memory, + // we will need to make sure the hef memory is not released here. + py::buffer_info info(py::buffer(data).request()); + return HefWrapper(MemoryView::create_const(info.ptr, info.size)); } HefWrapper HefWrapper::create_from_file(const std::string &hef_path) @@ -183,7 +186,7 @@ py::list HefWrapper::get_networks_names(const std::string &net_group_name) return py::cast(res); } -void HefWrapper::initialize_python_module(py::module &m) +void HefWrapper::bind(py::module &m) { py::class_(m, "Hef") .def("create_from_buffer", &HefWrapper::create_from_buffer) diff --git a/hailort/libhailort/bindings/python/src/hef_api.hpp b/hailort/libhailort/bindings/python/src/hef_api.hpp index 47f49e9..5e2ddf2 100644 --- a/hailort/libhailort/bindings/python/src/hef_api.hpp +++ b/hailort/libhailort/bindings/python/src/hef_api.hpp @@ -33,7 +33,7 @@ class HefWrapper { public: HefWrapper(const std::string &hef_path); HefWrapper(const MemoryView &hef_buffer); - static HefWrapper create_from_buffer(py::bytes data); + static HefWrapper create_from_buffer(const py::bytes &data); static HefWrapper create_from_file(const std::string &hef_path); py::list get_network_group_names(); py::list get_network_groups_infos(); @@ -60,7 +60,7 @@ public: py::dict create_configure_params_mipi_input(hailo_stream_interface_t output_interface, const hailo_mipi_input_stream_params_t &mipi_params); py::list get_networks_names(const std::string &net_group_name); - static void initialize_python_module(py::module &m); + static void bind(py::module &m); private: std::unique_ptr hef; diff --git a/hailort/libhailort/bindings/python/src/infer_model_api.cpp b/hailort/libhailort/bindings/python/src/infer_model_api.cpp new file mode 100644 index 0000000..6719db7 --- /dev/null +++ b/hailort/libhailort/bindings/python/src/infer_model_api.cpp @@ -0,0 +1,477 @@ +/** + * Copyright (c) 2020-2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file infer_model_api.cpp + * @brief Defines binding to a infer model class family usage over Python. + **/ +#include "infer_model_api.hpp" +#include "bindings_common.hpp" +#include "hailo/infer_model.hpp" + +#include +#include +#include +#include +#include +#include +#include // py::gil_scoped_release +#include // handle std::vector +#include // handle std::function +#include // handle std::chrono::milliseconds + +using namespace hailort; + +void InferModelWrapper::set_batch_size(uint16_t batch_size) +{ + m_infer_model->set_batch_size(batch_size); +} + +void InferModelWrapper::set_power_mode(hailo_power_mode_t power_mode) +{ + m_infer_model->set_power_mode(power_mode); +} + +std::vector InferModelWrapper::inputs() +{ + auto infer_streams = m_infer_model->inputs(); + std::vector wrappers; + for (auto &infer_stream : infer_streams) + { + wrappers.push_back(InferModelInferStreamWrapper(std::move(infer_stream))); + } + return wrappers; +} + +std::vector InferModelWrapper::outputs() +{ + auto infer_streams = m_infer_model->outputs(); + std::vector wrappers; + for (auto &infer_stream : infer_streams) + { + wrappers.push_back(InferModelInferStreamWrapper(std::move(infer_stream))); + } + return wrappers; +} + +ConfiguredInferModelWrapper InferModelWrapper::configure() +{ + auto configured_infer_model = m_infer_model->configure(); + VALIDATE_EXPECTED(configured_infer_model); + return ConfiguredInferModelWrapper(configured_infer_model.release(), m_is_using_service, + m_infer_model->get_output_names()); +} + +std::vector InferModelWrapper::get_input_names() +{ + return m_infer_model->get_input_names(); +} + +std::vector InferModelWrapper::get_output_names() +{ + return m_infer_model->get_output_names(); +} + +InferModelInferStreamWrapper InferModelWrapper::input(const std::string &name) +{ + auto infer_stream = name.empty() ? m_infer_model->input() : m_infer_model->input(name); + VALIDATE_EXPECTED(infer_stream); + return InferModelInferStreamWrapper(infer_stream.release()); +} + +InferModelInferStreamWrapper InferModelWrapper::output(const std::string &name) +{ + auto infer_stream = name.empty() ? m_infer_model->output() : m_infer_model->output(name); + VALIDATE_EXPECTED(infer_stream); + return InferModelInferStreamWrapper(infer_stream.release()); +} + +ConfiguredInferModelBindingsWrapper ConfiguredInferModelWrapper::create_bindings() +{ + auto bindings = m_configured_infer_model.create_bindings(); + VALIDATE_EXPECTED(bindings); + return ConfiguredInferModelBindingsWrapper(bindings.release(), m_output_names); +} + +void ConfiguredInferModelWrapper::activate() +{ + auto status = m_configured_infer_model.activate(); + VALIDATE_STATUS(status); +} + +void ConfiguredInferModelWrapper::deactivate() +{ + auto status = m_configured_infer_model.deactivate(); + VALIDATE_STATUS(status); +} + +ConfiguredInferModelBindingsInferStreamWrapper ConfiguredInferModelBindingsWrapper::input(const std::string &name) +{ + auto infer_stream = name.empty() ? m_bindings.input() : m_bindings.input(name); + VALIDATE_EXPECTED(infer_stream); + return ConfiguredInferModelBindingsInferStreamWrapper(infer_stream.release()); +} + +ConfiguredInferModelBindingsInferStreamWrapper ConfiguredInferModelBindingsWrapper::output(const std::string &name) +{ + auto infer_stream = name.empty() ? m_bindings.output() : m_bindings.output(name); + VALIDATE_EXPECTED(infer_stream); + return ConfiguredInferModelBindingsInferStreamWrapper(infer_stream.release()); +} + +const std::string InferModelInferStreamWrapper::name() const +{ + return m_infer_stream.name(); +} + +void InferModelInferStreamWrapper::set_format_type(hailo_format_type_t type) +{ + m_infer_stream.set_format_type(type); +} + +void InferModelInferStreamWrapper::set_format_order(hailo_format_order_t order) +{ + m_infer_stream.set_format_order(order); +} + +std::vector InferModelInferStreamWrapper::get_quant_infos() const +{ + return m_infer_stream.get_quant_infos(); +} + +std::vector InferModelInferStreamWrapper::shape() const +{ + auto shape = m_infer_stream.shape(); + auto format = m_infer_stream.format(); + hailo_nms_shape_t nms_shape; // if the format is non-NMS, this struct won't be used + + if (HailoRTCommon::is_nms(format.order)) + { + auto expected = m_infer_stream.get_nms_shape(); + VALIDATE_EXPECTED(expected); + nms_shape = expected.release(); + } + + return HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format); +} + +hailo_format_t InferModelInferStreamWrapper::format() const +{ + return m_infer_stream.format(); +} + +bool InferModelInferStreamWrapper::is_nms() const +{ + return m_infer_stream.is_nms(); +} + +void InferModelInferStreamWrapper::set_nms_score_threshold(float32_t threshold) +{ + m_infer_stream.set_nms_score_threshold(threshold); +} + +void InferModelInferStreamWrapper::set_nms_iou_threshold(float32_t threshold) +{ + m_infer_stream.set_nms_iou_threshold(threshold); +} + +void InferModelInferStreamWrapper::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class) +{ + m_infer_stream.set_nms_max_proposals_per_class(max_proposals_per_class); +} + +void InferModelInferStreamWrapper::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size) +{ + m_infer_stream.set_nms_max_accumulated_mask_size(max_accumulated_mask_size); +} + +void ConfiguredInferModelBindingsInferStreamWrapper::set_buffer(py::array buffer) +{ + MemoryView view(buffer.mutable_data(), static_cast(buffer.nbytes())); + auto status = m_infer_stream.set_buffer(view); + VALIDATE_STATUS(status); +} + +void ConfiguredInferModelWrapper::wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count) +{ + auto status = m_configured_infer_model.wait_for_async_ready(timeout, frames_count); + VALIDATE_STATUS(status); +} + +void ConfiguredInferModelWrapper::run( + ConfiguredInferModelBindingsWrapper bindings, + std::chrono::milliseconds timeout) +{ + auto status = m_configured_infer_model.run(bindings.get(), timeout); + VALIDATE_STATUS(status); +} + +AsyncInferJobWrapper ConfiguredInferModelWrapper::run_async( + std::vector &user_bindings, + AsyncInferCallBack pythonic_cb) +{ + std::function cb; + + // create an event that will be held by AsyncInferJobWrapper to know if the callback has been executed + auto is_callback_done_expected = Event::create_shared(Event::State::not_signalled); + VALIDATE_EXPECTED(is_callback_done_expected); + auto is_callback_done = is_callback_done_expected.release(); + + if (!m_is_using_service) + { + // a worker thread will handle the user callbacks + // this function should register the callback and notify the thread once the callback is ready to be executed + if(!m_callbacks_thread) + { + m_callbacks_thread = std::make_shared(&ConfiguredInferModelWrapper::execute_callbacks, this); + } + + // the pythonic_cb (i.e. user-defined callback) will signal once the callback has been executed + auto pythonic_cb_wrapper = [pythonic_cb, is_callback_done](const int error_code) + { + pythonic_cb(error_code); + is_callback_done->signal(); + }; + + // enqueue the pythonic_cb (i.e. user-defined callback) only when libhailort's + // async job is done. Once enqueued, the job will be dequeued by the worker thread + cb = [this, pythonic_cb_wrapper](const AsyncInferCompletionInfo &info) + { + { + std::lock_guard lock(m_queue_mutex); + // push the callback and the status to the queue. If the status is not HAILO_SUCCESS, + // the user-defined callback will be get an indication of the failure + m_callbacks_queue->push(std::make_pair(pythonic_cb_wrapper, info)); + } + m_cv.notify_one(); + }; + } + else + { + // the user callbacks will be called from libahilort, and this object won't have a worker thread to notify + cb = [pythonic_cb, is_callback_done](const AsyncInferCompletionInfo &info) + { + pythonic_cb(info.status); + is_callback_done->signal(); + }; + } + + std::vector bindings; + std::transform(user_bindings.begin(), user_bindings.end(), std::back_inserter(bindings), + [](ConfiguredInferModelBindingsWrapper &wrapper) { return wrapper.get(); }); + + std::vector user_output_buffers; + std::vector aligned_output_buffers; + for (auto &binding : bindings) + { + for (const auto &name : m_output_names) + { + auto stream = binding.output(name); + VALIDATE_EXPECTED(stream); + + auto buffer = stream->get_buffer(); + VALIDATE_EXPECTED(buffer); + + user_output_buffers.push_back(buffer->data()); + + auto aligned_buffer = Buffer::create_shared(buffer->size(), BufferStorageParams::create_dma()); + VALIDATE_EXPECTED(aligned_buffer); + + auto buf = aligned_buffer.release(); + aligned_output_buffers.push_back(buf); + + auto status = stream->set_buffer(MemoryView(buf->data(), buf->size())); + VALIDATE_STATUS(status); + } + } + + auto cb_wrapper_with_output_copy = [cb, user_output_buffers, aligned_output_buffers](const AsyncInferCompletionInfo &info) + { + for (size_t i = 0; i < user_output_buffers.size(); i++) + { + std::memcpy(user_output_buffers[i], aligned_output_buffers[i]->data(), aligned_output_buffers[i]->size()); + } + + cb(info); + }; + + auto job = m_configured_infer_model.run_async(bindings, cb_wrapper_with_output_copy); + VALIDATE_EXPECTED(job); + + // don't wait for the job at this location. The user should call job.wait() from the python side + job->detach(); + + return AsyncInferJobWrapper(job.release(), is_callback_done); +} + +void ConfiguredInferModelWrapper::set_scheduler_timeout(const std::chrono::milliseconds &timeout) +{ + auto status = m_configured_infer_model.set_scheduler_timeout(timeout); + VALIDATE_STATUS(status); +} + +void ConfiguredInferModelWrapper::set_scheduler_threshold(uint32_t threshold) +{ + auto status = m_configured_infer_model.set_scheduler_threshold(threshold); + VALIDATE_STATUS(status); +} + +void ConfiguredInferModelWrapper::set_scheduler_priority(uint8_t priority) +{ + auto status = m_configured_infer_model.set_scheduler_priority(priority); + VALIDATE_STATUS(status); +} + +size_t ConfiguredInferModelWrapper::get_async_queue_size() +{ + auto size = m_configured_infer_model.get_async_queue_size(); + VALIDATE_EXPECTED(size); + return size.release(); +} + +void ConfiguredInferModelWrapper::shutdown() +{ + auto status = m_configured_infer_model.shutdown(); + VALIDATE_STATUS(status); +} + +void ConfiguredInferModelWrapper::execute_callbacks() +{ + while (true) + { + std::unique_lock lock(m_queue_mutex); + m_cv.wait_for(lock, std::chrono::minutes(1), [this](){ return !m_callbacks_queue->empty() || !m_is_alive.load(); }); + + if (!m_is_alive.load()) + { + while (!m_callbacks_queue->empty()) { + auto cb_status_pair = m_callbacks_queue->front(); + auto &cb = cb_status_pair.first; + auto status = cb_status_pair.second; + cb(status.status); + m_callbacks_queue->pop(); + } + return; + } + + auto cb_status_pair = m_callbacks_queue->front(); + + m_callbacks_queue->pop(); + lock.unlock(); // release the lock before calling the callback, allowing other threads to push to the queue + + auto &cb = cb_status_pair.first; + auto status = cb_status_pair.second; + cb(status.status); + } +} + +void AsyncInferJobWrapper::wait(std::chrono::milliseconds timeout) +{ + // TODO: currently waiting for 2 TIMEOUT (worst case). Fix it + auto status = m_job.wait(timeout); + VALIDATE_STATUS(status); + + status = m_is_callback_done->wait(timeout); + VALIDATE_STATUS(status); +} + +void InferModelWrapper::bind(py::module &m) +{ + py::class_< + InferModelWrapper, + std::shared_ptr + >(m, "InferModel") + //.def("hef", &InferModelWrapper::hef) + .def("configure", &InferModelWrapper::configure) + .def("set_batch_size", &InferModelWrapper::set_batch_size) + .def("set_power_mode", &InferModelWrapper::set_power_mode) + .def("get_input_names", &InferModelWrapper::get_input_names) + .def("get_output_names", &InferModelWrapper::get_output_names) + .def("inputs", &InferModelWrapper::inputs) + .def("outputs", &InferModelWrapper::outputs) + .def("input", &InferModelWrapper::input) + .def("output", &InferModelWrapper::output) + ; +} + +void ConfiguredInferModelWrapper::bind(py::module &m) +{ + py::class_< + ConfiguredInferModelWrapper, + std::shared_ptr + >(m, "ConfiguredInferModel") + .def("create_bindings", &ConfiguredInferModelWrapper::create_bindings) + .def("activate", &ConfiguredInferModelWrapper::activate) + .def("deactivate", &ConfiguredInferModelWrapper::deactivate) + // wait_for_async_ready is a blocking call which could take a long time, and in the meantime, the pythonic + // callback that was passed to run_async might be called. Therefore, the GIL before calling must be released. + // This is done in order to avoid any further deadlock. Any c++ function that is called from python and that might + // be still running while another thread is trying to execute a python callback, should be able to release the GIL. + .def("wait_for_async_ready", &ConfiguredInferModelWrapper::wait_for_async_ready, py::call_guard()) + .def("run", &ConfiguredInferModelWrapper::run) + // run_async is not calling python callbacks*, so there is no need to keep the GIL while calling. + // Releasing the GIL before calling run_async will allow the callbacks already registered to be called. + // * callbacks will be called from another thread, and will acquire the GIL by themselves + .def("run_async", &ConfiguredInferModelWrapper::run_async, py::call_guard()) + .def("set_scheduler_timeout", &ConfiguredInferModelWrapper::set_scheduler_timeout) + .def("set_scheduler_threshold", &ConfiguredInferModelWrapper::set_scheduler_threshold) + .def("set_scheduler_priority", &ConfiguredInferModelWrapper::set_scheduler_priority) + .def("get_async_queue_size", &ConfiguredInferModelWrapper::get_async_queue_size) + .def("shutdown", &ConfiguredInferModelWrapper::shutdown) + ; +} + +void ConfiguredInferModelBindingsWrapper::bind(py::module &m) +{ + py::class_< + ConfiguredInferModelBindingsWrapper, + std::shared_ptr + >(m, "ConfiguredInferModelBindings") + .def("input", &ConfiguredInferModelBindingsWrapper::input) + .def("output", &ConfiguredInferModelBindingsWrapper::output) + ; +} + +void ConfiguredInferModelBindingsInferStreamWrapper::bind(py::module &m) +{ + py::class_< + ConfiguredInferModelBindingsInferStreamWrapper, + std::shared_ptr + >(m, "ConfiguredInferModelInferStream") + .def("set_buffer", &ConfiguredInferModelBindingsInferStreamWrapper::set_buffer) + ; +} + +void InferModelInferStreamWrapper::bind(py::module &m) +{ + py::class_< + InferModelInferStreamWrapper, + std::shared_ptr + >(m, "InferModelInferStream") + .def("name", &InferModelInferStreamWrapper::name) + .def("set_format_type", &InferModelInferStreamWrapper::set_format_type) + .def("set_format_order", &InferModelInferStreamWrapper::set_format_order) + .def("get_quant_infos", &InferModelInferStreamWrapper::get_quant_infos) + .def("shape", &InferModelInferStreamWrapper::shape) + .def("format", &InferModelInferStreamWrapper::format) + .def("is_nms", &InferModelInferStreamWrapper::is_nms) + .def("set_nms_score_threshold", &InferModelInferStreamWrapper::set_nms_score_threshold) + .def("set_nms_iou_threshold", &InferModelInferStreamWrapper::set_nms_iou_threshold) + .def("set_nms_max_proposals_per_class", &InferModelInferStreamWrapper::set_nms_max_proposals_per_class) + .def("set_nms_max_accumulated_mask_size", &InferModelInferStreamWrapper::set_nms_max_accumulated_mask_size) + ; +} + +void AsyncInferJobWrapper::bind(py::module &m) +{ + py::class_< + AsyncInferJobWrapper, std::shared_ptr + >(m, "AsyncInferJob") + // wait is a blocking call that can take a long time, and in the meantime, the pythonic + // callback that was passed to run_async might be called. Therefore, we release the GIL before calling. + // This is done in order to avoid a deadlock. Any c++ function that is called from python and that might + // be still running while another thread is trying to execute a python callback should release the GIL. + .def("wait", &AsyncInferJobWrapper::wait, py::call_guard()) + ; +} diff --git a/hailort/libhailort/bindings/python/src/infer_model_api.hpp b/hailort/libhailort/bindings/python/src/infer_model_api.hpp new file mode 100644 index 0000000..e444a25 --- /dev/null +++ b/hailort/libhailort/bindings/python/src/infer_model_api.hpp @@ -0,0 +1,195 @@ +/** + * Copyright (c) 2020-2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file infer_model_api.hpp + * @brief Defines binding to a infer model class family usage over Python. + **/ + +#ifndef INFER_MODEL_API_HPP_ +#define INFER_MODEL_API_HPP_ + +#include "hailo/hailort.h" +#include "hailo/event.hpp" +#include "hailo/infer_model.hpp" +#include "utils.hpp" +#include +#include +#include +#include +#include + +namespace hailort { + +class ConfiguredInferModelWrapper; +class ConfiguredInferModelBindingsWrapper; +class ConfiguredInferModelBindingsInferStreamWrapper; +class InferModelInferStreamWrapper; +class AsyncInferJobWrapper; + +using AsyncInferCallBack = std::function; +using AsyncInferCallBackAndStatus = std::pair; + +class InferModelWrapper final +{ +public: + InferModelWrapper(std::shared_ptr infer_model, bool is_using_service) : + m_infer_model(std::move(infer_model)), m_is_using_service(is_using_service) {} + void set_batch_size(uint16_t batch_size); + void set_power_mode(hailo_power_mode_t power_mode); + void set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency); + ConfiguredInferModelWrapper configure(); + std::vector get_input_names(); + std::vector get_output_names(); + std::vector inputs(); + std::vector outputs(); + InferModelInferStreamWrapper input(const std::string &name); + InferModelInferStreamWrapper output(const std::string &name); + + static void bind(py::module &m); + +private: + std::shared_ptr m_infer_model; + bool m_is_using_service; +}; + +class ConfiguredInferModelWrapper final +{ +public: + ConfiguredInferModelWrapper(ConfiguredInferModel &&configured_infer_model, bool is_using_service, + const std::vector &output_names) : + m_configured_infer_model(std::move(configured_infer_model)), + m_callbacks_queue(std::make_shared>()), + m_is_alive(true), + m_is_using_service(is_using_service), + m_output_names(output_names) + { + } + + ConfiguredInferModelWrapper(ConfiguredInferModelWrapper &&other) : + m_configured_infer_model(std::move(other.m_configured_infer_model)), + m_callbacks_queue(std::move(other.m_callbacks_queue)), + m_callbacks_thread(std::move(other.m_callbacks_thread)), + m_is_alive(true), + m_is_using_service(other.m_is_using_service), + m_output_names(other.m_output_names) + { + other.m_is_alive = false; + } + + ~ConfiguredInferModelWrapper() + { + if (m_is_alive) { + py::gil_scoped_release release; // don't block the user-defined callbacks while join()-ing + m_configured_infer_model.shutdown(); + m_is_alive = false; // signal the thread to stop + m_cv.notify_all(); // wake up the thread so it can return + if (m_callbacks_thread && m_callbacks_thread->joinable()) + { + m_callbacks_thread->join(); + } + } + } + + ConfiguredInferModelBindingsWrapper create_bindings(); + void activate(); + void deactivate(); + void wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count = 1); + void run(ConfiguredInferModelBindingsWrapper bindings, std::chrono::milliseconds timeout); + AsyncInferJobWrapper run_async( + std::vector &bindings, + AsyncInferCallBack pythonic_cb); + void set_scheduler_timeout(const std::chrono::milliseconds &timeout); + void set_scheduler_threshold(uint32_t threshold); + void set_scheduler_priority(uint8_t priority); + size_t get_async_queue_size(); + void shutdown(); + + static void bind(py::module &m); + +private: + void execute_callbacks(); + + ConfiguredInferModel m_configured_infer_model; + std::mutex m_queue_mutex; + std::condition_variable m_cv; + std::shared_ptr> m_callbacks_queue; + std::shared_ptr m_callbacks_thread; // worker thread. Executes user defined (pythonic) callbacks + std::atomic_bool m_is_alive; // allow main thread to write, while worker thread is reading + bool m_is_using_service; + std::vector m_output_names; +}; + +class ConfiguredInferModelBindingsWrapper final +{ +public: + ConfiguredInferModelBindingsWrapper(ConfiguredInferModel::Bindings&& bindings, std::vector output_names) : + m_bindings(std::move(bindings)), + m_output_names(output_names) + {} + ConfiguredInferModelBindingsInferStreamWrapper input(const std::string &name); + ConfiguredInferModelBindingsInferStreamWrapper output(const std::string &name); + ConfiguredInferModel::Bindings get() { return m_bindings; } + + static void bind(py::module &m); + +private: + ConfiguredInferModel::Bindings m_bindings; + std::vector m_output_names; +}; + +class ConfiguredInferModelBindingsInferStreamWrapper final +{ +public: + ConfiguredInferModelBindingsInferStreamWrapper(ConfiguredInferModel::Bindings::InferStream&& infer_stream) : + m_infer_stream(std::move(infer_stream)) {} + void set_buffer(py::array buffer); + + static void bind(py::module &m); + +private: + ConfiguredInferModel::Bindings::InferStream m_infer_stream; +}; + +class InferModelInferStreamWrapper final +{ +public: + InferModelInferStreamWrapper(InferModel::InferStream&& infer_stream) : + m_infer_stream(std::move(infer_stream)) {} + const std::string name() const; + void set_format_type(hailo_format_type_t type); + void set_format_order(hailo_format_order_t order); + std::vector get_quant_infos() const; + std::vector shape() const; + hailo_format_t format() const; + bool is_nms() const; + void set_nms_score_threshold(float32_t threshold); + void set_nms_iou_threshold(float32_t threshold); + void set_nms_max_proposals_per_class(uint32_t max_proposals_per_class); + void set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size); + + static void bind(py::module &m); +private: + InferModel::InferStream m_infer_stream; +}; + +class AsyncInferJobWrapper final +{ +public: + AsyncInferJobWrapper(AsyncInferJob&& job, EventPtr is_callback_done) : + m_job(std::move(job)), + m_is_callback_done(is_callback_done) + {} + void wait(std::chrono::milliseconds timeout); + + static void bind(py::module &m); + +private: + AsyncInferJob m_job; + EventPtr m_is_callback_done; +}; + +} + +#endif /* INFER_MODEL_API_HPP_ */ diff --git a/hailort/libhailort/bindings/python/src/network_group_api.cpp b/hailort/libhailort/bindings/python/src/network_group_api.cpp index db13dd4..1fb66e8 100644 --- a/hailort/libhailort/bindings/python/src/network_group_api.cpp +++ b/hailort/libhailort/bindings/python/src/network_group_api.cpp @@ -12,7 +12,7 @@ namespace hailort { -void ConfiguredNetworkGroupWrapper::add_to_python_module(py::module &m) +void ConfiguredNetworkGroupWrapper::bind(py::module &m) { py::class_(m, "ConfiguredNetworkGroup") .def("is_scheduled", &ConfiguredNetworkGroupWrapper::is_scheduled) @@ -26,6 +26,9 @@ void ConfiguredNetworkGroupWrapper::add_to_python_module(py::module &m) .def("set_scheduler_timeout", &ConfiguredNetworkGroupWrapper::set_scheduler_timeout) .def("set_scheduler_threshold", &ConfiguredNetworkGroupWrapper::set_scheduler_threshold) .def("set_scheduler_priority", &ConfiguredNetworkGroupWrapper::set_scheduler_priority) + .def("init_cache", &ConfiguredNetworkGroupWrapper::init_cache) + .def("get_cache_info", &ConfiguredNetworkGroupWrapper::get_cache_info) + .def("update_cache_offset", &ConfiguredNetworkGroupWrapper::update_cache_offset) .def("get_networks_names", &ConfiguredNetworkGroupWrapper::get_networks_names) .def("get_sorted_output_names", &ConfiguredNetworkGroupWrapper::get_sorted_output_names) .def("get_input_vstream_infos", &ConfiguredNetworkGroupWrapper::get_input_vstream_infos) @@ -63,7 +66,7 @@ void ActivatedAppContextManagerWrapper::exit() m_activated_net_group.reset(); } -void ActivatedAppContextManagerWrapper::add_to_python_module(py::module &m) +void ActivatedAppContextManagerWrapper::bind(py::module &m) { py::class_(m, "ActivatedApp") .def("__enter__", &ActivatedAppContextManagerWrapper::enter, py::return_value_policy::reference) @@ -86,10 +89,4 @@ void ActivatedAppContextManagerWrapper::add_to_python_module(py::module &m) ; } -void NetworkGroup_api_initialize_python_module(py::module &m) -{ - ConfiguredNetworkGroupWrapper::add_to_python_module(m); - ActivatedAppContextManagerWrapper::add_to_python_module(m); -} - } /* namespace hailort */ diff --git a/hailort/libhailort/bindings/python/src/network_group_api.hpp b/hailort/libhailort/bindings/python/src/network_group_api.hpp index 4f5f209..0d17d54 100644 --- a/hailort/libhailort/bindings/python/src/network_group_api.hpp +++ b/hailort/libhailort/bindings/python/src/network_group_api.hpp @@ -30,7 +30,7 @@ public: const ActivatedNetworkGroup& enter(); void exit(); - static void add_to_python_module(py::module &m); + static void bind(py::module &m); private: std::unique_ptr m_activated_net_group; ConfiguredNetworkGroup &m_net_group; @@ -136,6 +136,26 @@ public: VALIDATE_STATUS(status); } + void init_cache(uint32_t read_offset, int32_t write_offset_delta) + { + auto status = get().init_cache(read_offset, write_offset_delta); + VALIDATE_STATUS(status); + } + + hailo_cache_info_t get_cache_info() + { + auto cache_info = get().get_cache_info(); + VALIDATE_EXPECTED(cache_info); + + return cache_info.release(); + } + + void update_cache_offset(int32_t offset_delta_bytes) + { + auto status = get().update_cache_offset(offset_delta_bytes); + VALIDATE_STATUS(status); + } + auto get_networks_names() { auto network_infos = get().get_network_infos(); @@ -300,7 +320,7 @@ public: return std::make_shared(net_group.release(), store_guard_for_multi_process); } - static void add_to_python_module(py::module &m); + static void bind(py::module &m); private: // Normally, the ownership of the network group is the Device/VDevice objects. We keep weak_ptr diff --git a/hailort/libhailort/bindings/python/src/pyhailort.cpp b/hailort/libhailort/bindings/python/src/pyhailort.cpp index 5b0a8c5..03d0a47 100644 --- a/hailort/libhailort/bindings/python/src/pyhailort.cpp +++ b/hailort/libhailort/bindings/python/src/pyhailort.cpp @@ -12,6 +12,7 @@ using namespace std; #include "hailo/hailort_defaults.hpp" #include "hailo/network_rate_calculator.hpp" +#include "infer_model_api.hpp" #include "hef_api.hpp" #include "vstream_api.hpp" #include "vdevice_api.hpp" @@ -125,7 +126,7 @@ public: return name.value(); } - static void add_to_python_module(py::module &m) + static void bind(py::module &m) { py::class_(m, "NetworkRateLimiter") .def("set_rate_limit", &NetworkRateLimiter::set_rate_limit) @@ -249,6 +250,7 @@ PYBIND11_MODULE(_pyhailort, m) { .value("HAILO15H", HAILO_ARCH_HAILO15H) .value("PLUTO", HAILO_ARCH_PLUTO) .value("HAILO15M", HAILO_ARCH_HAILO15M) + .value("HAILO10H", HAILO_ARCH_HAILO10H) ; /* TODO: SDK-15648 */ @@ -833,7 +835,29 @@ PYBIND11_MODULE(_pyhailort, m) { py::class_(m, "VDeviceParams") .def(py::init<>()) - // Add device_ids + .def_property("device_ids", + [](const VDeviceParamsWrapper ¶ms) -> py::list { + py::list ids; + if (params.orig_params.device_ids != nullptr) { + for (size_t i = 0; i < params.orig_params.device_count; i++) { + ids.append(std::string(params.orig_params.device_ids[i].id)); + } + } + return ids; + }, + [](VDeviceParamsWrapper ¶ms, const py::list &device_ids) { + uint32_t count = static_cast(py::len(device_ids)); + params.ids.resize(count); + for (size_t i = 0; i < count; i++) { + std::string id_str = py::cast(device_ids[i]); + auto expected_device_id = HailoRTCommon::to_device_id(id_str); + VALIDATE_EXPECTED(expected_device_id); + params.ids[i] = expected_device_id.release(); + } + params.orig_params.device_ids = params.ids.data(); + params.orig_params.device_count = count; + } + ) .def_property("device_count", [](const VDeviceParamsWrapper& params) -> uint32_t { return params.orig_params.device_count; @@ -848,7 +872,6 @@ PYBIND11_MODULE(_pyhailort, m) { }, [](VDeviceParamsWrapper& params, hailo_scheduling_algorithm_t scheduling_algorithm) { params.orig_params.scheduling_algorithm = scheduling_algorithm; - params.orig_params.multi_process_service = (HAILO_SCHEDULING_ALGORITHM_NONE != scheduling_algorithm); } ) .def_property("group_id", @@ -860,15 +883,18 @@ PYBIND11_MODULE(_pyhailort, m) { params.orig_params.group_id = params.group_id_str.c_str(); } ) - .def_property_readonly("multi_process_service", + .def_property("multi_process_service", [](const VDeviceParamsWrapper& params) -> bool { return params.orig_params.multi_process_service; + }, + [](VDeviceParamsWrapper& params, bool multi_process_service) { + params.orig_params.multi_process_service = multi_process_service; } ) .def_static("default", []() { auto orig_params = HailoRTDefaults::get_vdevice_params(); orig_params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_NONE; - VDeviceParamsWrapper params_wrapper{orig_params, ""}; + VDeviceParamsWrapper params_wrapper{orig_params, "", {}}; return params_wrapper; }); ; @@ -928,6 +954,14 @@ PYBIND11_MODULE(_pyhailort, m) { )) ; + py::class_(m, "CacheInfo") + .def(py::init<>()) + .def(py::init()) + .def_readwrite("cache_size", &hailo_cache_info_t::cache_size) + .def_readwrite("current_read_offset", &hailo_cache_info_t::current_read_offset) + .def_readwrite("write_offset_delta", &hailo_cache_info_t::write_offset_delta) + ; + py::class_(m, "ThrottlingLevel", py::module_local()) .def_readonly("temperature_threshold", &hailo_throttling_level_t::temperature_threshold) .def_readonly("hysteresis_temperature_threshold", &hailo_throttling_level_t::hysteresis_temperature_threshold) @@ -1143,13 +1177,23 @@ PYBIND11_MODULE(_pyhailort, m) { .def_static("MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP", []() { return 1472;} ) ; - HefWrapper::initialize_python_module(m); - VStream_api_initialize_python_module(m); - VDevice_api_initialize_python_module(m); - NetworkGroup_api_initialize_python_module(m); - DeviceWrapper::add_to_python_module(m); - - NetworkRateLimiter::add_to_python_module(m); + ActivatedAppContextManagerWrapper::bind(m); + AsyncInferJobWrapper::bind(m); + ConfiguredInferModelBindingsInferStreamWrapper::bind(m); + ConfiguredInferModelBindingsWrapper::bind(m); + ConfiguredInferModelWrapper::bind(m); + ConfiguredNetworkGroupWrapper::bind(m); + DeviceWrapper::bind(m); + HefWrapper::bind(m); + InferModelInferStreamWrapper::bind(m); + InferModelWrapper::bind(m); + InferVStreamsWrapper::bind(m); + InputVStreamWrapper::bind(m); + InputVStreamsWrapper::bind(m); + NetworkRateLimiter::bind(m); + OutputVStreamWrapper::bind(m); + OutputVStreamsWrapper::bind(m); + VDeviceWrapper::bind(m); std::stringstream version; version << HAILORT_MAJOR_VERSION << "." << HAILORT_MINOR_VERSION << "." << HAILORT_REVISION_VERSION; diff --git a/hailort/libhailort/bindings/python/src/vdevice_api.cpp b/hailort/libhailort/bindings/python/src/vdevice_api.cpp new file mode 100644 index 0000000..fc93cfa --- /dev/null +++ b/hailort/libhailort/bindings/python/src/vdevice_api.cpp @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2020-2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file infer_model_api.cpp + * @brief Defines binding to a infer model class family usage over Python. + **/ +#include "vdevice_api.hpp" +#include "infer_model_api.hpp" + +using namespace hailort; + +InferModelWrapper VDeviceWrapper::create_infer_model_from_file(const std::string &hef_path, const std::string &network_name) +{ + auto infer_model = m_vdevice->create_infer_model(hef_path, network_name); + VALIDATE_EXPECTED(infer_model); + + return InferModelWrapper(infer_model.release(), m_is_using_service); +} + +InferModelWrapper VDeviceWrapper::create_infer_model_from_buffer(const py::bytes &buffer, const std::string &network_name) +{ + // there are 3 ways to get the buffer from python and convert it to MemoryView: + // 1. py::bytes -> std::string -> MemoryView + // 2. py::bytes -> py::buffer -> MemoryView (this is the one used here) + // 3. std::string -> MemoryView + // + // 1+3 are copying the data, while 2 isn't, resulting in 700X faster transfer between python and c++ (tested on yolov5s [~15MB]) + py::buffer_info info(py::buffer(buffer).request()); + MemoryView hef_buffer(MemoryView(info.ptr, static_cast(info.size))); + auto infer_model = m_vdevice->create_infer_model(hef_buffer, network_name); + VALIDATE_EXPECTED(infer_model); + + return InferModelWrapper(infer_model.release(), m_is_using_service); +} diff --git a/hailort/libhailort/bindings/python/src/vdevice_api.hpp b/hailort/libhailort/bindings/python/src/vdevice_api.hpp index 590e9fe..fa7dbbc 100644 --- a/hailort/libhailort/bindings/python/src/vdevice_api.hpp +++ b/hailort/libhailort/bindings/python/src/vdevice_api.hpp @@ -10,6 +10,7 @@ #ifndef VDEVICE_API_HPP_ #define VDEVICE_API_HPP_ +#include "hef_api.hpp" #include "utils.hpp" #include "network_group_api.hpp" @@ -18,6 +19,7 @@ #include "hailo/hailort_common.hpp" #include +#include #include #include #include @@ -31,12 +33,14 @@ namespace hailort { +class InferModelWrapper; + struct VDeviceParamsWrapper { hailo_vdevice_params_t orig_params; std::string group_id_str; + std::vector ids; }; - class VDeviceWrapper; using VDeviceWrapperPtr = std::shared_ptr; @@ -58,11 +62,10 @@ public: std::cerr << "VDevice device_ids can be set in params or device_ids argument. Both parameters were passed to the c'tor"; throw HailoRTStatusException(std::to_string(HAILO_INVALID_OPERATION)); } - auto modified_params = params; - auto device_ids_vector = HailoRTCommon::to_device_ids_vector(device_ids); - VALIDATE_EXPECTED(device_ids_vector); - modified_params.orig_params.device_ids = device_ids_vector->data(); - return std::make_shared(modified_params.orig_params); + if (!device_ids.empty()) { + return create_from_ids(device_ids); + } + return create(params); } static VDeviceWrapperPtr create_from_ids(const std::vector &device_ids) @@ -95,6 +98,7 @@ public: VALIDATE_EXPECTED(vdevice_expected); m_vdevice = vdevice_expected.release(); + m_is_using_service = params.multi_process_service; }; py::list get_physical_devices_ids() const @@ -128,28 +132,34 @@ public: m_vdevice.reset(); } + InferModelWrapper create_infer_model_from_file(const std::string &hef_path, const std::string &network_name); + InferModelWrapper create_infer_model_from_buffer(const py::bytes &buffer, const std::string &network_name); + + static void bind(py::module &m) + { + py::class_(m, "VDevice") + .def("create", py::overload_cast(&VDeviceWrapper::create)) + .def("create", py::overload_cast(&VDeviceWrapper::create)) + .def("create", py::overload_cast&>(&VDeviceWrapper::create)) + .def("create_from_ids", &VDeviceWrapper::create_from_ids) + .def("get_physical_devices_ids", &VDeviceWrapper::get_physical_devices_ids) + .def("configure", &VDeviceWrapper::configure) + .def("release", &VDeviceWrapper::release) + .def("create_infer_model_from_file", &VDeviceWrapper::create_infer_model_from_file) + .def("create_infer_model_from_buffer", &VDeviceWrapper::create_infer_model_from_buffer) + ; + } + private: std::unique_ptr m_vdevice; std::vector m_net_groups; + bool m_is_using_service; #ifdef HAILO_IS_FORK_SUPPORTED AtForkRegistry::AtForkGuard m_atfork_guard; #endif }; -void VDevice_api_initialize_python_module(py::module &m) -{ - py::class_(m, "VDevice") - .def("create", py::overload_cast(&VDeviceWrapper::create)) - .def("create", py::overload_cast(&VDeviceWrapper::create)) - .def("create", py::overload_cast&>(&VDeviceWrapper::create)) - .def("create_from_ids", &VDeviceWrapper::create_from_ids) - .def("get_physical_devices_ids", &VDeviceWrapper::get_physical_devices_ids) - .def("configure", &VDeviceWrapper::configure) - .def("release", &VDeviceWrapper::release) - ; -} - } /* namespace hailort */ #endif /* VDEVICE_API_HPP_ */ diff --git a/hailort/libhailort/bindings/python/src/vstream_api.cpp b/hailort/libhailort/bindings/python/src/vstream_api.cpp index 82dbb54..c509844 100644 --- a/hailort/libhailort/bindings/python/src/vstream_api.cpp +++ b/hailort/libhailort/bindings/python/src/vstream_api.cpp @@ -18,7 +18,7 @@ namespace hailort { -void InputVStreamWrapper::add_to_python_module(py::module &m) +void InputVStreamWrapper::bind(py::module &m) { py::class_>(m, "InputVStream") .def("send", [](InputVStream &self, py::array data) @@ -43,7 +43,10 @@ void InputVStreamWrapper::add_to_python_module(py::module &m) }) .def_property_readonly("shape", [](InputVStream &self) { - return *py::array::ShapeContainer(HailoRTBindingsCommon::get_pybind_shape(self.get_info(), self.get_user_buffer_format())); + auto shape = self.get_info().shape; + auto nms_shape = self.get_info().nms_shape; + auto format = self.get_user_buffer_format(); + return *py::array::ShapeContainer(HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format)); }) ; } @@ -100,7 +103,7 @@ void InputVStreamsWrapper::clear() VALIDATE_STATUS(status); } -void InputVStreamsWrapper::add_to_python_module(py::module &m) +void InputVStreamsWrapper::bind(py::module &m) { py::class_(m, "InputVStreams") .def(py::init(&InputVStreamsWrapper::create)) @@ -138,10 +141,13 @@ hailo_format_t OutputVStreamWrapper::get_user_buffer_format(OutputVStream &self) auto OutputVStreamWrapper::get_shape(OutputVStream &self) { - return *py::array::ShapeContainer(HailoRTBindingsCommon::get_pybind_shape(self.get_info(), self.get_user_buffer_format())); + auto shape = self.get_info().shape; + auto nms_shape = self.get_info().nms_shape; + auto format = self.get_user_buffer_format(); + return *py::array::ShapeContainer(HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format)); } -void OutputVStreamWrapper::add_to_python_module(py::module &m) +void OutputVStreamWrapper::bind(py::module &m) { py::class_>(m, "OutputVStream") .def("recv", [](OutputVStream &self) @@ -264,7 +270,7 @@ void OutputVStreamsWrapper::after_fork_in_child() } } -void OutputVStreamsWrapper::add_to_python_module(py::module &m) +void OutputVStreamsWrapper::bind(py::module &m) { py::class_(m, "OutputVStreams") .def(py::init(&OutputVStreamsWrapper::create)) @@ -351,12 +357,18 @@ std::vector InferVStreamsWrapper::get_shape(const std::string &stream_na { auto input = m_infer_pipeline->get_input_by_name(stream_name); if (HAILO_SUCCESS == input.status()) { - return HailoRTBindingsCommon::get_pybind_shape(input->get().get_info(), input->get().get_user_buffer_format()); + auto shape = input->get().get_info().shape; + auto nms_shape = input->get().get_info().nms_shape; + auto format = input->get().get_user_buffer_format(); + return HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format); } auto output = m_infer_pipeline->get_output_by_name(stream_name); if (HAILO_SUCCESS == output.status()) { - return HailoRTBindingsCommon::get_pybind_shape(output->get().get_info(), output->get().get_user_buffer_format()); + auto shape = output->get().get_info().shape; + auto nms_shape = output->get().get_info().nms_shape; + auto format = output->get().get_user_buffer_format(); + return HailoRTBindingsCommon::get_pybind_shape(shape, nms_shape, format); } std::cerr << "Stream " << stream_name << " not found"; @@ -387,7 +399,7 @@ void InputVStreamsWrapper::after_fork_in_child() } } -void InferVStreamsWrapper::add_to_python_module(py::module &m) +void InferVStreamsWrapper::bind(py::module &m) { py::class_(m, "InferVStreams") .def(py::init(&InferVStreamsWrapper::create)) @@ -419,13 +431,4 @@ InferVStreamsWrapper::InferVStreamsWrapper(std::shared_ptr &infer : m_infer_pipeline(std::move(infer_pipeline)) {} -void VStream_api_initialize_python_module(py::module &m) -{ - InputVStreamWrapper::add_to_python_module(m); - InputVStreamsWrapper::add_to_python_module(m); - OutputVStreamWrapper::add_to_python_module(m); - OutputVStreamsWrapper::add_to_python_module(m); - InferVStreamsWrapper::add_to_python_module(m); -} - } /* namespace hailort */ diff --git a/hailort/libhailort/bindings/python/src/vstream_api.hpp b/hailort/libhailort/bindings/python/src/vstream_api.hpp index a664598..851d05b 100644 --- a/hailort/libhailort/bindings/python/src/vstream_api.hpp +++ b/hailort/libhailort/bindings/python/src/vstream_api.hpp @@ -31,7 +31,7 @@ class ConfiguredNetworkGroupWrapper; class InputVStreamWrapper final { public: - static void add_to_python_module(py::module &m); + static void bind(py::module &m); }; @@ -52,7 +52,7 @@ public: void after_fork_in_parent(); void after_fork_in_child(); - static void add_to_python_module(py::module &m); + static void bind(py::module &m); InputVStreamsWrapper(std::unordered_map> &input_vstreams); @@ -70,7 +70,7 @@ public: static py::dtype get_dtype(OutputVStream &self); static hailo_format_t get_user_buffer_format(OutputVStream &self); static auto get_shape(OutputVStream &self); - static void add_to_python_module(py::module &m); + static void bind(py::module &m); }; class OutputVStreamsWrapper; @@ -89,7 +89,7 @@ public: void before_fork(); void after_fork_in_parent(); void after_fork_in_child(); - static void add_to_python_module(py::module &m); + static void bind(py::module &m); OutputVStreamsWrapper(std::unordered_map> &output_vstreams); @@ -116,7 +116,7 @@ public: void before_fork(); void after_fork_in_parent(); void after_fork_in_child(); - static void add_to_python_module(py::module &m); + static void bind(py::module &m); private: InferVStreamsWrapper(std::shared_ptr &infer_pipeline); diff --git a/hailort/libhailort/doc/CMakeLists.txt b/hailort/libhailort/doc/CMakeLists.txt index c2b13af..73d0964 100644 --- a/hailort/libhailort/doc/CMakeLists.txt +++ b/hailort/libhailort/doc/CMakeLists.txt @@ -27,6 +27,6 @@ if(DOXYGEN_FOUND) DIRECTORY "${DOXYGEN_OUTPUT_DIR}" DESTINATION "doc/" CONFIGURATIONS Release) - add_custom_target(doc DEPENDS doxygen) + add_custom_target(doc ALL DEPENDS doxygen) endif() \ No newline at end of file diff --git a/hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt b/hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt index 1221df5..e2d6a72 100644 --- a/hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(data_quantization_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt b/hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt index 9771d50..5379d9b 100644 --- a/hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(infer_pipeline_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt b/hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt index 52e9898..7b54dde 100644 --- a/hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(multi_device_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt b/hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt index 6d6ca10..d8677f7 100644 --- a/hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(multi_network_vstream_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/multi_network_vstream_example/multi_network_vstream_example.c b/hailort/libhailort/examples/c/multi_network_vstream_example/multi_network_vstream_example.c index 5b0f56c..ac05a5f 100644 --- a/hailort/libhailort/examples/c/multi_network_vstream_example/multi_network_vstream_example.c +++ b/hailort/libhailort/examples/c/multi_network_vstream_example/multi_network_vstream_example.c @@ -241,12 +241,12 @@ l_deactivate_network_group: (void)hailo_deactivate_network_group(activated_network_group); l_release_vstreams: for (size_t net_index = 0; net_index < NET_COUNT; net_index++) { - if (NULL != output_vstreams[net_index]) { + if (NULL != output_vstreams[net_index][0]) { (void)hailo_release_output_vstreams(output_vstreams[net_index], output_vstreams_size[net_index]); } } for (size_t net_index = 0; net_index < NET_COUNT; net_index++) { - if (NULL != input_vstreams[net_index]) { + if (NULL != input_vstreams[net_index][0]) { (void)hailo_release_input_vstreams(input_vstreams[net_index], input_vstreams_size[net_index]); } } diff --git a/hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt b/hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt index 1e29979..fd72f99 100644 --- a/hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(notification_callback_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt b/hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt index a632693..85a8e2e 100644 --- a/hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(power_measurement_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/raw_async_streams_single_thread_example/CMakeLists.txt b/hailort/libhailort/examples/c/raw_async_streams_single_thread_example/CMakeLists.txt index 90ef027..0bb957e 100644 --- a/hailort/libhailort/examples/c/raw_async_streams_single_thread_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/raw_async_streams_single_thread_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(raw_async_streams_single_thread_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt b/hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt index 2620378..24eb068 100644 --- a/hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(raw_streams_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt b/hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt index 3b2289e..66011c4 100644 --- a/hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(switch_network_groups_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/switch_network_groups_example/switch_network_groups_example.c b/hailort/libhailort/examples/c/switch_network_groups_example/switch_network_groups_example.c index 9a84f3f..4a4af9d 100644 --- a/hailort/libhailort/examples/c/switch_network_groups_example/switch_network_groups_example.c +++ b/hailort/libhailort/examples/c/switch_network_groups_example/switch_network_groups_example.c @@ -287,12 +287,12 @@ l_release_vstreams: for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) { for (size_t i = 0; i < num_input_vstreams[hef_index]; i++) { - if (NULL != src_data[hef_index] && NULL != src_data[hef_index][i]) { + if (NULL != src_data[hef_index][i]) { FREE(src_data[hef_index][i]); } } for (size_t i = 0; i < num_output_vstreams[hef_index]; i++) { - if (NULL != dst_data[hef_index] && NULL != dst_data[hef_index][i]) { + if (NULL != dst_data[hef_index][i]) { FREE(dst_data[hef_index][i]); } } diff --git a/hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt b/hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt index 80d2579..ac63eb8 100644 --- a/hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(switch_network_groups_manually_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt b/hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt index be5345a..398dd29 100644 --- a/hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(vstreams_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/cpp/async_infer_advanced_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/async_infer_advanced_example/CMakeLists.txt index bc83c0a..185f904 100644 --- a/hailort/libhailort/examples/cpp/async_infer_advanced_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/async_infer_advanced_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_async_infer_advanced_example async_infer_advanced_example.cpp) target_link_libraries(cpp_async_infer_advanced_example PRIVATE HailoRT::libhailort) diff --git a/hailort/libhailort/examples/cpp/async_infer_advanced_example/async_infer_advanced_example.cpp b/hailort/libhailort/examples/cpp/async_infer_advanced_example/async_infer_advanced_example.cpp index b5ed3df..0c8c581 100644 --- a/hailort/libhailort/examples/cpp/async_infer_advanced_example/async_infer_advanced_example.cpp +++ b/hailort/libhailort/examples/cpp/async_infer_advanced_example/async_infer_advanced_example.cpp @@ -17,7 +17,8 @@ #include #endif -#define FRAMES_COUNT (100) +#define BATCH_COUNT (100) +#define BATCH_SIZE (2) using namespace hailort; @@ -43,6 +44,7 @@ int main() std::cerr << "Failed create vdevice, status = " << vdevice.status() << std::endl; return vdevice.status(); } + std::cout << "VDevice created" << std::endl; // Create infer model from HEF file. auto infer_model_exp = vdevice.value()->create_infer_model("hefs/shortcut_net_nv12.hef"); @@ -50,9 +52,13 @@ int main() std::cerr << "Failed to create infer model, status = " << infer_model_exp.status() << std::endl; return infer_model_exp.status(); } + std::cout << "InferModel created" << std::endl; auto infer_model = infer_model_exp.release(); infer_model->output()->set_format_type(HAILO_FORMAT_TYPE_FLOAT32); + std::cout << "Set output format_type to float32" << std::endl; + infer_model->set_batch_size(BATCH_SIZE); + std::cout << "Set batch_size to " << BATCH_SIZE << std::endl; // Configure the infer model auto configured_infer_model = infer_model->configure(); @@ -60,19 +66,14 @@ int main() std::cerr << "Failed to create configured infer model, status = " << configured_infer_model.status() << std::endl; return configured_infer_model.status(); } + std::cout << "ConfiguredInferModel created" << std::endl; // The buffers are stored here as a guard for the memory. The buffer will be freed only after // configured_infer_model will be released. std::vector> buffer_guards; - // Create infer bindings - auto bindings = configured_infer_model->create_bindings(); - if (!bindings) { - std::cerr << "Failed to create infer bindings, status = " << bindings.status() << std::endl; - return bindings.status(); - } - - // Set the input buffers of the bindings. + // Create input buffers. + std::unordered_map input_buffers; for (const auto &input_name : infer_model->get_input_names()) { size_t input_frame_size = infer_model->input(input_name)->get_frame_size(); @@ -94,40 +95,62 @@ int main() pix_buffer.planes[1].plane_size = UV_PLANE_SIZE; pix_buffer.planes[1].user_ptr = reinterpret_cast(uv_plane_buffer.get()); - auto status = bindings->input(input_name)->set_pix_buffer(pix_buffer); - if (HAILO_SUCCESS != status) { - std::cerr << "Failed to set infer input buffer, status = " << status << std::endl; - return status; - } - + input_buffers[input_name] = pix_buffer; buffer_guards.push_back(y_plane_buffer); buffer_guards.push_back(uv_plane_buffer); } - // Set the output buffers of the bindings. + // Create output buffers. + std::unordered_map output_buffers; for (const auto &output_name : infer_model->get_output_names()) { size_t output_frame_size = infer_model->output(output_name)->get_frame_size(); auto output_buffer = page_aligned_alloc(output_frame_size); - auto status = bindings->output(output_name)->set_buffer(MemoryView(output_buffer.get(), output_frame_size)); - if (HAILO_SUCCESS != status) { - std::cerr << "Failed to set infer output buffer, status = " << status << std::endl; - return status; - } + output_buffers[output_name] = MemoryView(output_buffer.get(), output_frame_size); buffer_guards.push_back(output_buffer); } + std::cout << "Running inference..." << std::endl; AsyncInferJob last_infer_job; - for (uint32_t i = 0; i < FRAMES_COUNT; i++) { + for (uint32_t i = 0; i < BATCH_COUNT; i++) { // Waiting for available requests in the pipeline - auto status = configured_infer_model->wait_for_async_ready(std::chrono::milliseconds(1000)); + auto status = configured_infer_model->wait_for_async_ready(std::chrono::milliseconds(1000), BATCH_SIZE); if (HAILO_SUCCESS != status) { std::cerr << "Failed to wait for async ready, status = " << status << std::endl; return status; } - auto job = configured_infer_model->run_async(bindings.value(), [] (const AsyncInferCompletionInfo &/*completion_info*/) { - // Use completion_info to get the job status and the corresponding bindings + // In this example we infer the same buffers, so setting 'BATCH_SIZE' identical bindings in the 'multiple_bindings' vector + std::vector bindings_batch; + for (uint32_t b = 0; b < BATCH_SIZE; b++) { + auto bindings = configured_infer_model->create_bindings(); + if (!bindings) { + std::cerr << "Failed to create infer bindings, status = " << bindings.status() << std::endl; + return bindings.status(); + } + + for (auto &input_buffer : input_buffers) { + status = bindings->input(input_buffer.first)->set_pix_buffer(input_buffer.second); + if (HAILO_SUCCESS != status) { + std::cerr << "Failed to set infer input buffer, status = " << status << std::endl; + return status; + } + } + for (auto &output_buffer : output_buffers) { + status = bindings->output(output_buffer.first)->set_buffer(output_buffer.second); + if (HAILO_SUCCESS != status) { + std::cerr << "Failed to set infer output buffer, status = " << status << std::endl; + return status; + } + } + + bindings_batch.emplace_back(bindings.release()); + } + + auto job = configured_infer_model->run_async(bindings_batch, [] (const AsyncInferCompletionInfo &completion_info) { + // Use completion_info to get the async operation status + // Note that this callback must be executed as quickly as possible + (void)completion_info.status; }); if (!job) { std::cerr << "Failed to start async infer job, status = " << job.status() << std::endl; @@ -136,7 +159,7 @@ int main() // detach() is called in order for jobs to run in parallel (and not one after the other) job->detach(); - if (i == FRAMES_COUNT - 1) { + if (i == BATCH_COUNT - 1) { last_infer_job = job.release(); } } @@ -148,6 +171,6 @@ int main() return status; } - std::cout << "Inference finished successfully" << std::endl; + std::cout << "Inference finished successfully on " << BATCH_COUNT * BATCH_SIZE << " frames" << std::endl; return HAILO_SUCCESS; } diff --git a/hailort/libhailort/examples/cpp/async_infer_basic_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/async_infer_basic_example/CMakeLists.txt index 3b439ad..86886a7 100644 --- a/hailort/libhailort/examples/cpp/async_infer_basic_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/async_infer_basic_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_async_infer_basic_example async_infer_basic_example.cpp) target_link_libraries(cpp_async_infer_basic_example PRIVATE HailoRT::libhailort) diff --git a/hailort/libhailort/examples/cpp/async_infer_basic_example/async_infer_basic_example.cpp b/hailort/libhailort/examples/cpp/async_infer_basic_example/async_infer_basic_example.cpp index ec78ac7..b498f83 100644 --- a/hailort/libhailort/examples/cpp/async_infer_basic_example/async_infer_basic_example.cpp +++ b/hailort/libhailort/examples/cpp/async_infer_basic_example/async_infer_basic_example.cpp @@ -41,6 +41,7 @@ int main() std::cerr << "Failed create vdevice, status = " << vdevice.status() << std::endl; return vdevice.status(); } + std::cout << "VDevice created" << std::endl; // Create infer model from HEF file. auto infer_model_exp = vdevice.value()->create_infer_model(HEF_FILE); @@ -48,6 +49,7 @@ int main() std::cerr << "Failed to create infer model, status = " << infer_model_exp.status() << std::endl; return infer_model_exp.status(); } + std::cout << "InferModel created" << std::endl; auto infer_model = infer_model_exp.release(); // Configure the infer model @@ -56,6 +58,7 @@ int main() std::cerr << "Failed to create configured infer model, status = " << configured_infer_model.status() << std::endl; return configured_infer_model.status(); } + std::cout << "ConfiguredInferModel created" << std::endl; // The buffers are stored here as a guard for the memory. The buffer will be freed only after // configured_infer_model will be released. @@ -90,7 +93,9 @@ int main() buffer_guards.push_back(output_buffer); } + std::cout << "ConfiguredInferModel::Bindings created and configured" << std::endl; + std::cout << "Running inference..." << std::endl; // Run the async infer job. auto job = configured_infer_model->run_async(bindings.value()); if (!job) { diff --git a/hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt index cbdacf7..5c5265c 100644 --- a/hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_infer_pipeline_example infer_pipeline_example.cpp) target_link_libraries(cpp_infer_pipeline_example PRIVATE HailoRT::libhailort) diff --git a/hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt index 62aa56c..49681ec 100644 --- a/hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_multi_device_example multi_device_example.cpp) target_link_libraries(cpp_multi_device_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt index 7c00350..6bf25bb 100644 --- a/hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_multi_network_vstream_example multi_network_vstream_example.cpp) target_link_libraries(cpp_multi_network_vstream_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt index 9153a42..7ddec34 100644 --- a/hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_multi_process_example multi_process_example.cpp) target_link_libraries(cpp_multi_process_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt index 791e03b..c57af5a 100644 --- a/hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_notification_callback_example notification_callback_example.cpp) target_link_libraries(cpp_notification_callback_example PRIVATE HailoRT::libhailort) diff --git a/hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt index c130400..939d60c 100644 --- a/hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_power_measurement_example power_measurement_example.cpp) target_link_libraries(cpp_power_measurement_example PRIVATE HailoRT::libhailort) diff --git a/hailort/libhailort/examples/cpp/raw_async_streams_multi_thread_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/raw_async_streams_multi_thread_example/CMakeLists.txt index a8609de..a7e276c 100644 --- a/hailort/libhailort/examples/cpp/raw_async_streams_multi_thread_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/raw_async_streams_multi_thread_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_raw_async_streams_multi_thread_example raw_async_streams_multi_thread_example.cpp) target_link_libraries(cpp_raw_async_streams_multi_thread_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/raw_async_streams_single_thread_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/raw_async_streams_single_thread_example/CMakeLists.txt index d9f60d0..97f4201 100644 --- a/hailort/libhailort/examples/cpp/raw_async_streams_single_thread_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/raw_async_streams_single_thread_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_raw_async_streams_single_thread_example raw_async_streams_single_thread_example.cpp) target_link_libraries(cpp_raw_async_streams_single_thread_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/raw_streams_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/raw_streams_example/CMakeLists.txt index f502d36..59920d7 100644 --- a/hailort/libhailort/examples/cpp/raw_streams_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/raw_streams_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_raw_streams_example raw_streams_example.cpp) target_link_libraries(cpp_raw_streams_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/switch_network_groups_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/switch_network_groups_example/CMakeLists.txt index 85bac9c..27f6f1b 100644 --- a/hailort/libhailort/examples/cpp/switch_network_groups_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/switch_network_groups_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_switch_network_groups_example switch_network_groups_example.cpp) target_link_libraries(cpp_switch_network_groups_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/CMakeLists.txt index 1ade60f..33b5fe0 100644 --- a/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_switch_network_groups_manually_example switch_network_groups_manually_example.cpp) target_link_libraries(cpp_switch_network_groups_manually_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/vstreams_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/vstreams_example/CMakeLists.txt index faed0c9..601d318 100644 --- a/hailort/libhailort/examples/cpp/vstreams_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/vstreams_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(HailoRT 4.17.1 EXACT REQUIRED) +find_package(HailoRT 4.18.0 EXACT REQUIRED) add_executable(cpp_vstreams_example vstreams_example.cpp) target_link_libraries(cpp_vstreams_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/hef.proto b/hailort/libhailort/hef.proto index 59625f8..1b65d2a 100644 --- a/hailort/libhailort/hef.proto +++ b/hailort/libhailort/hef.proto @@ -88,6 +88,7 @@ enum ProtoHEFHwArch { PROTO__HW_ARCH__HAILO8L = 3; PROTO__HW_ARCH__HAILO15H = 103; PROTO__HW_ARCH__HAILO15M = 4; + PROTO__HW_ARCH__HAILO10H = 5; // Reserving low numbers to public hw archs PROTO__HW_ARCH__SAGE_A0 = 100; @@ -580,9 +581,21 @@ message ProtoHEFAction { ProtoHEFActionEnableNMS enable_nms = 13; ProtoHEFActionWriteDataByType write_data_by_type = 14; ProtoHEFActionSwitchLcuBatch switch_lcu_batch = 15; + ProtoHEFActionWriteDataCcwPtr write_data_ccw_ptr = 16; } } +message ProtoHEFActionWriteDataCcwPtr { + // The offset which the data is in + uint64 offset = 1; + + // The size of the data in bytes + uint32 size = 2; + + // cfg_channel_index + uint32 cfg_channel_index = 3; +} + message ProtoHEFActionWriteData { // The address to write the data to uint64 address = 1; @@ -869,6 +882,7 @@ enum ProtoHEFEdgeConnectionType { PROTO__EDGE_CONNECTION_TYPE__BOUNDARY = 0; PROTO__EDGE_CONNECTION_TYPE__INTERMEDIATE = 1; PROTO__EDGE_CONNECTION_TYPE__DDR = 2; + PROTO__EDGE_CONNECTION_TYPE__CACHE = 3; } message ProtoHEFContextSwitchInformation { @@ -880,6 +894,7 @@ message ProtoHEFContextSwitchInformation { uint32 buffers = 6; uint32 connected_sys_index = 7; repeated ProtoHEFConnectedContextInfo connected_contexts = 8; + int32 cache_id = 9; } message ProtoHEFConnectedContextInfo { diff --git a/hailort/libhailort/include/hailo/device.hpp b/hailort/libhailort/include/hailo/device.hpp index 88f0959..f5b421d 100644 --- a/hailort/libhailort/include/hailo/device.hpp +++ b/hailort/libhailort/include/hailo/device.hpp @@ -730,6 +730,45 @@ public: */ virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction); + /** + * Maps the dmabuf represented by the file descriptor @a dmabuf_fd for DMA transfers to/from this device, in the specified + * @a data_direction. + * DMA mapping of buffers in advance may improve the performance of async API. This improvement will become + * apparent when the buffer is reused multiple times across different async operations. + * + * For high level API (aka InferModel), buffers bound using ConfiguredInferModel::Bindings::InferStream::set_buffer + * can be mapped. + * + * For low level API (aka InputStream/OutputStream), buffers passed to InputStream::write_async and + * OutputStream::read_async can be mapped. + * + * @param[in] dmabuf_fd The file descriptor of the dmabuf to be mapped. + * @param[in] size The buffer's size in bytes. + * @param[in] direction The direction of the mapping. For input streams, use `HAILO_DMA_BUFFER_DIRECTION_H2D` + * and for output streams, use `HAILO_DMA_BUFFER_DIRECTION_D2H`. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * + * @note The DMA mapping will be released upon calling dma_unmap() with @a dmabuf_fd, @a size and @a data_direction, or + * when the @a Device object is destroyed. + * @note This API is currently experimental. + */ + virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction); + + /** + * Un-maps a dmabuf buffer represented by the file descriptor @a dmabuf_fd for DMA transfers to/from this device, in the direction + * @a direction. + * + * @param[in] dmabuf_fd The file descriptor of the dmabuf to be un-mapped. + * @param[in] size The buffer's size in bytes. + * @param[in] direction The direction of the mapping. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * @note This API is currently experimental. + */ + virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction); + + virtual hailo_status direct_write_memory(uint32_t address, const void *buffer, uint32_t size); virtual hailo_status direct_read_memory(uint32_t address, void *buffer, uint32_t size); hailo_status set_overcurrent_state(bool should_activate); @@ -748,6 +787,9 @@ public: hailo_status continue_context_switch_breakpoint(uint8_t breakpoint_id); hailo_status clear_context_switch_breakpoint(uint8_t breakpoint_id); Expected get_context_switch_breakpoint_status(uint8_t breakpoint_id); + hailo_status init_cache_info(const hailo_cache_info_t &cache_info); + Expected get_cache_info(); + hailo_status update_cache_read_offset(int32_t read_offset_delta); virtual ~Device() = default; Device(const Device &) = delete; diff --git a/hailort/libhailort/include/hailo/hailort.h b/hailort/libhailort/include/hailo/hailort.h index 66ff945..9850049 100644 --- a/hailort/libhailort/include/hailo/hailort.h +++ b/hailort/libhailort/include/hailo/hailort.h @@ -147,7 +147,7 @@ typedef uint16_t nms_bbox_counter_t; HAILO_STATUS__X(59, HAILO_THREAD_NOT_ACTIVATED /*!< The given thread has not been activated */)\ HAILO_STATUS__X(60, HAILO_THREAD_NOT_JOINABLE /*!< The given thread is not joinable */)\ HAILO_STATUS__X(61, HAILO_NOT_FOUND /*!< Could not find element */)\ - HAILO_STATUS__X(62, HAILO_RESERVED_STATUS /*!< Reserved for future use */)\ + HAILO_STATUS__X(62, HAILO_COMMUNICATION_CLOSED /*!< The communication between endpoints is closed */)\ HAILO_STATUS__X(63, HAILO_STREAM_ABORT /*!< Stream recv/send was aborted */)\ HAILO_STATUS__X(64, HAILO_PCIE_DRIVER_NOT_INSTALLED /*!< Pcie driver is not installed */)\ HAILO_STATUS__X(65, HAILO_NOT_AVAILABLE /*!< Component is not available */)\ @@ -420,6 +420,7 @@ typedef enum hailo_device_architecture_e { HAILO_ARCH_HAILO15H, HAILO_ARCH_PLUTO, HAILO_ARCH_HAILO15M, + HAILO_ARCH_HAILO10H, /** Max enum value to maintain ABI Integrity */ HAILO_ARCH_MAX_ENUM = HAILO_MAX_ENUM @@ -837,8 +838,9 @@ typedef enum { // **************************************************************************************** // /** Hailo buffer flags */ typedef enum { - HAILO_BUFFER_FLAGS_NONE = 0, /*!< No flags - heap allocated buffer */ - HAILO_BUFFER_FLAGS_DMA = 1 << 0, /*!< Buffer is mapped to DMA (will be page aligned implicitly) */ + HAILO_BUFFER_FLAGS_NONE = 0, /*!< No flags - heap allocated buffer */ + HAILO_BUFFER_FLAGS_DMA = 1 << 0, /*!< Buffer is mapped to DMA (will be page aligned implicitly) */ + HAILO_BUFFER_FLAGS_CONTINUOUS = 1 << 1, /*!< Buffer is physically continuous (will be page aligned implicitly) */ /** Max enum value to maintain ABI Integrity */ HAILO_BUFFER_FLAGS_MAX_ENUM = HAILO_MAX_ENUM @@ -1476,10 +1478,18 @@ typedef enum { } hailo_latency_measurement_flags_t; typedef struct { - /** This parameter is only used in multi-context network_groups. + /** + * Sets the batch size of the InferModel. + * This parameter determines the number of frames that be sent for inference in a single batch. + * If a scheduler is enabled, this parameter determines the 'burst size' - the max number of frames after which the scheduler will attempt + * to switch to another model. + * * User is advised to modify this (single network parameter) or @a hailo_configure_network_group_params_t batch size parameter. Not both. * In case user wishes to work with the same batch size for all networks inside a network group, user is advised to set batch_size in @a hailo_configure_network_group_params_t. - * In case user wished to work with batch size per network, user is advised to use this parameter. Default network batch size is @a HAILO_DEFAULT_BATCH_SIZE */ + * In case user wished to work with batch size per network, user is advised to use this parameter. + + * note: Default value is @a HAILO_DEFAULT_BATCH_SIZE - means automatic batch determined by hailort. + */ uint16_t batch_size; } hailo_network_parameters_t; @@ -1527,6 +1537,9 @@ typedef struct { char name[HAILO_MAX_NETWORK_NAME_SIZE]; } hailo_network_info_t; +/** Notification IDs and structures section start */ +#pragma pack(push, 1) + /** Notification IDs, for each notification, one of the ::hailo_notification_message_parameters_t union will be set. */ typedef enum { /** Matches hailo_notification_message_parameters_t::rx_error_notification. */ @@ -1555,6 +1568,8 @@ typedef enum { HAILO_NOTIFICATION_ID_HW_INFER_MANAGER_INFER_DONE, /** Matches hailo_notification_message_parameters_t::context_switch_run_time_error */ HAILO_NOTIFICATION_ID_CONTEXT_SWITCH_RUN_TIME_ERROR_EVENT, + /** Matched hailo_notification_message_parameters_t::start_update_cache_offset_notification */ + HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET, /** Must be last! */ HAILO_NOTIFICATION_ID_COUNT, @@ -1644,6 +1659,10 @@ typedef struct { uint32_t infer_cycles; } hailo_hw_infer_manager_infer_done_notification_message_t; +typedef struct { + uint64_t cache_id_bitmask; +} hailo_start_update_cache_offset_notification_message_t; + typedef struct { uint32_t exit_status; uint8_t network_group_index; @@ -1676,6 +1695,8 @@ typedef union { hailo_hw_infer_manager_infer_done_notification_message_t hw_infer_manager_infer_done_notification; /** context switch run time error event */ hailo_context_switch_run_time_error_message_t context_switch_run_time_error; + /** Start cache offset update notification */ + hailo_start_update_cache_offset_notification_message_t start_update_cache_offset_notification; } hailo_notification_message_parameters_t; /** Notification data that will be passed to the callback passed in ::hailo_notification_callback */ @@ -1685,6 +1706,9 @@ typedef struct { hailo_notification_message_parameters_t body; } hailo_notification_t; +#pragma pack(pop) +/** Notification IDs and structures section end */ + /** * A notification callback. See ::hailo_set_notification_callback * @@ -1759,6 +1783,12 @@ typedef struct { uint32_t rate; } hailo_rate_limit_t; +typedef struct { + uint32_t cache_size; + uint32_t current_read_offset; + int32_t write_offset_delta; +} hailo_cache_info_t; + typedef enum { HAILO_SENSOR_TYPES_GENERIC = 0, HAILO_SENSOR_TYPES_ONSEMI_AR0220AT, @@ -2828,7 +2858,7 @@ HAILORTAPI hailo_status hailo_network_group_get_output_stream_infos(hailo_config HAILORTAPI hailo_status hailo_shutdown_network_group(hailo_configured_network_group network_group); /** - * Activates hailo_device inner-resources for context_switch inference. + * Activates hailo_device inner-resources for inference. * * @param[in] network_group NetworkGroup to be activated. * @param[in] activation_params Optional parameters for the activation (may be NULL). @@ -2840,7 +2870,7 @@ HAILORTAPI hailo_status hailo_activate_network_group(hailo_configured_network_gr hailo_activated_network_group *activated_network_group_out); /** - * De-activates hailo_device inner-resources for context_switch inference. + * De-activates hailo_device inner-resources for inference. * * @param[in] activated_network_group NetworkGroup to deactivate. * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. @@ -3027,6 +3057,86 @@ HAILORTAPI hailo_status hailo_vdevice_dma_map_buffer(hailo_vdevice vdevice, void HAILORTAPI hailo_status hailo_vdevice_dma_unmap_buffer(hailo_vdevice vdevice, void *address, size_t size, hailo_dma_buffer_direction_t direction); +/** + * Maps the dmabuf represented by file descriptor @a dmabuf_fd for DMA transfers to/from the given @a device, in the specified + * @a data_direction. + * DMA mapping of buffers in advance may improve the performance of async API. This improvement will become + * apparent when the buffer is reused multiple times across different async operations. + * For low level API (aka ::hailo_input_stream or ::hailo_output_stream), buffers passed to + * ::hailo_stream_write_raw_buffer_async and ::hailo_stream_read_raw_buffer_async can be mapped. + * + * @param[in] device A ::hailo_device object. + * @param[in] dmabuf_fd The file decsriptor of the dmabuf to be mapped + * @param[in] size The buffer's size in bytes + * @param[in] direction The direction of the mapping. For input streams, use `HAILO_DMA_BUFFER_DIRECTION_H2D` + * and for output streams, use `HAILO_DMA_BUFFER_DIRECTION_D2H`. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * + * @note The DMA mapping will be released upon calling ::hailo_device_dma_unmap_dmabuf with @a dmabuf_fd, @a size and + * @a data_direction, or when the @a device object is destroyed. + * @note The dmabuf pointed to by @a dmabuf_fd cannot be released until it is unmapped (via + * ::hailo_device_dma_map_dmabuf or ::hailo_release_device). + * @note This API is currently experimental. + */ +HAILORTAPI hailo_status hailo_device_dma_map_dmabuf(hailo_device device, int dmabuf_fd, size_t size, + hailo_dma_buffer_direction_t direction); + +/** + * Un-maps a dmabuf represented by file descriptor @a dmabuf_fd for DMA transfers to/from the given @a device, in the direction + * @a direction. + * + * @param[in] device A ::hailo_device object. + * @param[in] dmabuf_fd The file descriptor of the dmabuf to be un-mapped. + * @param[in] size The buffer's size in bytes. + * @param[in] direction The direction of the mapping. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * @note This API is currently experimental. + */ +HAILORTAPI hailo_status hailo_device_dma_unmap_dmabuf(hailo_device device, int dmabuf_fd, size_t size, + hailo_dma_buffer_direction_t direction); + +/** + * Maps the dmabuf represented by the file descriptor @a dmabuf_fd for DMA transfers to/from the given @a vdevice, in the specified + * @a data_direction. + * DMA mapping of buffers in advance may improve the performance of async API. This improvement will become + * apparent when the buffer is reused multiple times across different async operations. + * For low level API (aka ::hailo_input_stream or ::hailo_output_stream), buffers passed to + * ::hailo_stream_write_raw_buffer_async and ::hailo_stream_read_raw_buffer_async can be mapped. + * + * @param[in] vdevice A ::hailo_vdevice object. + * @param[in] dmabuf_fd The file descriptor of the dmabuf to be mapped + * @param[in] size The buffer's size in bytes + * @param[in] direction The direction of the mapping. For input streams, use `HAILO_DMA_BUFFER_DIRECTION_H2D` + * and for output streams, use `HAILO_DMA_BUFFER_DIRECTION_D2H`. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * + * @note The DMA mapping will be released upon calling ::hailo_vdevice_dma_unmap_dmabuf with @a dmabuf_fd, @a size and + * @a data_direction, or when the @a vdevice object is destroyed. + * @note The dmabuf pointed to by @a dmabuf_fd cannot be released until it is unmapped (via + * ::hailo_vdevice_dma_unmap_dmabuf or ::hailo_release_vdevice). + * @note This API is currently experimental. + */ +HAILORTAPI hailo_status hailo_vdevice_dma_map_dmabuf(hailo_vdevice vdevice, int dmabuf_fd, size_t size, + hailo_dma_buffer_direction_t direction); + +/** + * Un-maps a dmabuf pointed to by @a dmabuf_fd for DMA transfers to/from the given @a vdevice, in the direction + * @a direction. + * + * @param[in] vdevice A ::hailo_vdevice object. + * @param[in] dmabuf_fd The file descriptor of the dmabuf to be un-mapped. + * @param[in] size The buffer's size in bytes. + * @param[in] direction The direction of the mapping. + * @note This API is currently experimental. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + */ +HAILORTAPI hailo_status hailo_vdevice_dma_unmap_dmabuf(hailo_vdevice vdevice, int dmabuf_fd, size_t size, + hailo_dma_buffer_direction_t direction); + /** @} */ // end of group_buffer_functions /** @defgroup group_stream_functions Stream functions diff --git a/hailort/libhailort/include/hailo/hailort_common.hpp b/hailort/libhailort/include/hailo/hailort_common.hpp index 85076f3..2f11231 100644 --- a/hailort/libhailort/include/hailo/hailort_common.hpp +++ b/hailort/libhailort/include/hailo/hailort_common.hpp @@ -193,6 +193,8 @@ public: return "PLUTO"; case HAILO_ARCH_HAILO15M: return "HAILO15M"; + case HAILO_ARCH_HAILO10H: + return "HAILO10H"; default: return "UNKNOWN ARCHITECTURE"; } @@ -416,12 +418,17 @@ public: static bool is_hailo1x_device_type(const hailo_device_architecture_t dev_arch) { // Compare with HAILO1X device archs - return (HAILO_ARCH_HAILO15H == dev_arch) || (HAILO_ARCH_HAILO15M == dev_arch) || (HAILO_ARCH_PLUTO == dev_arch); + return (HAILO_ARCH_HAILO15H == dev_arch) || (HAILO_ARCH_HAILO15M == dev_arch) || (HAILO_ARCH_PLUTO == dev_arch) || + (HAILO_ARCH_HAILO10H == dev_arch); } static Expected to_device_id(const std::string &device_id); static Expected> to_device_ids_vector(const std::vector &device_ids_str); - static Expected as_hailo_pix_buffer(MemoryView &memory_view, hailo_format_order_t order); + static Expected as_hailo_pix_buffer(MemoryView memory_view, hailo_format_order_t order); + + static bool is_power_measurement_supported(const hailo_device_architecture_t &hw_arch); + static bool is_current_measurement_supported(const hailo_device_architecture_t &hw_arch); + static bool is_temp_measurement_supported(const hailo_device_architecture_t &hw_arch); }; #ifndef HAILO_EMULATOR diff --git a/hailort/libhailort/include/hailo/hailort_dma-heap.h b/hailort/libhailort/include/hailo/hailort_dma-heap.h new file mode 100644 index 0000000..1992c76 --- /dev/null +++ b/hailort/libhailort/include/hailo/hailort_dma-heap.h @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2020-2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file hailort_dma-heap.h + * @brief DMABUF Heaps Userspace + **/ + +#ifndef _HAILO_HAILORT_DMAHEAP_H +#define _HAILO_HAILORT_DMAHEAP_H + +#include +#include + +/** + * DOC: DMABUF Heaps Userspace API + */ + +/* Valid FD_FLAGS are O_CLOEXEC, O_RDONLY, O_WRONLY, O_RDWR */ +#define DMA_HEAP_VALID_FD_FLAGS (O_CLOEXEC | O_ACCMODE) + +/* Currently no heap flags */ +#define DMA_HEAP_VALID_HEAP_FLAGS (0) + +/** + * struct dma_heap_allocation_data - metadata passed from userspace for + * allocations + * @len: size of the allocation + * @fd: will be populated with a fd which provides the + * handle to the allocated dma-buf + * @fd_flags: file descriptor flags used when allocating + * @heap_flags: flags passed to heap + * + * Provided by userspace as an argument to the ioctl + */ +struct dma_heap_allocation_data { + __u64 len; + __u32 fd; + __u32 fd_flags; + __u64 heap_flags; +}; + +#define DMA_HEAP_IOC_MAGIC 'H' + +/** + * DOC: DMA_HEAP_IOCTL_ALLOC - allocate memory from pool + * + * Takes a dma_heap_allocation_data struct and returns it with the fd field + * populated with the dmabuf handle of the allocation. + */ +#define DMA_HEAP_IOCTL_ALLOC _IOWR(DMA_HEAP_IOC_MAGIC, 0x0,\ + struct dma_heap_allocation_data) + +#endif /* _HAILO_HAILORT_DMAHEAP_H */ diff --git a/hailort/libhailort/include/hailo/hef.hpp b/hailort/libhailort/include/hailo/hef.hpp index b44e628..16c18d8 100644 --- a/hailort/libhailort/include/hailo/hef.hpp +++ b/hailort/libhailort/include/hailo/hef.hpp @@ -469,7 +469,7 @@ private: friend class ConfiguredNetworkGroupBase; friend class CoreOp; friend class VDeviceBase; - friend class InferModel; + friend class InferModelBase; #ifdef HAILO_SUPPORT_MULTI_PROCESS friend class HailoRtRpcClient; diff --git a/hailort/libhailort/include/hailo/infer_model.hpp b/hailort/libhailort/include/hailo/infer_model.hpp index c92995e..589a2fd 100644 --- a/hailort/libhailort/include/hailo/infer_model.hpp +++ b/hailort/libhailort/include/hailo/infer_model.hpp @@ -14,14 +14,12 @@ #include "hailo/hef.hpp" #include "hailo/vdevice.hpp" -#include -#include - /** hailort namespace */ namespace hailort { -class ConfiguredInferModelImpl; +class AsyncInferJobBase; +class ConfiguredInferModelBase; class AsyncInferRunnerImpl; /*! Asynchronous inference job representation is used to manage and control an inference job that is running asynchronously. */ @@ -52,11 +50,10 @@ public: void detach(); private: - friend class ConfiguredInferModelImpl; + friend class AsyncInferJobBase; - class Impl; - AsyncInferJob(std::shared_ptr pimpl); - std::shared_ptr m_pimpl; + AsyncInferJob(std::shared_ptr pimpl); + std::shared_ptr m_pimpl; bool m_should_wait_in_dtor; }; @@ -83,6 +80,14 @@ public: /** * Sets the edge's buffer to a new one, of type MemoryView. * + * For best performance and to avoid memory copies, buffers should be aligned to the system PAGE_SIZE. + * + * On output streams, the actual buffer allocated size must be aligned to PAGE_SIZE as well - otherwise some + * memory corruption might occur at the end of the last page. For example, if the buffer size is 4000 + * bytes, the actual buffer size should be at least 4096 bytes. To fill all requirements, it is recommended + * to allocate the buffer with standard page allocation function provided by the os (mmap on linux, + * VirtualAlloc in windows). + * * @param[in] view The new buffer to be set. * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. */ @@ -98,6 +103,8 @@ public: /** * Sets the edge's buffer to a new one, of type hailo_pix_buffer_t. * + * Each plane in the \ref hailo_pix_buffer_t must feet the requirements listed in \ref set_buffer. + * * @param[in] pix_buffer The new buffer to be set. * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. * @note Supported only for inputs. @@ -130,7 +137,7 @@ public: Expected get_dma_buffer(); private: - friend class ConfiguredInferModelImpl; + friend class ConfiguredInferModelBase; friend class AsyncInferRunnerImpl; class Impl; @@ -173,7 +180,7 @@ public: Expected output(const std::string &name); private: - friend class ConfiguredInferModelImpl; + friend class ConfiguredInferModelBase; Bindings(std::unordered_map &&inputs, std::unordered_map &&outputs); @@ -194,15 +201,16 @@ public: * The readiness of the model is determined by the ability to push buffers to the asynchronous inference pipeline. * * @param[in] timeout Amount of time to wait until the model is ready in milliseconds. + * @param[in] frames_count The count of buffers you intent to infer in the next request. Useful for batch inference. * * @return Upon success, returns ::HAILO_SUCCESS. Otherwise: * - If @a timeout has passed and the model is not ready, returns ::HAILO_TIMEOUT. * - In any other error case, returns ::hailo_status error. */ - hailo_status wait_for_async_ready(std::chrono::milliseconds timeout); + hailo_status wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count = 1); /** - * Activates hailo device inner-resources for context_switch inference. + * Activates hailo device inner-resources for inference. * * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns ::hailo_status error. * @note Calling this function is invalid in case scheduler is enabled. @@ -210,10 +218,11 @@ public: hailo_status activate(); /** - * Deactivates hailo device inner-resources for context_switch inference. + * Deactivates hailo device inner-resources for inference. + * @return Returns ::HAILO_SUCCESS. * @note Calling this function is invalid in case scheduler is enabled. */ - void deactivate(); + hailo_status deactivate(); /** * Launches a synchronous inference operation with the provided bindings. @@ -234,12 +243,28 @@ public: * @param[in] callback The function to be called upon completion of the asynchronous inference operation. * * @return Upon success, returns an instance of Expected representing the launched job. - * Otherwise, returns Unexpected of ::hailo_status error. + * Otherwise, returns Unexpected of ::hailo_status error, and the interface shuts down completly. * @note @a callback should execute as quickly as possible. + * @note The bindings' buffers should be kept intact until the async job is completed */ Expected run_async(Bindings bindings, std::function callback = ASYNC_INFER_EMPTY_CALLBACK); + /** + * Launches an asynchronous inference operation with the provided bindings. + * The completion of the operation is notified through the provided callback function. + * Overload for multiple-bindings inference (useful for batch inference). + * + * @param[in] bindings The bindings for the inputs and outputs of the model. + * @param[in] callback The function to be called upon completion of the asynchronous inference operation. + * + * @return Upon success, returns an instance of Expected representing the launched job. + * Otherwise, returns Unexpected of ::hailo_status error, and the interface shuts down completly. + * @note The bindings' buffers should be kept intact until the async job is completed + */ + Expected run_async(const std::vector &bindings, + std::function callback = ASYNC_INFER_EMPTY_CALLBACK); + /** * @return Upon success, returns Expected of LatencyMeasurementResult object containing the output latency result. * Otherwise, returns Unexpected of ::hailo_status error. @@ -294,15 +319,17 @@ public: /** * Shuts the inference down. After calling this method, the model is no longer usable. + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error */ - void shutdown(); + hailo_status shutdown(); private: - friend class InferModel; + friend class InferModelBase; + friend class ConfiguredInferModelBase; - ConfiguredInferModel(std::shared_ptr pimpl); + ConfiguredInferModel(std::shared_ptr pimpl); - std::shared_ptr m_pimpl; + std::shared_ptr m_pimpl; }; /** @@ -332,10 +359,10 @@ struct HAILORTAPI AsyncInferCompletionInfo * This class is used to set up the model for inference and includes methods for setting and getting the model's parameters. * By calling the configure function, the user can create a ConfiguredInferModel object, which is used to run inference. */ -class HAILORTAPI InferModel final +class HAILORTAPI InferModel { public: - ~InferModel() = default; + virtual ~InferModel() = default; /** * Represents the parameters of a stream. @@ -426,7 +453,7 @@ public: /** * Set maximum accumulated mask size for all the detections in a frame. * - * Note: Used in order to change the output buffer frame size, + * @note: Used in order to change the output buffer frame size * in cases where the output buffer is too small for all the segmentation detections. * * @param[in] max_accumulated_mask_size NMS max accumulated mask size. @@ -434,8 +461,13 @@ public: void set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size); private: - friend class InferModel; - friend class VDevice; + friend class InferModelBase; + friend class InferModelHrpcClient; + + float32_t nms_score_threshold() const; + float32_t nms_iou_threshold() const; + uint32_t nms_max_proposals_per_class() const; + uint32_t nms_max_accumulated_mask_size() const; class Impl; InferStream(std::shared_ptr pimpl); @@ -446,14 +478,19 @@ public: /** * @return A constant reference to the Hef object associated with this InferModel. */ - const Hef &hef() const; + virtual const Hef &hef() const = 0; /** * Sets the batch size of the InferModel. + * This parameter determines the number of frames that be sent for inference in a single batch. + * If a scheduler is enabled, this parameter determines the 'burst size' - the max number of frames after which the scheduler will attempt + * to switch to another model. + * + * note: Default value is HAILO_DEFAULT_BATCH_SIZE - means automatic batch determined by hailort. * * @param[in] batch_size The new batch size to be set. */ - void set_batch_size(uint16_t batch_size); + virtual void set_batch_size(uint16_t batch_size) = 0; /** * Sets the power mode of the InferModel. @@ -461,7 +498,7 @@ public: * * @param[in] power_mode The new power mode to be set. */ - void set_power_mode(hailo_power_mode_t power_mode); + virtual void set_power_mode(hailo_power_mode_t power_mode) = 0; /** * Sets the latency measurement flags of the InferModel. @@ -469,7 +506,7 @@ public: * * @param[in] latency The new latency measurement flags to be set. */ - void set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency); + virtual void set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency) = 0; /** * Configures the InferModel object. Also checks the validity of the configuration's formats. @@ -478,7 +515,7 @@ public: * Otherwise, returns Unexpected of ::hailo_status error. * @note InferModel can be configured once. */ - Expected configure(); + virtual Expected configure() = 0; /** * Returns the single input's InferStream object. @@ -487,7 +524,7 @@ public: * @note If InferModel has multiple inputs, will return ::HAILO_INVALID_OPERATION. * In that case - use input(const std::string &name) instead. */ - Expected input(); + virtual Expected input() = 0; /** * Returns the single output's InferStream object. @@ -496,7 +533,7 @@ public: * @note If InferModel has multiple outputs, will return ::HAILO_INVALID_OPERATION. * In that case - use output(const std::string &name) instead. */ - Expected output(); + virtual Expected output() = 0; /** * Gets an input's InferStream object. @@ -504,7 +541,7 @@ public: * @param[in] name The name of the input edge. * @return Upon success, returns Expected of the relevant InferStream object. Otherwise, returns a ::hailo_status error. */ - Expected input(const std::string &name); + virtual Expected input(const std::string &name) = 0; /** * Gets an output's InferStream object. @@ -512,49 +549,33 @@ public: * @param[in] name The name of the output edge. * @return Upon success, returns Expected of the relevant InferStream object. Otherwise, returns a ::hailo_status error. */ - Expected output(const std::string &name); + virtual Expected output(const std::string &name) = 0; /** * @return A constant reference to the vector of input InferStream objects, each representing an input edge. */ - const std::vector &inputs() const; + virtual const std::vector &inputs() const = 0; /** * @return A constant reference to the vector of output InferStream objects, each representing an output edge. */ - const std::vector &outputs() const; + virtual const std::vector &outputs() const = 0; /** * @return A constant reference to a vector of strings, each representing the name of an input stream. */ - const std::vector &get_input_names() const; + virtual const std::vector &get_input_names() const = 0; /** * @return A constant reference to a vector of strings, each representing the name of an output stream. */ - const std::vector &get_output_names() const; - - InferModel(InferModel &&); + virtual const std::vector &get_output_names() const = 0; - Expected configure_for_ut(std::shared_ptr async_infer_runner, + virtual Expected configure_for_ut(std::shared_ptr async_infer_runner, const std::vector &input_names, const std::vector &output_names, - std::shared_ptr net_group = nullptr); - -private: - friend class VDevice; - - InferModel(VDevice &vdevice, Hef &&hef, std::unordered_map &&inputs, - std::unordered_map &&outputs); - - std::reference_wrapper m_vdevice; - Hef m_hef; - std::unordered_map m_inputs; - std::unordered_map m_outputs; - std::vector m_inputs_vector; - std::vector m_outputs_vector; - std::vector m_input_names; - std::vector m_output_names; - ConfigureNetworkParams m_config_params; + const std::unordered_map inputs_frame_sizes = {}, + const std::unordered_map outputs_frame_sizes = {}, + std::shared_ptr net_group = nullptr) = 0; }; } /* namespace hailort */ diff --git a/hailort/libhailort/include/hailo/network_group.hpp b/hailort/libhailort/include/hailo/network_group.hpp index 6d5e170..cfd7f7c 100644 --- a/hailort/libhailort/include/hailo/network_group.hpp +++ b/hailort/libhailort/include/hailo/network_group.hpp @@ -29,7 +29,23 @@ class OpMetadata; using PostProcessOpMetadataPtr = std::shared_ptr; } -using NamedBuffersCallbacks = std::unordered_map>>; +enum class BufferType +{ + UNINITIALIZED, + VIEW, + PIX_BUFFER, + DMA_BUFFER, +}; + +struct BufferRepresentation { + BufferType buffer_type; + union { + MemoryView view; + hailo_dma_buffer_t dma_buffer; + }; +}; + +using NamedBuffersCallbacks = std::unordered_map>>; class InputVStream; class OutputVStream; @@ -211,7 +227,7 @@ public: const std::map &outputs_params) = 0; /** - * Activates hailo device inner-resources for context_switch inference. + * Activates hailo device inner-resources for inference. * @return Upon success, returns Expected of a pointer to ActivatedNetworkGroup object. * Otherwise, returns Unexpected of ::hailo_status error. @@ -219,7 +235,7 @@ public: Expected> activate(); /** - * Activates hailo device inner-resources for context_switch inference. + * Activates hailo device inner-resources for inference. * * @param[in] network_group_params Parameters for the activation. * @return Upon success, returns Expected of a pointer to ActivatedNetworkGroup object. @@ -438,6 +454,10 @@ public: virtual hailo_status set_nms_max_bboxes_per_class(const std::string &edge_name, uint32_t max_bboxes_per_class) = 0; virtual hailo_status set_nms_max_accumulated_mask_size(const std::string &edge_name, uint32_t max_accumulated_mask_size) = 0; + virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) = 0; + virtual Expected get_cache_info() const = 0; + virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) = 0; + protected: ConfiguredNetworkGroup(); diff --git a/hailort/libhailort/include/hailo/stream.hpp b/hailort/libhailort/include/hailo/stream.hpp index 5423ac1..35ad3d8 100644 --- a/hailort/libhailort/include/hailo/stream.hpp +++ b/hailort/libhailort/include/hailo/stream.hpp @@ -48,8 +48,17 @@ public: */ hailo_status status; - const void *buffer_addr; /* Points to the transferred buffer. */ + union { + const void *buffer_addr; /* Points to the transferred buffer. */ + int dmabuf_fd; /* File descriptor to dmabuf*/ + }; size_t buffer_size; /* Size of the transferred buffer. */ + + CompletionInfo(hailo_status status, const uint8_t *addr, size_t size): + status(status), buffer_addr(static_cast(addr)), buffer_size(size) {} + + CompletionInfo(hailo_status status, int fd, size_t size): + status(status), dmabuf_fd(fd), buffer_size(size) {} }; /** Async transfer complete callback prototype. */ @@ -223,6 +232,35 @@ public: */ virtual hailo_status write_async(const void *buffer, size_t size, const TransferDoneCallback &user_callback) = 0; + /** + * Writes the contents of the dmabuf associated with the fd @a dmabuf_fd to the stream asynchronously, initiating a deferred operation that will be + * completed later. + * - If the function call succeeds (i.e., write_async() returns ::HAILO_SUCCESS), the deferred operation has been + * initiated. Until @a user_callback is called, the user cannot change or delete @a dmabuf_fd. + * - If the function call fails (i.e., write_async() returns a status other than ::HAILO_SUCCESS), the deferred + * operation will not be initiated and @a user_callback will not be invoked. The user is free to change or delete + * @a dmabuf_fd. + * - @a user_callback is triggered upon successful completion or failure of the deferred operation. The callback + * receives a \ref CompletionInfo object containing a fd representing the transferred buffer (@a dmabuf_fd) and the + * transfer status (@a status). + * + * @param[in] dmabuf_fd The File descriptor to the dmabuf + * @param[in] size The size of the given buffer, expected to be get_frame_size(). + * @param[in] user_callback The callback that will be called when the transfer is complete + * or has failed. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise: + * - If the stream queue is full, returns ::HAILO_QUEUE_IS_FULL. In this case please wait + * until @a user_callback is called on previous writes, or call wait_for_async_ready(). + * The size of the queue can be determined by calling get_async_max_queue_size(). + * - In any other error case, returns a ::hailo_status error. + * + * @note @a user_callback should run as quickly as possible. + * @note The dmabuf fd must be a linux-system dmabuf fd - otherwise behavior will fail. + * @note This API of write_async is currently experimental. + **/ + virtual hailo_status write_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) = 0; + /** * @returns A ::hailo_stream_info_t object containing the stream's info. */ @@ -284,6 +322,7 @@ public: /** Context passed to the \ref TransferDoneCallback after the async operation is done or has failed. */ struct CompletionInfo { + public: /** * Status of the async transfer. * - ::HAILO_SUCCESS - When transfer is complete successfully. @@ -292,8 +331,18 @@ public: */ hailo_status status; - void *buffer_addr; /* Points to the transferred buffer. */ + union { + void *buffer_addr; /* Points to the transferred buffer. */ + int dmabuf_fd; /* File descriptor to dmabuf*/ + }; + size_t buffer_size; /* Size of the transferred buffer. */ + + CompletionInfo(hailo_status status, uint8_t *addr, size_t size): + status(status), buffer_addr(static_cast(addr)), buffer_size(size) {} + + CompletionInfo(hailo_status status, int fd, size_t size): + status(status), dmabuf_fd(fd), buffer_size(size) {} }; /** Async transfer complete callback prototype. */ @@ -502,6 +551,35 @@ public: */ virtual hailo_status read_async(void *buffer, size_t size, const TransferDoneCallback &user_callback) = 0; + /** + * Reads into dmabuf represented by fd @a dmabuf_fd from the stream asynchronously, initiating a deferred operation that will be completed + * later. + * - If the function call succeeds (i.e., read_async() returns ::HAILO_SUCCESS), the deferred operation has been + * initiated. Until @a user_callback is called, the user cannot change or delete @a dmabuf_fd. + * - If the function call fails (i.e., read_async() returns a status other than ::HAILO_SUCCESS), the deferred + * operation will not be initiated and @a user_callback will not be invoked. The user is free to change or + * delete @a dmabuf_fd. + * - @a user_callback is triggered upon successful completion or failure of the deferred operation. + * The callback receives a \ref CompletionInfo object containing a fd representing the transferred buffer + * (@a dmabuf_fd) and the transfer status (@a status). If the operation has completed successfully, the contents + * of the dmabuf will have been updated by the read operation. + * + * @param[in] dmabuf_fd The File descriptor to the dmabuf + * @param[in] size The size of the given buffer, expected to be get_frame_size(). + * @param[in] user_callback The callback that will be called when the transfer is complete or has failed. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise: + * - If the stream queue is full, returns ::HAILO_QUEUE_IS_FULL. + * In this case, please wait until @a user_callback is called on previous + * reads, or call wait_for_async_ready(). The size of the queue can be + * determined by calling get_async_max_queue_size(). + * - In any other error case, returns a ::hailo_status error. + * @note @a user_callback should execute as quickly as possible. + * @note The dmabuf fd must be a linux-system dmabuf fd - otherwise behavior will fail. + * @note This API of read_async is currently experimental. + */ + virtual hailo_status read_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) = 0; + // get_network_group_activated_event is same as this function virtual EventPtr &get_core_op_activated_event() = 0; protected: diff --git a/hailort/libhailort/include/hailo/vdevice.hpp b/hailort/libhailort/include/hailo/vdevice.hpp index 726c42b..e23ce6f 100644 --- a/hailort/libhailort/include/hailo/vdevice.hpp +++ b/hailort/libhailort/include/hailo/vdevice.hpp @@ -72,10 +72,23 @@ public: * @param[in] network_name A string of the network name (optional). * @return Upon success, returns Expected of a shared pointer of infer model. * Otherwise, returns Unexpected of ::hailo_status error. + * @note the Hef file must be maintained until the completion of the configuration phase. */ virtual Expected> create_infer_model(const std::string &hef_path, const std::string &network_name = ""); + /** + * Creates the infer model from an hef buffer + * + * @param[in] hef_buffer A pointer to a buffer containing the hef file. + * @param[in] network_name A string of the network name (optional). + * @return Upon success, returns Expected of a shared pointer of infer model. + * Otherwise, returns Unexpected of ::hailo_status error. + * @note the Hef buffer must be maintained until the completion of the configuration phase. + */ + virtual Expected> create_infer_model(const MemoryView hef_buffer, + const std::string &network_name = ""); + /** * Gets the underlying physical devices. * @@ -158,6 +171,42 @@ public: */ virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) = 0; + /** + * Maps the dmabuf represented by the file descriptor @a dmabuf_fd for DMA transfers to/from this vdevice, in the specified + * @a data_direction. + * DMA mapping of buffers in advance may improve the performance of async API. This improvement will become + * apparent when the buffer is reused multiple times across different async operations. + * + * For high level API (aka InferModel), buffers bound using ConfiguredInferModel::Bindings::InferStream::set_buffer + * can be mapped. + * + * For low level API (aka InputStream/OutputStream), buffers passed to InputStream::write_async and + * OutputStream::read_async can be mapped. + * + * @param[in] dmabuf_fd The file descriptor of the dmabuf to be mapped. + * @param[in] size The buffer's size in bytes. + * @param[in] direction The direction of the mapping. For input streams, use `HAILO_DMA_BUFFER_DIRECTION_H2D` + * and for output streams, use `HAILO_DMA_BUFFER_DIRECTION_D2H`. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * + * @note The DMA mapping will be released upon calling dma_unmap_dmabuf() with @a dmabuf_fd, @a size and @a data_direction, or + * when the @a VDevice object is destroyed. + */ + virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) = 0; + + /** + * Un-maps a dmabuf buffer represented by the file descriptor @a dmabuf_fd for DMA transfers to/from this vdevice, in the direction + * @a direction. + * + * @param[in] dmabuf_fd The file descriptor of the dmabuf to be un-mapped. + * @param[in] size The buffer's size in bytes. + * @param[in] direction The direction of the mapping. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + */ + virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) = 0; + virtual hailo_status before_fork(); virtual hailo_status after_fork_in_parent(); virtual hailo_status after_fork_in_child(); @@ -169,6 +218,7 @@ public: VDevice &operator=(VDevice &&other) = delete; static bool service_over_ip_mode(); + static bool force_hrpc_client(); protected: VDevice() = default; diff --git a/hailort/libhailort/src/CMakeLists.txt b/hailort/libhailort/src/CMakeLists.txt index b9921de..5136024 100644 --- a/hailort/libhailort/src/CMakeLists.txt +++ b/hailort/libhailort/src/CMakeLists.txt @@ -54,13 +54,15 @@ relative_to_absolute_paths(C_OS_SOURCES ${C_OS_SOURCES}) relative_to_absolute_paths(COMMON_C_SOURCES ${COMMON_C_SOURCES}) relative_to_absolute_paths(HAILO_OS_DIR ${HAILO_OS_DIR}) relative_to_absolute_paths(HAILO_FULL_OS_DIR ${HAILO_FULL_OS_DIR}) +relative_to_absolute_paths(DRIVER_OS_DIR ${DRIVER_OS_DIR}) relative_to_absolute_paths(HAILO_DRIVER_SRC_FILES ${HAILO_DRIVER_SRC_FILES}) set(HAILO_OS_DIR ${HAILO_OS_DIR} CACHE INTERNAL "Absolute path of os-dir") set(HAILO_FULL_OS_DIR ${HAILO_FULL_OS_DIR} CACHE INTERNAL "Absolute Full path of os-dir") +set(DRIVER_OS_DIR ${DRIVER_OS_DIR} CACHE INTERNAL "Absolute Full path of driver os-dir") set(HAILO_DRIVER_SRC_FILES ${HAILO_DRIVER_SRC_FILES} CACHE INTERNAL "Absolute Full path of driver src files") set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} CACHE INTERNAL "Absolute paths of hailort's cpp source files") set(COMMON_C_SOURCES ${COMMON_C_SOURCES} CACHE INTERNAL "Absolute paths of common source files") -set(HAILORT_SRCS_ABS ${HAILORT_CPP_SOURCES} ${HAILORT_COMMON_CPP_SOURCES} ${COMMON_C_SOURCES} CACHE INTERNAL "All absolute paths of hailort's source files") +set(HAILORT_SRCS_ABS ${HAILORT_CPP_SOURCES} ${HAILORT_COMMON_CPP_SOURCES} ${COMMON_C_SOURCES} ${HRPC_CPP_SOURCES} ${HRPC_PROTOCOL_CPP_SOURCES} CACHE INTERNAL "All absolute paths of hailort's source files") SET_SOURCE_FILES_PROPERTIES(${C_SOURCES} PROPERTIES LANGUAGE CXX) add_library(libhailort SHARED ${HAILORT_SRCS_ABS}) @@ -91,6 +93,7 @@ target_link_libraries(libhailort PRIVATE scheduler_mon_proto) target_link_libraries(libhailort PRIVATE spdlog::spdlog) target_link_libraries(libhailort PRIVATE readerwriterqueue) target_link_libraries(libhailort PRIVATE Eigen3::Eigen) +target_link_libraries(libhailort PRIVATE rpc_proto) if(HAILO_BUILD_SERVICE) target_link_libraries(libhailort PRIVATE grpc++_unsecure) target_link_libraries(libhailort PRIVATE hailort_rpc_grpc_proto) @@ -152,6 +155,7 @@ target_include_directories(libhailort $ $ $ + $ ) target_compile_definitions(libhailort PUBLIC diff --git a/hailort/libhailort/src/core_op/CMakeLists.txt b/hailort/libhailort/src/core_op/CMakeLists.txt index b2d401d..432b3e8 100644 --- a/hailort/libhailort/src/core_op/CMakeLists.txt +++ b/hailort/libhailort/src/core_op/CMakeLists.txt @@ -7,12 +7,14 @@ set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/resource_manager_builder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/config_buffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/intermediate_buffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/cache_buffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/channel_allocator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/action_list_buffer_builder/control_action_list_buffer_builder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/periph_calculator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/internal_buffer_manager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/internal_buffer_planner.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/cache_manager.cpp ) set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/core_op/core_op.cpp b/hailort/libhailort/src/core_op/core_op.cpp index d195d04..87a91e5 100644 --- a/hailort/libhailort/src/core_op/core_op.cpp +++ b/hailort/libhailort/src/core_op/core_op.cpp @@ -29,7 +29,7 @@ namespace hailort CoreOp::CoreOp( const ConfigureNetworkParams &config_params, std::shared_ptr metadata, - ActiveCoreOpHolder &active_core_op_holder, hailo_status &status) : + ActiveCoreOpHolder &active_core_op_holder, hailo_status &status, bool is_scheduled) : m_config_params(config_params), m_active_core_op_holder(active_core_op_holder), m_min_configured_batch_size(get_smallest_configured_batch_size(config_params)), @@ -38,13 +38,15 @@ CoreOp::CoreOp( m_metadata(metadata), m_vdevice_core_op_handle(INVALID_CORE_OP_HANDLE) { - auto event = Event::create_shared(Event::State::not_signalled); - if (!event) { - LOGGER__ERROR("Failed to create activation event"); - status = event.status(); - return; + if (!is_scheduled) { + auto event = Event::create_shared(Event::State::not_signalled); + if (!event) { + LOGGER__ERROR("Failed to create activation event"); + status = event.status(); + return; + } + m_core_op_activated_event = event.release(); } - m_core_op_activated_event = event.release(); m_activation_time_accumulator = make_shared_nothrow>("activation_time"); if (nullptr == m_activation_time_accumulator) { @@ -65,12 +67,10 @@ CoreOp::CoreOp( Expected get_latency(LatencyMeterPtr &latency_meter, bool clear) { - auto hw_latency = latency_meter->get_latency(clear); - if (HAILO_NOT_AVAILABLE == hw_latency.status()) { - return make_unexpected(HAILO_NOT_AVAILABLE); - } - CHECK_EXPECTED(hw_latency, "Failed getting latency"); - return hw_latency.release(); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_NOT_AVAILABLE, auto hw_latency, + latency_meter->get_latency(clear), "Failed getting latency"); + + return hw_latency; } /* Network group base functions */ @@ -79,9 +79,7 @@ Expected CoreOp::get_latency_measurement(const std::st bool clear = ((m_config_params.latency & HAILO_LATENCY_CLEAR_AFTER_GET) == HAILO_LATENCY_CLEAR_AFTER_GET); LatencyMeasurementResult result = {}; - auto latency_meters_exp = get_latency_meters(); - CHECK_EXPECTED(latency_meters_exp); - auto latency_meters = latency_meters_exp.release(); + TRY(auto latency_meters, get_latency_meters()); if (network_name.empty()) { if (1 != m_input_streams.size()) { @@ -94,7 +92,7 @@ Expected CoreOp::get_latency_measurement(const std::st if (HAILO_NOT_AVAILABLE == hw_latency.status()) { continue; } - CHECK_EXPECTED(hw_latency); + CHECK_EXPECTED(hw_latency); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here latency_sum += hw_latency.value(); measurements_count++; } @@ -108,12 +106,10 @@ Expected CoreOp::get_latency_measurement(const std::st LOGGER__DEBUG("No latency measurements was found for network {}", network_name); return make_unexpected(HAILO_NOT_FOUND); } - auto hw_latency = get_latency(latency_meters->at(network_name), clear); - if (HAILO_NOT_AVAILABLE == hw_latency.status()) { - return make_unexpected(HAILO_NOT_AVAILABLE); - } - CHECK_EXPECTED(hw_latency); - result.avg_hw_latency = hw_latency.value(); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_NOT_AVAILABLE, const auto hw_latency, + get_latency(latency_meters->at(network_name), clear)); + + result.avg_hw_latency = hw_latency; } return result; } @@ -171,10 +167,9 @@ hailo_status CoreOp::deactivate() CHECK(!is_scheduled(), HAILO_INVALID_OPERATION, "Manually deactivate a core-op is not allowed when the core-op scheduler is active!"); - auto core_op_ref = m_active_core_op_holder.get(); - CHECK_EXPECTED_AS_STATUS(core_op_ref, "Trying to deactivate while no network is running"); + TRY(auto core_op_ref, m_active_core_op_holder.get(), "Trying to deactivate while no network is running"); - CHECK(this == std::addressof(core_op_ref->get()), HAILO_INTERNAL_FAILURE, + CHECK(this == std::addressof(core_op_ref.get()), HAILO_INTERNAL_FAILURE, "Trying to deactivate different core-op"); m_active_core_op_holder.clear(); @@ -393,24 +388,28 @@ Expected CoreOp::get_intermediate_buffer(const IntermediateBufferKey &) return make_unexpected(HAILO_NOT_SUPPORTED); } +Expected CoreOp::get_cache_buffer(uint32_t) +{ + LOGGER__ERROR("Getting cache buffer is not supported for this core op"); + return make_unexpected(HAILO_NOT_SUPPORTED); +} + +Expected> CoreOp::get_cache_buffers() +{ + LOGGER__ERROR("Getting cache buffers is not supported for this core op"); + return make_unexpected(HAILO_NOT_SUPPORTED); +} + hailo_status CoreOp::wrap_streams_for_remote_process() { for (auto &input_stream_pair : m_input_streams) { auto base_stream = input_stream_pair.second; - - auto remote_proc_stream = RemoteProcessInputStream::create(base_stream); - CHECK_EXPECTED_AS_STATUS(remote_proc_stream); - - input_stream_pair.second = remote_proc_stream.release(); + TRY(input_stream_pair.second, RemoteProcessInputStream::create(base_stream)); } for (auto &output_stream_pair : m_output_streams) { auto base_stream = output_stream_pair.second; - - auto remote_proc_stream = RemoteProcessOutputStream::create(base_stream); - CHECK_EXPECTED_AS_STATUS(remote_proc_stream); - - output_stream_pair.second = remote_proc_stream.release(); + TRY(output_stream_pair.second, RemoteProcessOutputStream::create(base_stream)); } return HAILO_SUCCESS; @@ -421,15 +420,13 @@ Expected CoreOp::get_async_max_queue_size() const size_t queue_size = std::numeric_limits::max(); for (const auto &input : m_input_streams) { - auto stream_queue_size = input.second->get_async_max_queue_size(); - CHECK_EXPECTED(stream_queue_size); - queue_size = std::min(queue_size, *stream_queue_size); + TRY(auto stream_queue_size, input.second->get_async_max_queue_size()); + queue_size = std::min(queue_size, stream_queue_size); } for (const auto &output : m_output_streams) { - auto stream_queue_size = output.second->get_async_max_queue_size(); - CHECK_EXPECTED(stream_queue_size); - queue_size = std::min(queue_size, *stream_queue_size); + TRY(auto stream_queue_size, output.second->get_async_max_queue_size()); + queue_size = std::min(queue_size, stream_queue_size); } return queue_size; @@ -473,9 +470,7 @@ const ConfigureNetworkParams CoreOp::get_config_params() const Expected> CoreOp::create_input_stream_from_config_params(Device &device, const hailo_stream_parameters_t &stream_params, const std::string &stream_name) { - auto layer_info = get_layer_info(stream_name); - CHECK_EXPECTED(layer_info); - + TRY(const auto layer_info, get_layer_info(stream_name)); CHECK_AS_EXPECTED(device.is_stream_interface_supported(stream_params.stream_interface), HAILO_INVALID_OPERATION, "Device does not supports the given stream interface streams. Please update input_stream_params for stream {}.", stream_name); @@ -486,27 +481,21 @@ Expected> CoreOp::create_input_stream_from_conf // Fallthrough case HAILO_STREAM_INTERFACE_INTEGRATED: { - auto input_stream_exp = create_vdma_input_stream(device, stream_name, layer_info.value(), stream_params); - CHECK_EXPECTED(input_stream_exp); - input_stream = input_stream_exp.release(); + TRY(input_stream, create_vdma_input_stream(device, stream_name, layer_info, stream_params)); break; } case HAILO_STREAM_INTERFACE_ETH: { - auto input_stream_exp = EthernetInputStream::create(device, - layer_info.value(), stream_params.eth_input_params, m_core_op_activated_event); - CHECK_EXPECTED(input_stream_exp); - input_stream = input_stream_exp.release(); + TRY(input_stream, EthernetInputStream::create(device, + layer_info, stream_params.eth_input_params, m_core_op_activated_event)); break; } case HAILO_STREAM_INTERFACE_MIPI: { - auto input_stream_exp = MipiInputStream::create(device, - layer_info.value(), stream_params.mipi_input_params, m_core_op_activated_event); - CHECK_EXPECTED(input_stream_exp); - input_stream = input_stream_exp.release(); + TRY(input_stream, MipiInputStream::create(device, + layer_info, stream_params.mipi_input_params, m_core_op_activated_event)); break; } @@ -590,19 +579,17 @@ Expected> CoreOp::create_vdma_input_stream(Devi HAILO_INTERNAL_FAILURE, "Invalid device type"); VdmaDevice *vdma_device = reinterpret_cast(&device); - auto vdma_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(stream_name); - CHECK_EXPECTED(vdma_channel_ptr_exp, "Failed to get vdma channel for output stream {}", stream_name); + TRY(auto vdma_channel, get_boundary_vdma_channel_by_stream_name(stream_name), + "Failed to get vdma channel for output stream {}", stream_name); - return VdmaInputStream::create(stream_params.stream_interface, *vdma_device, vdma_channel_ptr_exp.value(), + return VdmaInputStream::create(stream_params.stream_interface, *vdma_device, vdma_channel, layer_info, m_core_op_activated_event); } Expected> CoreOp::create_output_stream_from_config_params(Device &device, const hailo_stream_parameters_t &stream_params, const std::string &stream_name) { - auto layer_info = get_layer_info(stream_name); - CHECK_EXPECTED(layer_info); - + TRY(const auto layer_info, get_layer_info(stream_name)); CHECK_AS_EXPECTED(device.is_stream_interface_supported(stream_params.stream_interface), HAILO_INVALID_OPERATION, "Device does not supports the given stream interface streams. Please update input_stream_params for stream {}.", stream_name); @@ -613,19 +600,14 @@ Expected> CoreOp::create_output_stream_from_co // Fallthrough case HAILO_STREAM_INTERFACE_INTEGRATED: { - auto output_stream_exp = create_vdma_output_stream(device, stream_name, layer_info.value(), stream_params); - CHECK_EXPECTED(output_stream_exp); - output_stream = output_stream_exp.release(); + TRY(output_stream, create_vdma_output_stream(device, stream_name, layer_info, stream_params)); break; } case HAILO_STREAM_INTERFACE_ETH: { - auto output_stream_exp = EthernetOutputStream::create(device, - layer_info.value(), stream_params.eth_output_params, - m_core_op_activated_event); - CHECK_EXPECTED(output_stream_exp); - output_stream = output_stream_exp.release(); + TRY(output_stream, EthernetOutputStream::create(device, + layer_info, stream_params.eth_output_params, m_core_op_activated_event)); break; } @@ -634,7 +616,7 @@ Expected> CoreOp::create_output_stream_from_co return make_unexpected(HAILO_NOT_IMPLEMENTED); } - if (HAILO_FORMAT_ORDER_HAILO_NMS == layer_info->format.order) { + if (HAILO_FORMAT_ORDER_HAILO_NMS == layer_info.format.order) { // In NMS we create some new stream object that wraps the original stream (and converts // bbox/burst reads into frame reads). // After HRT-10553 is implemented, we won't need this wrapper anymore. @@ -643,10 +625,8 @@ Expected> CoreOp::create_output_stream_from_co const auto batch_size = get_smallest_configured_batch_size(m_config_params); const auto max_queue_size = batch_size * MAX_ACTIVE_TRANSFERS_SCALE; - auto nms_stream = NmsOutputStream::create(base_stream, layer_info.value(), max_queue_size, - m_core_op_activated_event, stream_params.stream_interface); - CHECK_EXPECTED(nms_stream); - output_stream = nms_stream.release(); + TRY(output_stream, NmsOutputStream::create(base_stream, layer_info, max_queue_size, + m_core_op_activated_event, stream_params.stream_interface)); } return output_stream; @@ -660,14 +640,13 @@ Expected> CoreOp::create_vdma_output_stream(De HAILO_INTERNAL_FAILURE, "Invalid device type"); VdmaDevice *vdma_device = reinterpret_cast(&device); - auto batch_size_exp = get_stream_batch_size(stream_name); - CHECK_EXPECTED(batch_size_exp); + TRY(auto vdma_channel, get_boundary_vdma_channel_by_stream_name(stream_name), + "Failed to get vdma channel for output stream {}", stream_name); - auto vdma_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(stream_name); - CHECK_EXPECTED(vdma_channel_ptr_exp, "Failed to get vdma channel for output stream {}", stream_name); + TRY(auto result, VdmaOutputStream::create(stream_params.stream_interface, *vdma_device, vdma_channel, + layer_info, m_core_op_activated_event)); - return VdmaOutputStream::create(stream_params.stream_interface, *vdma_device, vdma_channel_ptr_exp.value(), - layer_info, m_core_op_activated_event); + return Expected>(result); } hailo_status CoreOp::create_streams_from_config_params(Device &device) @@ -676,23 +655,23 @@ hailo_status CoreOp::create_streams_from_config_params(Device &device) switch (stream_parameters_pair.second.direction) { case HAILO_H2D_STREAM: { - auto stream = create_input_stream_from_config_params(device, - stream_parameters_pair.second, - stream_parameters_pair.first); - CHECK_EXPECTED_AS_STATUS(stream); + TRY(auto stream, + create_input_stream_from_config_params(device, + stream_parameters_pair.second, + stream_parameters_pair.first)); - auto status = add_input_stream(stream.release(), stream_parameters_pair.second); + auto status = add_input_stream(std::move(stream), stream_parameters_pair.second); CHECK_SUCCESS(status); } break; case HAILO_D2H_STREAM: { - auto stream = create_output_stream_from_config_params(device, - stream_parameters_pair.second, - stream_parameters_pair.first); - CHECK_EXPECTED_AS_STATUS(stream); + TRY(auto stream, + create_output_stream_from_config_params(device, + stream_parameters_pair.second, + stream_parameters_pair.first)); - auto status = add_output_stream(stream.release(), stream_parameters_pair.second); + auto status = add_output_stream(std::move(stream), stream_parameters_pair.second); CHECK_SUCCESS(status); } break; @@ -707,28 +686,24 @@ hailo_status CoreOp::create_streams_from_config_params(Device &device) Expected CoreOp::get_input_streams_by_network(const std::string &network_name) { - auto input_stream_infos = m_metadata->get_input_stream_infos(network_name); - CHECK_EXPECTED(input_stream_infos); + TRY(auto input_stream_infos, m_metadata->get_input_stream_infos(network_name)); InputStreamRefVector result; - for (auto &stream_info : input_stream_infos.value()) { - auto stream_ref = get_input_stream_by_name(stream_info.name); - CHECK_EXPECTED(stream_ref); - result.emplace_back(stream_ref.release()); + for (auto &stream_info : input_stream_infos) { + TRY(auto stream_ref, get_input_stream_by_name(stream_info.name)); + result.emplace_back(stream_ref); } return result; } Expected CoreOp::get_output_streams_by_network(const std::string &network_name) { - auto output_stream_infos = m_metadata->get_output_stream_infos(network_name); - CHECK_EXPECTED(output_stream_infos); + TRY(auto output_stream_infos, m_metadata->get_output_stream_infos(network_name)); OutputStreamRefVector result; - for (auto &stream_info : output_stream_infos.value()) { - auto stream_ref = get_output_stream_by_name(stream_info.name); - CHECK_EXPECTED(stream_ref); - result.emplace_back(stream_ref.release()); + for (auto &stream_info : output_stream_infos) { + TRY(auto stream_ref, get_output_stream_by_name(stream_info.name)); + result.emplace_back(stream_ref); } return result; } diff --git a/hailort/libhailort/src/core_op/core_op.hpp b/hailort/libhailort/src/core_op/core_op.hpp index 17f350e..9a8968f 100644 --- a/hailort/libhailort/src/core_op/core_op.hpp +++ b/hailort/libhailort/src/core_op/core_op.hpp @@ -101,6 +101,8 @@ public: bool is_default_batch_size() const; virtual Expected get_intermediate_buffer(const IntermediateBufferKey &key); + virtual Expected get_cache_buffer(uint32_t cache_id); + virtual Expected> get_cache_buffers(); hailo_status wrap_streams_for_remote_process(); @@ -119,12 +121,19 @@ public: */ hailo_status infer_async(InferRequest &&request); + virtual bool has_caches() const = 0; + virtual Expected get_cache_read_size() const = 0; + virtual Expected get_cache_write_size() const = 0; + virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) = 0; + virtual Expected get_cache_info() const = 0; + virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) = 0; + std::map> m_input_streams; std::map> m_output_streams; protected: CoreOp(const ConfigureNetworkParams &config_params, std::shared_ptr metadata, - ActiveCoreOpHolder &active_core_op_holder, hailo_status &status); + ActiveCoreOpHolder &active_core_op_holder, hailo_status &status, bool is_scheduled = false); Expected> create_output_stream_from_config_params(Device &device, const hailo_stream_parameters_t &stream_params, const std::string &stream_name); diff --git a/hailort/libhailort/src/core_op/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.cpp b/hailort/libhailort/src/core_op/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.cpp index ee9b179..7794152 100644 --- a/hailort/libhailort/src/core_op/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.cpp @@ -8,6 +8,8 @@ **/ #include "ddr_action_list_buffer_builder.hpp" +#include "common/os_utils.hpp" +#include "vdma/integrated/integrated_device.hpp" namespace hailort { @@ -21,9 +23,10 @@ namespace hailort // Like the dsp etc...need to check with them before doing so. For now - this should almost always retirn in the mapped area and we will verify // to double check -DDRActionListBufferBuilder::DDRActionListBufferBuilder(vdma::ContinuousBuffer &&buffer) : +DDRActionListBufferBuilder::DDRActionListBufferBuilder(void* user_address, uint64_t dma_address) : ActionListBufferBuilder(ActionListBufferBuilder::Type::DDR), - m_action_list_buffer(std::move(buffer)), + m_user_address(user_address), + m_dma_address(dma_address), m_write_offset(0), m_current_context_info{} {} @@ -39,18 +42,20 @@ bool DDRActionListBufferBuilder::verify_dma_addr(vdma::ContinuousBuffer &buffer) } Expected> DDRActionListBufferBuilder::create(size_t num_contexts, - HailoRTDriver &driver) + VdmaDevice &vdma_device) { - // Try to allocate continous buffer for action list in DDR - auto continous_alloc = vdma::ContinuousBuffer::create(num_contexts * - sizeof(CONTROL_PROTOCOL__context_switch_context_info_chunk_t), driver); - - // TODO HRT-12512 - Add fallback to Control if continous buffer allocation fails - CHECK_EXPECTED(continous_alloc); - // Verify that continous buffer is in allocated region - CHECK_AS_EXPECTED(verify_dma_addr(continous_alloc.value()), HAILO_INTERNAL_FAILURE, - "Failed to allocate continous buffer in M4 mapped memory region"); - return make_shared_nothrow(continous_alloc.release()); + auto integrated_device = dynamic_cast(&vdma_device); + + size_t size_of_contexts = HailoRTCommon::align_to(num_contexts * + sizeof(CONTROL_PROTOCOL__context_switch_context_info_chunk_t), OsUtils::get_page_size()); + + TRY(auto addr_pair, integrated_device->allocate_infinite_action_list_buffer(size_of_contexts)); + + auto ddr_action_list_buiffer_builder = make_shared_nothrow( + addr_pair.first, addr_pair.second); + CHECK_NOT_NULL_AS_EXPECTED(ddr_action_list_buiffer_builder, HAILO_OUT_OF_HOST_MEMORY); + + return ddr_action_list_buiffer_builder; } hailo_status DDRActionListBufferBuilder::write_action(MemoryView action, @@ -77,8 +82,8 @@ hailo_status DDRActionListBufferBuilder::write_action(MemoryView action, if (is_last_action_in_context) { const auto write_size = sizeof(CONTROL_PROTOCOL__context_switch_context_info_chunk_t); - auto status = m_action_list_buffer.write(&m_current_context_info, write_size, m_write_offset); - CHECK_SUCCESS(status); + memcpy(static_cast(reinterpret_cast(m_user_address) + m_write_offset), &m_current_context_info, + write_size); m_write_offset += write_size; } @@ -87,7 +92,7 @@ hailo_status DDRActionListBufferBuilder::write_action(MemoryView action, uint64_t DDRActionListBufferBuilder::get_mapped_buffer_dma_address() const { - return m_action_list_buffer.dma_address(); + return m_dma_address; } } /* namespace hailort */ \ No newline at end of file diff --git a/hailort/libhailort/src/core_op/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.hpp b/hailort/libhailort/src/core_op/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.hpp index 05b4b21..ddb4ee1 100644 --- a/hailort/libhailort/src/core_op/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/action_list_buffer_builder/ddr_action_list_buffer_builder.hpp @@ -13,6 +13,7 @@ #include "context_switch_defs.h" #include "core_op/resource_manager/action_list_buffer_builder/action_list_buffer_builder.hpp" #include "vdma/memory/continuous_buffer.hpp" +#include "vdma/vdma_device.hpp" #define DDR_ACTION_LIST_ENV_VAR ("HAILO_DDR_ACTION_LIST") #define DDR_ACTION_LIST_ENV_VAR_VALUE ("1") @@ -22,18 +23,21 @@ namespace hailort class DDRActionListBufferBuilder : public ActionListBufferBuilder { public: - DDRActionListBufferBuilder(vdma::ContinuousBuffer &&buffer); + DDRActionListBufferBuilder(void* user_address, uint64_t dma_address); virtual ~DDRActionListBufferBuilder() = default; - static Expected> create(size_t num_contexts, HailoRTDriver &driver); + static Expected> create(size_t num_contexts, VdmaDevice &vdma_device); virtual hailo_status write_action(MemoryView action, CONTROL_PROTOCOL__context_switch_context_type_t context_type, bool is_new_context, bool last_action_buffer_in_context) override; virtual uint64_t get_mapped_buffer_dma_address() const override; -private: - vdma::ContinuousBuffer m_action_list_buffer; + // TODO: HRT-12512 : Can remove this check when / if continuous buffer comes from designated region static bool verify_dma_addr(vdma::ContinuousBuffer &buffer); +private: + // vdma::ContinuousBuffer m_action_list_buffer; + void* m_user_address; + uint64_t m_dma_address; size_t m_write_offset; CONTROL_PROTOCOL__context_switch_context_info_chunk_t m_current_context_info; }; diff --git a/hailort/libhailort/src/core_op/resource_manager/cache_buffer.cpp b/hailort/libhailort/src/core_op/resource_manager/cache_buffer.cpp new file mode 100644 index 0000000..d92e2d1 --- /dev/null +++ b/hailort/libhailort/src/core_op/resource_manager/cache_buffer.cpp @@ -0,0 +1,109 @@ + +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file cache_buffer.cpp + * @brief Wrapper for intermediate buffers used as caches + **/ + +#include "cache_buffer.hpp" +#include "hailo/hailort.h" +#include "vdma/memory/sg_buffer.hpp" + +namespace hailort +{ + +Expected CacheBuffer::create(HailoRTDriver &driver, uint32_t cache_size, + uint32_t input_size, uint32_t output_size) +{ + CHECK(cache_size > 0, HAILO_INVALID_ARGUMENT); + CHECK((input_size > 0) && (input_size < cache_size), HAILO_INVALID_ARGUMENT, + "Invalid cache input size: {} (cache size: {})", input_size, cache_size); + CHECK((output_size > 0) && (output_size < cache_size), HAILO_INVALID_ARGUMENT, + "Invalid cache output size: {} (cache size: {})", output_size, cache_size); + + // Cache buffers are by sg buffers + TRY(auto buffer, vdma::SgBuffer::create(driver, cache_size, HailoRTDriver::DmaDirection::BOTH)); + auto buffer_ptr = make_shared_nothrow(std::move(buffer)); + CHECK_NOT_NULL(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY); + return CacheBuffer(cache_size, input_size, output_size, buffer_ptr); +} + +CacheBuffer::CacheBuffer(uint32_t cache_size, uint32_t input_size, uint32_t output_size, + std::shared_ptr backing_buffer) : + m_cache_size(cache_size), + m_input_size(input_size), + m_output_size(output_size), + m_backing_buffer(backing_buffer) +{} + +ExpectedRef CacheBuffer::set_input_channel(HailoRTDriver &driver, vdma::ChannelId channel_id) +{ + if (m_cache_input) { + return std::ref(*m_cache_input); + } + + static const auto SINGLE_BATCH = 1; + static const auto BUFFER_START = 0; + TRY(auto intermediate_buffer, IntermediateBuffer::create_shared(driver, m_input_size, SINGLE_BATCH, channel_id, + IntermediateBuffer::StreamingType::BURST, m_backing_buffer, BUFFER_START)); + m_cache_input = intermediate_buffer; + return std::ref(*m_cache_input); +} + +ExpectedRef CacheBuffer::set_output_channel(HailoRTDriver &driver, vdma::ChannelId channel_id) +{ + if (m_cache_output) { + return std::ref(*m_cache_output); + } + + static const auto SINGLE_BATCH = 1; + static const auto BUFFER_START = 0; + TRY(auto intermediate_buffer, IntermediateBuffer::create_shared(driver, m_output_size, SINGLE_BATCH, channel_id, + IntermediateBuffer::StreamingType::BURST, m_backing_buffer, BUFFER_START)); + m_cache_output = intermediate_buffer; + return std::ref(*m_cache_output); +} + +ExpectedRef CacheBuffer::get_input() +{ + CHECK(m_cache_input, HAILO_INTERNAL_FAILURE, "Input not set"); + return std::ref(*m_cache_input); +} + +ExpectedRef CacheBuffer::get_output() +{ + CHECK(m_cache_output, HAILO_INTERNAL_FAILURE, "Output not set"); + return std::ref(*m_cache_output); +} + +Expected CacheBuffer::read_entire_cache() +{ + CHECK(m_cache_input && m_cache_output, HAILO_INTERNAL_FAILURE, "Input or output not set"); + + return m_cache_input->read(m_cache_size); +} + +uint32_t CacheBuffer::cache_size() const +{ + return m_cache_size; +} + +uint32_t CacheBuffer::input_size() const +{ + return m_input_size; +} + +uint32_t CacheBuffer::output_size() const +{ + return m_output_size; +} + +bool CacheBuffer::is_configured() const +{ + return m_cache_input && m_cache_output; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/core_op/resource_manager/cache_buffer.hpp b/hailort/libhailort/src/core_op/resource_manager/cache_buffer.hpp new file mode 100644 index 0000000..52bf4f0 --- /dev/null +++ b/hailort/libhailort/src/core_op/resource_manager/cache_buffer.hpp @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file cache_buffer.hpp + * @brief Wrapper for intermediate buffers used as caches + **/ + +#ifndef _HAILO_CACHE_BUFFER_HPP_ +#define _HAILO_CACHE_BUFFER_HPP_ + +#include "hailo/hailort.h" +#include "core_op/resource_manager/intermediate_buffer.hpp" + +namespace hailort +{ + +class CacheBuffer final +{ +public: + static Expected create(HailoRTDriver &driver, uint32_t cache_size, + uint32_t input_size, uint32_t output_size); + + CacheBuffer(CacheBuffer &&) = default; + CacheBuffer(const CacheBuffer &) = delete; + CacheBuffer &operator=(CacheBuffer &&) = delete; + CacheBuffer &operator=(const CacheBuffer &) = delete; + ~CacheBuffer() = default; + + // Set input/output channels to/from the cache. Will only be set once for each direction. + // (subsequent calls will return the same IntermediateBuffer.) + ExpectedRef set_input_channel(HailoRTDriver &driver, vdma::ChannelId channel_id); + ExpectedRef set_output_channel(HailoRTDriver &driver, vdma::ChannelId channel_id); + ExpectedRef get_input(); + ExpectedRef get_output(); + Expected read_entire_cache(); + uint32_t cache_size() const; + uint32_t input_size() const; + uint32_t output_size() const; + // Returns true if both input and output channels are set. + bool is_configured() const; + +private: + CacheBuffer(uint32_t cache_size, uint32_t input_size, uint32_t output_size, + std::shared_ptr backing_buffer); + + const uint32_t m_cache_size; + const uint32_t m_input_size; + const uint32_t m_output_size; + const std::shared_ptr m_backing_buffer; + // Each cache buffer has an input and output IntermediateBuffer - + // * They both share the same backing buffer. + // * They each have separate descriptor lists that will be programmed separately. + // * This way we can read/write/reprogram the cache buffer without affecting the other direction. + std::shared_ptr m_cache_input; + std::shared_ptr m_cache_output; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_CACHE_BUFFER_HPP_ */ diff --git a/hailort/libhailort/src/core_op/resource_manager/cache_manager.cpp b/hailort/libhailort/src/core_op/resource_manager/cache_manager.cpp new file mode 100644 index 0000000..1e1233e --- /dev/null +++ b/hailort/libhailort/src/core_op/resource_manager/cache_manager.cpp @@ -0,0 +1,331 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file cache_manager.cpp + * @brief Manges creation and configuration of cache buffers + **/ + +#include "cache_manager.hpp" +#include "hailo/hailort.h" + +namespace hailort +{ + +Expected CacheManager::create_shared(HailoRTDriver &driver) +{ + auto cache_manager = make_shared_nothrow(driver); + CHECK_NOT_NULL(cache_manager, HAILO_OUT_OF_HOST_MEMORY); + + return cache_manager; +} + +CacheManager::CacheManager(HailoRTDriver &driver) : + m_driver(driver), + m_caches_created(false), + m_initialized(false), + m_cache_input_size(0), + m_cache_output_size(0), + m_cache_size(0), + m_read_offset_bytes(0), + m_write_offset_bytes_delta(0), + m_cache_buffers(), + m_uninitialized_caches() +{ +} + +hailo_status CacheManager::create_caches_from_core_op(std::shared_ptr core_op_metadata) +{ + if (m_caches_created) { + // Already created caches, nothing to do + // In debug, validate that the cache sizes + ids are the same as the ones we already have + assert(m_cache_input_size == get_cache_input_size(core_op_metadata)); + assert(m_cache_output_size == get_cache_output_size(core_op_metadata)); + assert(validate_cache_ids(core_op_metadata, m_cache_buffers)); + + return HAILO_SUCCESS; + } + + if (!core_op_has_caches(core_op_metadata)) { + // No cache layers found, nothing to do + return HAILO_SUCCESS; + } + + m_cache_input_size = get_cache_input_size(core_op_metadata); + m_cache_output_size = get_cache_output_size(core_op_metadata); + // TODO: cache size should be a param of the hef (via sdk) + // that way we can also immediately know if caches are used or not + // it should be a param that appears once in the hef, and is used by all caches (HRT-13584) + m_cache_size = m_cache_input_size + m_cache_output_size; + m_write_offset_bytes_delta = m_cache_input_size; + + assert(validate_cache_edge_layers(core_op_metadata, m_cache_input_size, m_cache_output_size)); + auto status = allocate_cache_buffers(core_op_metadata); + CHECK_SUCCESS(status, "Failed to allocate cache buffers"); + + m_caches_created = true; + + return HAILO_SUCCESS; + +} + +bool CacheManager::core_op_has_caches(std::shared_ptr core_op_metadata) +{ + for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) { + if (!context_metadata.get_cache_input_layers().empty() || !context_metadata.get_cache_output_layers().empty()) { + return true; + } + } + + return false; +} + +bool CacheManager::validate_cache_edge_layers(std::shared_ptr core_op_metadata, + uint32_t cache_input_size, uint32_t cache_output_size) +{ + std::unordered_map cache_id_count; + for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) { + for (const auto &layer_info : context_metadata.get_cache_input_layers()) { + cache_id_count[layer_info.cache_info.id]++; + if (cache_input_size != LayerInfoUtils::get_layer_transfer_size(layer_info)) { + return false; + } + } + + for (const auto &layer_info : context_metadata.get_cache_output_layers()) { + cache_id_count[layer_info.cache_info.id]++; + if (cache_output_size != LayerInfoUtils::get_layer_transfer_size(layer_info)) { + return false; + } + } + } + + static const uint32_t EXPECTED_CACHE_ID_COUNT = 2; // Each cache has 2 layers (input and output) + for (const auto &cache_id : cache_id_count) { + if (cache_id.second != EXPECTED_CACHE_ID_COUNT) { + return false; + } + } + + return true; +} + +uint32_t CacheManager::get_cache_input_size(std::shared_ptr core_op_metadata) +{ + // All cache layers have the same input size (this will be asserted in debug) + for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) { + for (const auto &layer_info : context_metadata.get_cache_input_layers()) { + return LayerInfoUtils::get_layer_transfer_size(layer_info); + } + } + + // No cache layers found + return 0; +} + +uint32_t CacheManager::get_cache_output_size(std::shared_ptr core_op_metadata) +{ + // All cache layers have the same output size (this will be asserted in debug) + for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) { + for (const auto &layer_info : context_metadata.get_cache_output_layers()) { + return LayerInfoUtils::get_layer_transfer_size(layer_info); + } + } + + // No cache layers found + return 0; +} + +bool CacheManager::validate_cache_ids(std::shared_ptr core_op_metadata, + const std::unordered_map ¤t_cache_buffers) +{ + std::unordered_set cache_ids; + for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) { + for (const auto &layer_info : context_metadata.get_cache_input_layers()) { + cache_ids.insert(layer_info.cache_info.id); + } + + for (const auto &layer_info : context_metadata.get_cache_output_layers()) { + cache_ids.insert(layer_info.cache_info.id); + } + } + + if (cache_ids.size() != current_cache_buffers.size()) { + return false; + } + + for (const auto &cache_id : cache_ids) { + if (std::end(current_cache_buffers) == current_cache_buffers.find(cache_id)) { + return false; + } + } + + return true; +} + +ExpectedRef CacheManager::get_cache_buffer(uint32_t cache_id) +{ + const auto cache_buffer_it = m_cache_buffers.find(cache_id); + if (std::end(m_cache_buffers) != cache_buffer_it) { + return std::ref(cache_buffer_it->second); + } + + return make_unexpected(HAILO_NOT_FOUND); +} + +hailo_status CacheManager::allocate_cache_buffers(std::shared_ptr core_op_metadata) +{ + // It's enough to go over cache_output_layers, as each cache has both input and output layers (that share the same buffer) + for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) { + for (const auto &layer_info : context_metadata.get_cache_output_layers()) { + const auto cache_id = layer_info.cache_info.id; + TRY(auto cache_buffer, CacheBuffer::create(m_driver, m_cache_size, m_cache_input_size, m_cache_output_size)); + auto emplace_res = m_cache_buffers.emplace(cache_id, std::move(cache_buffer)); + CHECK(emplace_res.second, HAILO_INTERNAL_FAILURE); + + // The cache buffer is yet to be initialized (will be initalized when it is configured with input/output channels) + m_uninitialized_caches.insert(cache_id); + } + } + + return HAILO_SUCCESS; +} + +hailo_status CacheManager::program_cache_buffers() +{ + // Set the cache to the initial configuration (program the descriptors to the initial offset) + static const auto INITIAL_CONFIGURATION_OFFSET = 0; + return update_cache_offset(INITIAL_CONFIGURATION_OFFSET); +} + +hailo_status CacheManager::try_complete_cache_initialization() +{ + // If all caches are now initialized, program their desc list and set the CacheManager as initialized + if (m_uninitialized_caches.empty() && !m_initialized) { + m_initialized = true; + + auto status = program_cache_buffers(); + CHECK_SUCCESS(status, "Failed to program cache buffers"); + } + + return HAILO_SUCCESS; +} + +ExpectedRef CacheManager::set_cache_input_channel(uint32_t cache_id, uint16_t batch_size, + vdma::ChannelId channel_id) +{ + + // TODO: Support non-1 batches? (HRT-13628) + CHECK(1 == batch_size, HAILO_INVALID_ARGUMENT, "Cache input batch size must be 1"); + TRY(auto cache_buffer, get_cache_buffer(cache_id)); + if (m_initialized) { + // Cache is already initialized, return the input channel + return cache_buffer.get().get_input(); + } + TRY(auto result, cache_buffer.get().set_input_channel(m_driver, channel_id)); + + // If the cache is now fully configured, remove it from the uninitialized set + if (cache_buffer.get().is_configured()) { + m_uninitialized_caches.erase(cache_id); + auto status = try_complete_cache_initialization(); + CHECK_SUCCESS(status); + } + + return result; +} + +ExpectedRef CacheManager::set_cache_output_channel(uint32_t cache_id, uint16_t batch_size, + vdma::ChannelId channel_id) +{ + // TODO: Support non-1 batches? (HRT-13628) + CHECK(1 == batch_size, HAILO_INVALID_ARGUMENT, "Cache output batch size must be 1"); + TRY(auto cache_buffer, get_cache_buffer(cache_id)); + if (m_initialized) { + // Cache is already initialized, return the output channel + return cache_buffer.get().get_output(); + } + TRY(auto result, cache_buffer.get().set_output_channel(m_driver, channel_id)); + + // If the cache is now fully configured, remove it from the uninitialized set + if (cache_buffer.get().is_configured()) { + m_uninitialized_caches.erase(cache_id); + auto status = try_complete_cache_initialization(); + CHECK_SUCCESS(status); + } + + return result; +} + +std::unordered_map &CacheManager::get_cache_buffers() +{ + return m_cache_buffers; +} + +hailo_status CacheManager::init_caches(uint32_t initial_read_offset_bytes, int32_t write_offset_bytes_delta) +{ + if (!m_caches_created) { + // No cache layers found, nothing to do + LOGGER__WARNING("No cache layers found, but init_cache was called"); + return HAILO_SUCCESS; + } + + CHECK(initial_read_offset_bytes < m_cache_size, HAILO_INVALID_ARGUMENT); + CHECK(write_offset_bytes_delta != 0, HAILO_INVALID_ARGUMENT); + + m_read_offset_bytes = initial_read_offset_bytes; + m_write_offset_bytes_delta = write_offset_bytes_delta; + + LOGGER__WARNING("Initializing caches [read_offset={}, write_offset_delta={}]", + m_read_offset_bytes, m_write_offset_bytes_delta); + + return program_cache_buffers(); +} + +hailo_status CacheManager::update_cache_offset(int32_t offset_delta_bytes) +{ + if (!m_caches_created) { + // No cache layers found, nothing to do + LOGGER__WARNING("No cache layers found, but update_cache_offset was called"); + return HAILO_SUCCESS; + } + + CHECK(m_initialized, HAILO_INVALID_OPERATION, "CacheManager not initialized"); + + auto status = HAILO_UNINITIALIZED; + auto new_read_offset = (m_read_offset_bytes + offset_delta_bytes) % m_cache_size; + auto new_write_offset = (m_read_offset_bytes + offset_delta_bytes + m_write_offset_bytes_delta) % m_cache_size; + + for (auto &cache_buffer : m_cache_buffers) { + TRY(auto cache_input, cache_buffer.second.get_input()); + status = cache_input.get().reprogram_descriptors(new_read_offset); + CHECK_SUCCESS(status, "Failed to reprogram read cache descriptors to offset 0x{:x} (cache_id {})", + new_read_offset, cache_buffer.first); + + TRY(auto cache_output, cache_buffer.second.get_output()); + status = cache_output.get().reprogram_descriptors(new_write_offset); + CHECK_SUCCESS(status, "Failed to reprogram write cache descriptors to offset 0x{:x} (cache_id {})", + new_write_offset, cache_buffer.first); + } + + m_read_offset_bytes = new_read_offset; + + return HAILO_SUCCESS; +} + +uint32_t CacheManager::get_cache_size() const +{ + return m_cache_size; +} + +uint32_t CacheManager::get_read_offset_bytes() const +{ + return m_read_offset_bytes; +} + +int32_t CacheManager::get_write_offset_bytes_delta() const +{ + return m_write_offset_bytes_delta; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/core_op/resource_manager/cache_manager.hpp b/hailort/libhailort/src/core_op/resource_manager/cache_manager.hpp new file mode 100644 index 0000000..1c8f9ab --- /dev/null +++ b/hailort/libhailort/src/core_op/resource_manager/cache_manager.hpp @@ -0,0 +1,88 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file cache_manager.hpp + * @brief Manges creation and configuration of cache buffers + **/ + +#ifndef _HAILO_CACHE_MANAGER_HPP_ +#define _HAILO_CACHE_MANAGER_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" +#include "core_op/resource_manager/cache_buffer.hpp" +#include "hef/core_op_metadata.hpp" + +#include +#include + +namespace hailort +{ + +class CacheManager; +using CacheManagerPtr = std::shared_ptr; +class CacheManager final +{ +public: + // TODO: Support getting initial_read_offset_bytes + write_offset_bytes_delta from configured_network_params + // s.t. the CacheManager can be created with the correct offsets, and init_caches won't be needed at the start. + // Currently, the CacheManager is created with the m_read_offset_bytes=0 and + // m_write_offset_bytes_delta=m_cache_input_size (i.e. right after where data was read from) (HRT-14288) + static Expected create_shared(HailoRTDriver &driver); + + CacheManager(HailoRTDriver &driver); + CacheManager(CacheManager &&) = default; + CacheManager(const CacheManager &) = delete; + CacheManager &operator=(CacheManager &&) = delete; + CacheManager &operator=(const CacheManager &) = delete; + ~CacheManager() = default; + + hailo_status create_caches_from_core_op(std::shared_ptr core_op_metadata); + ExpectedRef set_cache_input_channel(uint32_t cache_id, uint16_t batch_size, vdma::ChannelId channel_id); + ExpectedRef set_cache_output_channel(uint32_t cache_id, uint16_t batch_size, vdma::ChannelId channel_id); + std::unordered_map &get_cache_buffers(); + + // Note: These functions are not thread-safe! + // Programs the CacheManager instance with the given offsets, overriding the current offsets. + hailo_status init_caches(uint32_t initial_read_offset_bytes, int32_t write_offset_bytes_delta); + // Updates the read offset by the given delta + hailo_status update_cache_offset(int32_t offset_delta_bytes); + + uint32_t get_cache_size() const; + uint32_t get_read_offset_bytes() const; + int32_t get_write_offset_bytes_delta() const; + +private: + static bool core_op_has_caches(std::shared_ptr core_op_metadata); + static bool validate_cache_edge_layers(std::shared_ptr core_op_metadata, + uint32_t cache_input_size, uint32_t cache_output_size); + static uint32_t get_cache_input_size(std::shared_ptr core_op_metadata); + static uint32_t get_cache_output_size(std::shared_ptr core_op_metadata); + static bool validate_cache_ids(std::shared_ptr core_op_metadata, + const std::unordered_map ¤t_cache_buffers); + ExpectedRef get_cache_buffer(uint32_t cache_id); + hailo_status allocate_cache_buffers(std::shared_ptr core_op_metadata); + hailo_status program_cache_buffers(); + hailo_status try_complete_cache_initialization(); + + HailoRTDriver &m_driver; + bool m_caches_created; + // This class is initialized (and the member is set to true) when all caches are allocated and configured with + // input/output channels. This is done in two steps: (1) cache allocation; (2) channel configuration + // Two steps are necessary because this class allocates the buffers, however the input/output channels are assigned + // by the resource manager + bool m_initialized; + uint32_t m_cache_input_size; + uint32_t m_cache_output_size; + uint32_t m_cache_size; + uint32_t m_read_offset_bytes; + int32_t m_write_offset_bytes_delta; + std::unordered_map m_cache_buffers; + std::unordered_set m_uninitialized_caches; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_CACHE_MANAGER_HPP_ */ diff --git a/hailort/libhailort/src/core_op/resource_manager/config_buffer.cpp b/hailort/libhailort/src/core_op/resource_manager/config_buffer.cpp index ecc323d..bf2c120 100644 --- a/hailort/libhailort/src/core_op/resource_manager/config_buffer.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/config_buffer.cpp @@ -19,28 +19,27 @@ namespace hailort { Expected> ConfigBuffer::create_buffer(HailoRTDriver &driver, vdma::ChannelId channel_id, - const std::vector &cfg_sizes, const uint32_t buffer_size) + const std::vector &bursts_sizes, const uint32_t buffer_size) { auto buffer_ptr = should_use_ccb(driver) ? create_ccb_buffer(driver, buffer_size) : - create_sg_buffer(driver, channel_id, cfg_sizes); + create_sg_buffer(driver, channel_id, bursts_sizes); if (should_use_ccb(driver) && (HAILO_OUT_OF_HOST_CMA_MEMORY == buffer_ptr.status())) { /* Try to use sg buffer instead */ - return create_sg_buffer(driver, channel_id, cfg_sizes); + return create_sg_buffer(driver, channel_id, bursts_sizes); } else { return buffer_ptr; } } Expected ConfigBuffer::create(HailoRTDriver &driver, vdma::ChannelId channel_id, - const std::vector &cfg_sizes) + const std::vector &bursts_sizes) { - const auto buffer_size = std::accumulate(cfg_sizes.begin(), cfg_sizes.end(), 0); + const auto buffer_size = std::accumulate(bursts_sizes.begin(), bursts_sizes.end(), 0); CHECK_AS_EXPECTED(IS_FIT_IN_UINT32(buffer_size), HAILO_INTERNAL_FAILURE, "config buffer size exceeded UINT32 range limit"); - auto buffer_ptr = create_buffer(driver, channel_id, cfg_sizes, static_cast(buffer_size)); - CHECK_EXPECTED(buffer_ptr); + TRY(auto buffer_ptr, create_buffer(driver, channel_id, bursts_sizes, static_cast(buffer_size))); - return ConfigBuffer(buffer_ptr.release(), channel_id, buffer_size); + return ConfigBuffer(std::move(buffer_ptr), channel_id, buffer_size); } ConfigBuffer::ConfigBuffer(std::unique_ptr &&buffer, @@ -54,11 +53,10 @@ ConfigBuffer::ConfigBuffer(std::unique_ptr &&buffer, Expected ConfigBuffer::program_descriptors() { // TODO HRT-9657: remove DEVICE interrupts - auto descriptors_count = - m_buffer->program_descriptors(m_acc_buffer_offset, InterruptsDomain::DEVICE, m_acc_desc_count); - CHECK_EXPECTED(descriptors_count); + TRY(auto descriptors_count, + m_buffer->program_descriptors(m_acc_buffer_offset, InterruptsDomain::DEVICE, m_acc_desc_count)); - m_acc_desc_count += descriptors_count.value(); + m_acc_desc_count += descriptors_count; m_acc_buffer_offset = 0; return descriptors_count; @@ -136,33 +134,30 @@ hailo_status ConfigBuffer::write_inner(const MemoryView &data) } Expected> ConfigBuffer::create_sg_buffer(HailoRTDriver &driver, - vdma::ChannelId channel_id, const std::vector &cfg_sizes) + vdma::ChannelId channel_id, const std::vector &bursts_sizes) { static const auto NOT_CIRCULAR = false; // For config channels (In Hailo15), the page size must be a multiplication of host default page size. // Therefore we use the flag force_default_page_size for those types of buffers. static const auto FORCE_DEFAULT_PAGE_SIZE = true; static const auto FORCE_BATCH_SIZE = true; - auto buffer_size_requirements = vdma::BufferSizesRequirements::get_buffer_requirements_multiple_transfers( - vdma::VdmaBuffer::Type::SCATTER_GATHER, driver.desc_max_page_size(), 1, cfg_sizes, NOT_CIRCULAR, - FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE); - CHECK_EXPECTED(buffer_size_requirements); - const auto page_size = buffer_size_requirements->desc_page_size(); - const auto descs_count = buffer_size_requirements->descs_count(); - const auto buffer_size = buffer_size_requirements->buffer_size(); - - auto buffer = vdma::SgBuffer::create(driver, buffer_size, HailoRTDriver::DmaDirection::H2D); - CHECK_EXPECTED(buffer); - - auto buffer_ptr = make_shared_nothrow(buffer.release()); + TRY(const auto buffer_size_requirements, vdma::BufferSizesRequirements::get_buffer_requirements_multiple_transfers( + vdma::VdmaBuffer::Type::SCATTER_GATHER, driver.desc_max_page_size(), 1, bursts_sizes, NOT_CIRCULAR, + FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE)); + const auto page_size = buffer_size_requirements.desc_page_size(); + const auto descs_count = buffer_size_requirements.descs_count(); + const auto buffer_size = buffer_size_requirements.buffer_size(); + + TRY(auto buffer, vdma::SgBuffer::create(driver, buffer_size, HailoRTDriver::DmaDirection::H2D)); + + auto buffer_ptr = make_shared_nothrow(std::move(buffer)); CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY); static const auto DEFAULT_OFFSET = 0; - auto edge_layer = vdma::SgEdgeLayer::create(std::move(buffer_ptr), buffer_size, DEFAULT_OFFSET, driver, descs_count, - page_size, NOT_CIRCULAR, channel_id); - CHECK_EXPECTED(edge_layer); + TRY(auto edge_layer, vdma::SgEdgeLayer::create(std::move(buffer_ptr), buffer_size, DEFAULT_OFFSET, driver, descs_count, + page_size, NOT_CIRCULAR, channel_id)); - auto edge_layer_ptr = make_unique_nothrow(edge_layer.release()); + auto edge_layer_ptr = make_unique_nothrow(std::move(edge_layer)); CHECK_NOT_NULL_AS_EXPECTED(edge_layer_ptr, HAILO_OUT_OF_HOST_MEMORY); return std::unique_ptr(std::move(edge_layer_ptr)); @@ -178,30 +173,22 @@ Expected> ConfigBuffer::create_ccb_buffer(H static const auto FORCE_BATCH_SIZE = true; static const auto DEFAULT_BATCH_SIZE = 1; static const auto IS_VDMA_ALIGNED_BUFFER = true; - auto buffer_size_requirements = vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer( + TRY(const auto buffer_size_requirements, vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer( vdma::VdmaBuffer::Type::CONTINUOUS, driver.desc_max_page_size(), DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE, - buffer_size, NOT_CIRCULAR, FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER); - CHECK_EXPECTED(buffer_size_requirements); - - const auto page_size = buffer_size_requirements->desc_page_size(); - const auto descs_count = buffer_size_requirements->descs_count(); - auto buffer = vdma::ContinuousBuffer::create(buffer_size_requirements->buffer_size(), driver); - /* Don't print error here since this might be expected error that the libhailoRT can recover from - (out of host memory). If it's not the case, there is a print in hailort_driver.cpp file */ - if (HAILO_OUT_OF_HOST_CMA_MEMORY == buffer.status()) { - return make_unexpected(buffer.status()); - } else { - CHECK_EXPECTED(buffer); - } + buffer_size, NOT_CIRCULAR, FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER)); + + const auto page_size = buffer_size_requirements.desc_page_size(); + const auto descs_count = buffer_size_requirements.descs_count(); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_OUT_OF_HOST_CMA_MEMORY, auto buffer, + vdma::ContinuousBuffer::create(buffer_size_requirements.buffer_size(), driver)); - auto buffer_ptr = make_shared_nothrow(buffer.release()); + auto buffer_ptr = make_shared_nothrow(std::move(buffer)); CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY); static const auto DEFAULT_OFFSET = 0; - auto edge_layer = vdma::ContinuousEdgeLayer::create(std::move(buffer_ptr), buffer_size, DEFAULT_OFFSET, page_size, descs_count); - CHECK_EXPECTED(edge_layer); + TRY(auto edge_layer, vdma::ContinuousEdgeLayer::create(std::move(buffer_ptr), buffer_size, DEFAULT_OFFSET, page_size, descs_count)); - auto edge_layer_ptr = make_unique_nothrow(edge_layer.release()); + auto edge_layer_ptr = make_unique_nothrow(std::move(edge_layer)); CHECK_NOT_NULL_AS_EXPECTED(edge_layer_ptr, HAILO_OUT_OF_HOST_MEMORY); return std::unique_ptr(std::move(edge_layer_ptr)); @@ -209,22 +196,17 @@ Expected> ConfigBuffer::create_ccb_buffer(H bool ConfigBuffer::should_use_ccb(HailoRTDriver &driver) { - switch (driver.dma_type()) { - case HailoRTDriver::DmaType::PCIE: - return false; - case HailoRTDriver::DmaType::DRAM: - if (nullptr != std::getenv("HAILO_FORCE_CONF_CHANNEL_OVER_DESC")) { - LOGGER__WARNING("Using desc instead of CCB for config channel is not optimal for performance.\n"); - return false; - } - else { - return true; - } + if (driver.dma_type() != HailoRTDriver::DmaType::DRAM) { + return false; // not supported } - // Shouldn't reach here - assert(false); - return false; + if (nullptr != std::getenv("HAILO_FORCE_CONF_CHANNEL_OVER_DESC")) { + LOGGER__WARNING("Using desc instead of CCB for config channel is not optimal for performance.\n"); + return false; + } + else { + return true; + } } } /* hailort */ \ No newline at end of file diff --git a/hailort/libhailort/src/core_op/resource_manager/config_buffer.hpp b/hailort/libhailort/src/core_op/resource_manager/config_buffer.hpp index 4569521..5f8ab21 100644 --- a/hailort/libhailort/src/core_op/resource_manager/config_buffer.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/config_buffer.hpp @@ -26,7 +26,7 @@ class ConfigBuffer final { public: static Expected create(HailoRTDriver &driver, vdma::ChannelId channel_id, - const std::vector &cfg_sizes); + const std::vector &bursts_sizes); // Write data to config channel hailo_status write(const MemoryView &data); @@ -63,7 +63,7 @@ private: std::unique_ptr m_buffer; vdma::ChannelId m_channel_id; - const size_t m_total_buffer_size; + const size_t m_total_buffer_size; size_t m_acc_buffer_offset; uint32_t m_acc_desc_count; size_t m_current_buffer_size; diff --git a/hailort/libhailort/src/core_op/resource_manager/intermediate_buffer.cpp b/hailort/libhailort/src/core_op/resource_manager/intermediate_buffer.cpp index 77b2e80..3b2f08b 100644 --- a/hailort/libhailort/src/core_op/resource_manager/intermediate_buffer.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/intermediate_buffer.cpp @@ -18,60 +18,79 @@ namespace hailort { Expected> IntermediateBuffer::create_edge_layer( - std::shared_ptr &&buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, + std::shared_ptr buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type) { const bool is_circular = (streaming_type == StreamingType::CIRCULAR_CONTINUOS); auto buffer_exp = (vdma::VdmaBuffer::Type::CONTINUOUS == buffer->type()) ? - create_ccb_edge_layer(std::move(buffer), buffer_offset, driver, transfer_size, max_batch_size, is_circular) : - create_sg_edge_layer(std::move(buffer), buffer_offset, driver, transfer_size, max_batch_size, d2h_channel_id, is_circular); + create_ccb_edge_layer(buffer, buffer_offset, driver, transfer_size, max_batch_size, is_circular) : + create_sg_edge_layer(buffer, buffer_offset, driver, transfer_size, max_batch_size, d2h_channel_id, is_circular); return buffer_exp; } Expected IntermediateBuffer::create(HailoRTDriver &driver, uint32_t transfer_size, uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type, - std::shared_ptr &&buffer, size_t buffer_offset) + std::shared_ptr buffer, size_t buffer_offset) { - auto edge_layer_exp = create_edge_layer(std::move(buffer), buffer_offset, driver, transfer_size, max_batch_size, - d2h_channel_id, streaming_type); - CHECK_EXPECTED(edge_layer_exp); - auto edge_layer_ptr = edge_layer_exp.release(); + LOGGER__TRACE("Creating IntermediateBuffer: transfer_size = {}, max_batch_size = {}, d2h_channel_id = {}, " + "streaming_type = {}, buffer = 0x{:X}, buffer_offset = {}", + transfer_size, max_batch_size, d2h_channel_id, streaming_type, (uintptr_t)buffer.get(), buffer_offset); + + TRY(auto edge_layer_ptr, create_edge_layer(buffer, buffer_offset, driver, transfer_size, max_batch_size, + d2h_channel_id, streaming_type)); if (streaming_type == StreamingType::BURST) { // We have max_batch_size transfers, so we program them one by one. The last transfer should report interrupt // to the device. - size_t acc_offset = 0; + size_t desc_acc_offset = 0; + size_t buffer_acc_offset = 0; for (uint16_t i = 0; i < max_batch_size; i++) { const auto last_desc_interrupts_domain = ((max_batch_size - 1) == i) ? InterruptsDomain::DEVICE : InterruptsDomain::NONE; - auto desc_count_local = edge_layer_ptr->program_descriptors(transfer_size, last_desc_interrupts_domain, acc_offset); - CHECK_EXPECTED(desc_count_local, "Failed to program descs for inter context channels. Given max_batch_size is too big."); - acc_offset += desc_count_local.value(); + TRY(const auto desc_count_local, edge_layer_ptr->program_descriptors(transfer_size, + last_desc_interrupts_domain, desc_acc_offset, buffer_acc_offset), + "Failed to program descs for inter context channels. Given max_batch_size is too big."); + desc_acc_offset += desc_count_local; + buffer_acc_offset += (desc_count_local * edge_layer_ptr->desc_page_size()); } } else { // Program all descriptors, no need for interrupt. const auto interrupts_domain = InterruptsDomain::NONE; const auto total_size = edge_layer_ptr->descs_count() * edge_layer_ptr->desc_page_size(); - auto desc_count_local = edge_layer_ptr->program_descriptors(total_size, interrupts_domain, 0); - CHECK_EXPECTED(desc_count_local); + TRY(const auto desc_count_local, edge_layer_ptr->program_descriptors(total_size, interrupts_domain, 0)); + (void)desc_count_local; } - return IntermediateBuffer(std::move(edge_layer_ptr), transfer_size, max_batch_size); + return IntermediateBuffer(std::move(edge_layer_ptr), transfer_size, streaming_type, max_batch_size); +} + +Expected> IntermediateBuffer::create_shared(HailoRTDriver &driver, + uint32_t transfer_size, uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type, + std::shared_ptr buffer, size_t buffer_offset) +{ + TRY(auto intermediate_buffer, create(driver, transfer_size, max_batch_size, d2h_channel_id, streaming_type, + buffer, buffer_offset)); + + auto intermediate_buffer_ptr = make_shared_nothrow(std::move(intermediate_buffer)); + CHECK_NOT_NULL_AS_EXPECTED(intermediate_buffer_ptr, HAILO_OUT_OF_HOST_MEMORY); + return intermediate_buffer_ptr; } -Expected IntermediateBuffer::read() +Expected IntermediateBuffer::read(size_t size) { - const auto size = m_transfer_size * m_dynamic_batch_size; - assert(size <= m_edge_layer->size()); + if (size == 0) { + size = m_transfer_size * m_dynamic_batch_size; + } + CHECK_AS_EXPECTED(size <= m_edge_layer->backing_buffer_size(), HAILO_INTERNAL_FAILURE, + "Requested size {} is bigger than buffer size {}", size, m_edge_layer->backing_buffer_size()); - auto res = Buffer::create(size); - CHECK_EXPECTED(res); + TRY(auto res, Buffer::create(size)); - auto status = m_edge_layer->read(res->data(), size, 0); + auto status = m_edge_layer->read(res.data(), size, 0); CHECK_SUCCESS_AS_EXPECTED(status); - return res.release(); + return res; } CONTROL_PROTOCOL__host_buffer_info_t IntermediateBuffer::get_host_buffer_info() const @@ -79,59 +98,95 @@ CONTROL_PROTOCOL__host_buffer_info_t IntermediateBuffer::get_host_buffer_info() return m_edge_layer->get_host_buffer_info(m_transfer_size); } +hailo_status IntermediateBuffer::reprogram_descriptors(size_t buffer_offset) +{ + CHECK(m_streaming_type == StreamingType::BURST, HAILO_INTERNAL_FAILURE, + "Reprogramming descriptors is only supported for burst streaming type"); + + CHECK(buffer_offset % m_edge_layer->desc_page_size() == 0, HAILO_INTERNAL_FAILURE, + "Buffer offset must be aligned to descriptor page size"); + + assert(m_edge_layer->backing_buffer_size() >= buffer_offset); + const auto size_to_end = m_edge_layer->backing_buffer_size() - buffer_offset; + const auto first_chunk_size = std::min(size_to_end, static_cast(m_transfer_size)); + + // Program the first chunk of descriptors - from the buffer offset to the end of the buffer + static const auto BIND = true; + static const auto DESC_LIST_START = 0; + TRY(const uint32_t first_chunk_desc_count, m_edge_layer->program_descriptors(first_chunk_size, + InterruptsDomain::NONE, DESC_LIST_START, buffer_offset, BIND)); + + uint32_t second_chunk_desc_count = 0; + if (first_chunk_size < m_transfer_size) { + // Program the second chunk of descriptors - from the start of the buffer till the end of the remaining size + static const auto BUFFER_START = 0; + const auto second_chunk_size = m_transfer_size - first_chunk_size; + TRY(second_chunk_desc_count, m_edge_layer->program_descriptors(second_chunk_size, InterruptsDomain::NONE, + first_chunk_desc_count, BUFFER_START, BIND)); + } + + const auto expected_desc_count = m_edge_layer->descs_count() - 1; + CHECK(first_chunk_desc_count + second_chunk_desc_count == expected_desc_count, HAILO_INTERNAL_FAILURE, + "Expected {} descriptors, got {}", expected_desc_count, first_chunk_desc_count + second_chunk_desc_count); + + return HAILO_SUCCESS; +} + +uint32_t IntermediateBuffer::transfer_size() const +{ + return m_transfer_size; +} + IntermediateBuffer::IntermediateBuffer(std::unique_ptr &&edge_layer, uint32_t transfer_size, - uint16_t batch_size) : + StreamingType streaming_type, uint16_t batch_size) : m_edge_layer(std::move(edge_layer)), m_transfer_size(transfer_size), + m_streaming_type(streaming_type), m_dynamic_batch_size(batch_size) {} Expected> IntermediateBuffer::create_sg_edge_layer( - std::shared_ptr &&buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, + std::shared_ptr buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t batch_size, vdma::ChannelId d2h_channel_id, bool is_circular) { static const auto DONT_FORCE_DEFAULT_PAGE_SIZE = false; static const auto FORCE_BATCH_SIZE = true; static const auto IS_VDMA_ALIGNED_BUFFER = true; - auto buffer_requirements = vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer( + TRY(const auto buffer_requirements, vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer( vdma::VdmaBuffer::Type::SCATTER_GATHER, driver.desc_max_page_size(), batch_size, batch_size, transfer_size, - is_circular, DONT_FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER); - CHECK_EXPECTED(buffer_requirements); - const auto desc_page_size = buffer_requirements->desc_page_size(); - const auto descs_count = buffer_requirements->descs_count(); - const auto buffer_size = buffer_requirements->buffer_size(); + is_circular, DONT_FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER)); + const auto desc_page_size = buffer_requirements.desc_page_size(); + const auto descs_count = buffer_requirements.descs_count(); + const auto buffer_size = buffer_requirements.buffer_size(); - auto edge_layer = vdma::SgEdgeLayer::create(std::dynamic_pointer_cast(buffer), buffer_size, - buffer_offset, driver, descs_count, desc_page_size, is_circular, d2h_channel_id); - CHECK_EXPECTED(edge_layer); + TRY(auto edge_layer, vdma::SgEdgeLayer::create(std::dynamic_pointer_cast(buffer), buffer_size, + buffer_offset, driver, descs_count, desc_page_size, is_circular, d2h_channel_id)); - auto edge_layer_ptr = make_unique_nothrow(edge_layer.release()); + auto edge_layer_ptr = make_unique_nothrow(std::move(edge_layer)); CHECK_NOT_NULL_AS_EXPECTED(edge_layer_ptr, HAILO_OUT_OF_HOST_MEMORY); return std::unique_ptr(std::move(edge_layer_ptr)); } -Expected> IntermediateBuffer::create_ccb_edge_layer(std::shared_ptr &&buffer, +Expected> IntermediateBuffer::create_ccb_edge_layer(std::shared_ptr buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t batch_size, bool is_circular) { static const auto DONT_FORCE_DEFAULT_PAGE_SIZE = false; static const auto FORCE_BATCH_SIZE = true; static const auto IS_VDMA_ALIGNED_BUFFER = true; - auto buffer_size_requirements = vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer( + TRY(const auto buffer_size_requirements, vdma::BufferSizesRequirements::get_buffer_requirements_single_transfer( vdma::VdmaBuffer::Type::CONTINUOUS, driver.desc_max_page_size(), batch_size, batch_size, transfer_size, - is_circular, DONT_FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER); - CHECK_EXPECTED(buffer_size_requirements); + is_circular, DONT_FORCE_DEFAULT_PAGE_SIZE, FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER)); - const auto page_size = buffer_size_requirements->desc_page_size(); - const auto descs_count = buffer_size_requirements->descs_count(); - const auto buffer_size = buffer_size_requirements->buffer_size(); + const auto page_size = buffer_size_requirements.desc_page_size(); + const auto descs_count = buffer_size_requirements.descs_count(); + const auto buffer_size = buffer_size_requirements.buffer_size(); - auto edge_layer = vdma::ContinuousEdgeLayer::create(std::dynamic_pointer_cast(buffer), - buffer_size, buffer_offset, page_size, descs_count); - CHECK_EXPECTED(edge_layer); + TRY(auto edge_layer, vdma::ContinuousEdgeLayer::create(std::dynamic_pointer_cast(buffer), + buffer_size, buffer_offset, page_size, descs_count)); - auto edge_layer_ptr = make_unique_nothrow(edge_layer.release()); + auto edge_layer_ptr = make_unique_nothrow(std::move(edge_layer)); CHECK_NOT_NULL_AS_EXPECTED(edge_layer_ptr, HAILO_OUT_OF_HOST_MEMORY); return std::unique_ptr(std::move(edge_layer_ptr)); diff --git a/hailort/libhailort/src/core_op/resource_manager/intermediate_buffer.hpp b/hailort/libhailort/src/core_op/resource_manager/intermediate_buffer.hpp index 8661caf..f2394cf 100644 --- a/hailort/libhailort/src/core_op/resource_manager/intermediate_buffer.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/intermediate_buffer.hpp @@ -35,25 +35,33 @@ public: static Expected create(HailoRTDriver &driver, uint32_t transfer_size, uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type, - std::shared_ptr &&buffer, size_t buffer_offset); + std::shared_ptr buffer, size_t buffer_offset); + static Expected> create_shared(HailoRTDriver &driver, uint32_t transfer_size, + uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type, + std::shared_ptr buffer, size_t buffer_offset); - Expected read(); + // If size is 0, the entire buffer is read (based on the transfer size passed in the create function) + Expected read(size_t size=0); CONTROL_PROTOCOL__host_buffer_info_t get_host_buffer_info() const; + hailo_status reprogram_descriptors(size_t buffer_offset); + uint32_t transfer_size() const; private: - IntermediateBuffer(std::unique_ptr &&buffer, uint32_t transfer_size, uint16_t batch_size); + IntermediateBuffer(std::unique_ptr &&buffer, uint32_t transfer_size, + StreamingType streaming_type, uint16_t batch_size); - static Expected> create_sg_edge_layer(std::shared_ptr &&buffer, + static Expected> create_sg_edge_layer(std::shared_ptr buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t batch_size, vdma::ChannelId d2h_channel_id, bool is_circular); - static Expected> create_ccb_edge_layer(std::shared_ptr &&buffer, + static Expected> create_ccb_edge_layer(std::shared_ptr buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t batch_size, bool is_circular); - static Expected> create_edge_layer(std::shared_ptr &&buffer, + static Expected> create_edge_layer(std::shared_ptr buffer, size_t buffer_offset, HailoRTDriver &driver, uint32_t transfer_size, uint16_t max_batch_size, vdma::ChannelId d2h_channel_id, StreamingType streaming_type); std::unique_ptr m_edge_layer; const uint32_t m_transfer_size; + const StreamingType m_streaming_type; uint16_t m_dynamic_batch_size; }; diff --git a/hailort/libhailort/src/core_op/resource_manager/internal_buffer_manager.cpp b/hailort/libhailort/src/core_op/resource_manager/internal_buffer_manager.cpp index 3dd26db..51a1626 100644 --- a/hailort/libhailort/src/core_op/resource_manager/internal_buffer_manager.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/internal_buffer_manager.cpp @@ -24,9 +24,6 @@ namespace hailort { -// Macros that check status. If status is HAILO_CANT_MEET_BUFFER_REQUIREMENTS, return without printing error to the prompt. -#define CHECK_EXPECTED_OUT_OF_CMA_MEMORY(type) if (HAILO_OUT_OF_HOST_CMA_MEMORY == (type).status()) {return make_unexpected(HAILO_OUT_OF_HOST_CMA_MEMORY);} CHECK_SUCCESS(type); - Expected> InternalBufferManager::create(HailoRTDriver &driver, const ConfigureNetworkParams &config_params) { @@ -169,10 +166,9 @@ hailo_status InternalBufferManager::add_config_buffer_info(const uint16_t contex Expected> InternalBufferManager::create_intermediate_sg_buffer( const size_t buffer_size) { - auto buffer = vdma::SgBuffer::create(m_driver, buffer_size, HailoRTDriver::DmaDirection::BOTH); - CHECK_EXPECTED(buffer); + TRY(auto buffer, vdma::SgBuffer::create(m_driver, buffer_size, HailoRTDriver::DmaDirection::BOTH)); - auto buffer_ptr = make_shared_nothrow(buffer.release()); + auto buffer_ptr = make_shared_nothrow(std::move(buffer)); CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY); return std::shared_ptr(std::move(buffer_ptr)); @@ -181,10 +177,10 @@ Expected> InternalBufferManager::create_interm Expected> InternalBufferManager::create_intermediate_ccb_buffer( const size_t buffer_size) { - auto buffer = vdma::ContinuousBuffer::create(buffer_size, m_driver); - CHECK_EXPECTED_OUT_OF_CMA_MEMORY(buffer); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_OUT_OF_HOST_CMA_MEMORY, auto buffer, + vdma::ContinuousBuffer::create(buffer_size, m_driver)); - auto buffer_ptr = make_shared_nothrow(buffer.release()); + auto buffer_ptr = make_shared_nothrow(std::move(buffer)); CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY); return std::shared_ptr(std::move(buffer_ptr)); @@ -300,7 +296,7 @@ hailo_status InternalBufferManager::execute_plan(InternalBufferPlanning &buffer_ for (const auto &edge_layer_offset : buffer_plan.edge_layer_offsets) { m_edge_layer_to_buffer_map.emplace( edge_layer_offset.first, - EdgeLayerToBufferMap{buffer_ptr.value(), edge_layer_offset.second}); + EdgeLayerBuffer{buffer_ptr.value(), edge_layer_offset.second}); } // Add edge layers to executed list for (const auto &edge_layer_info : buffer_plan.edge_layer_infos) { @@ -314,14 +310,24 @@ hailo_status InternalBufferManager::execute_plan(InternalBufferPlanning &buffer_ return execution_status; } -Expected InternalBufferManager::get_intermediate_buffer(const EdgeLayerKey &key) +ExpectedRef InternalBufferManager::get_layer_buffer_info(const EdgeLayerKey &key) +{ + const auto buffer_it = m_edge_layer_infos.find(key); + if (std::end(m_edge_layer_infos) == buffer_it) { + return make_unexpected(HAILO_NOT_FOUND); + } + + return ExpectedRef(buffer_it->second); +} + +Expected InternalBufferManager::get_intermediate_buffer(const EdgeLayerKey &key) { const auto buffer_it = m_edge_layer_to_buffer_map.find(key); if (std::end(m_edge_layer_to_buffer_map) == buffer_it) { return make_unexpected(HAILO_NOT_FOUND); } - return Expected(buffer_it->second); + return Expected(buffer_it->second); } } /* namespace hailort */ diff --git a/hailort/libhailort/src/core_op/resource_manager/internal_buffer_manager.hpp b/hailort/libhailort/src/core_op/resource_manager/internal_buffer_manager.hpp index 5a93200..78a7b29 100644 --- a/hailort/libhailort/src/core_op/resource_manager/internal_buffer_manager.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/internal_buffer_manager.hpp @@ -37,9 +37,12 @@ public: hailo_status add_config_buffer_info(const uint16_t context_index, const size_t config_stream_index, const std::vector &cfg_sizes); hailo_status add_layer_buffer_info(const LayerInfo &layer_info); - Expected get_intermediate_buffer(const EdgeLayerKey &key); + ExpectedRef get_layer_buffer_info(const EdgeLayerKey &key); + Expected get_intermediate_buffer(const EdgeLayerKey &key); hailo_status plan_and_execute(InternalBufferPlanner::Type default_planner_type, const size_t number_of_contexts); + private: + InternalBufferManager(HailoRTDriver &driver, const ConfigureNetworkParams &config_params); // Add buffer info phase functions void add_buffer_info(const EdgeLayerKey &edge_layer_key, const EdgeLayerInfo &buffer_info); @@ -65,10 +68,7 @@ private: const ConfigureNetworkParams &m_config_params; // m_edge_layer_infos is filled by add_buffer_info API std::map m_edge_layer_infos; - - std::map m_edge_layer_to_buffer_map; - - InternalBufferManager(HailoRTDriver &driver, const ConfigureNetworkParams &config_params); + std::map m_edge_layer_to_buffer_map; }; } /* namespace hailort */ diff --git a/hailort/libhailort/src/core_op/resource_manager/internal_buffer_planner.cpp b/hailort/libhailort/src/core_op/resource_manager/internal_buffer_planner.cpp index bf36948..51ff94e 100644 --- a/hailort/libhailort/src/core_op/resource_manager/internal_buffer_planner.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/internal_buffer_planner.cpp @@ -17,8 +17,7 @@ constexpr size_t NAIVE_PLANNING_EDGE_LAYER_OFFSET = 0; -// Macros that check status. If status is HAILO_CANT_MEET_BUFFER_REQUIREMENTS, return without printing error to the prompt. -#define CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(type) if (HAILO_CANT_MEET_BUFFER_REQUIREMENTS == type.status()) {return make_unexpected(HAILO_CANT_MEET_BUFFER_REQUIREMENTS);} CHECK_SUCCESS(type); +// Macro that check status. If status is HAILO_CANT_MEET_BUFFER_REQUIREMENTS, return without printing error to the prompt. #define CHECK_STATUS_CANT_MEET_REQUIREMENTS(status) if (HAILO_CANT_MEET_BUFFER_REQUIREMENTS == status) {return make_unexpected(status);} CHECK_SUCCESS(status); namespace hailort @@ -64,6 +63,9 @@ bool InternalBufferPlanner::should_edge_layer_use_ccb(const LayerType &layer_typ else { return true; } + case LayerType::CACHE: + // Cache layers are always sg + return false; default: // Shouldn't reach here assert(false); @@ -87,14 +89,14 @@ Expected InternalBufferPlanner::create_naive_buffer_plan edge_layer_offsets.emplace_back(edge_layer_info.first, NAIVE_PLANNING_EDGE_LAYER_OFFSET); vdma::VdmaBuffer::Type buffer_type = should_edge_layer_use_ccb(edge_layer_info.second.type, dma_type, force_sg_buffer_type) ? vdma::VdmaBuffer::Type::CONTINUOUS : vdma::VdmaBuffer::Type::SCATTER_GATHER; - const auto buffer_requirements = return_buffer_requirements(edge_layer_info.second, buffer_type, max_page_size); - CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(buffer_requirements); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, const auto buffer_requirements, + return_buffer_requirements(edge_layer_info.second, buffer_type, max_page_size)); buffer_planning.emplace_back( BufferPlan{ buffer_type, - buffer_requirements->buffer_size(), - buffer_requirements->buffer_size(), + buffer_requirements.buffer_size(), + buffer_requirements.buffer_size(), edge_layer_offsets, plan_edge_layer_infos}); } @@ -210,16 +212,16 @@ hailo_status InternalBufferPlanner::add_edge_layer_to_planning( std::vector> &context_buffer_usage_vector, BufferPlan &buffer_plan, const vdma::VdmaBuffer::Type buffer_type, uint16_t max_page_size) { - const auto buffer_requirements = return_buffer_requirements(edge_layer.second, buffer_type, max_page_size); - CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(buffer_requirements); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, const auto buffer_requirements, + return_buffer_requirements(edge_layer.second, buffer_type, max_page_size)); // Check if there is enough space in the current context buffer. const auto start_context = edge_layer.second.start_context; const auto end_context = edge_layer.second.end_context; const auto buffer_map = build_availibility_map(context_buffer_usage_vector, start_context, end_context); - const auto edge_layer_size = buffer_requirements->buffer_size(); - const auto buffer_offset_alignment = buffer_requirements->desc_page_size(); + const auto edge_layer_size = buffer_requirements.buffer_size(); + const auto buffer_offset_alignment = buffer_requirements.desc_page_size(); const auto buffer_offset = find_new_buffer_offset(buffer_map, edge_layer_size, buffer_offset_alignment); auto end_of_edge_layer_offset = buffer_offset + edge_layer_size; @@ -288,17 +290,15 @@ Expected InternalBufferPlanner::create_optimized_buffer_ InternalBufferPlanning buffer_planning; // Second - create buffer planning for each buffer type if (!ccb_edge_layers.empty()) { - auto ccb_buffer_planning = - create_single_buffer_planning(ccb_edge_layers, number_of_contexts, vdma::VdmaBuffer::Type::CONTINUOUS, max_page_size); - CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(ccb_buffer_planning); - buffer_planning.insert(buffer_planning.end(), ccb_buffer_planning->begin(), ccb_buffer_planning->end()); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, const auto ccb_buffer_planning, + create_single_buffer_planning(ccb_edge_layers, number_of_contexts, vdma::VdmaBuffer::Type::CONTINUOUS, max_page_size)); + buffer_planning.insert(buffer_planning.end(), ccb_buffer_planning.begin(), ccb_buffer_planning.end()); } if (!sg_edge_layers.empty()) { - auto sg_buffer_planning = - create_single_buffer_planning(sg_edge_layers, number_of_contexts, vdma::VdmaBuffer::Type::SCATTER_GATHER, max_page_size); - CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(sg_buffer_planning); - buffer_planning.insert(buffer_planning.end(), sg_buffer_planning->begin(), sg_buffer_planning->end()); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, auto sg_buffer_planning, + create_single_buffer_planning(sg_edge_layers, number_of_contexts, vdma::VdmaBuffer::Type::SCATTER_GATHER, max_page_size)); + buffer_planning.insert(buffer_planning.end(), sg_buffer_planning.begin(), sg_buffer_planning.end()); } return buffer_planning; @@ -368,14 +368,14 @@ hailo_status InternalBufferPlanner::change_edge_layer_buffer_offset(InternalBuff { TRY(auto edge_layer_info, get_edge_info_from_buffer_plan(buffer_planning, edge_layer_key)); for (auto &buffer_plan : buffer_planning) { - const auto buffer_requirements = return_buffer_requirements(edge_layer_info, buffer_plan.buffer_type, max_page_size); - CHECK_EXPECTED_CANT_MEET_REQUIREMENTS(buffer_requirements); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_CANT_MEET_BUFFER_REQUIREMENTS, const auto buffer_requirements, + return_buffer_requirements(edge_layer_info, buffer_plan.buffer_type, max_page_size)); for (auto &edge_layer_offset : buffer_plan.edge_layer_offsets) { if (edge_layer_offset.first == edge_layer_key) { edge_layer_offset.second = new_offset; - if (edge_layer_offset.second + buffer_requirements->buffer_size() > buffer_plan.buffer_size) { - buffer_plan.buffer_size = edge_layer_offset.second + buffer_requirements->buffer_size(); + if (edge_layer_offset.second + buffer_requirements.buffer_size() > buffer_plan.buffer_size) { + buffer_plan.buffer_size = edge_layer_offset.second + buffer_requirements.buffer_size(); } return HAILO_SUCCESS; } diff --git a/hailort/libhailort/src/core_op/resource_manager/internal_buffer_planner.hpp b/hailort/libhailort/src/core_op/resource_manager/internal_buffer_planner.hpp index 2e1a650..986f2d7 100644 --- a/hailort/libhailort/src/core_op/resource_manager/internal_buffer_planner.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/internal_buffer_planner.hpp @@ -35,7 +35,7 @@ struct EdgeLayerInfo { bool reuse_buffer; }; -struct EdgeLayerToBufferMap { +struct EdgeLayerBuffer { std::shared_ptr buffer; size_t offset; }; diff --git a/hailort/libhailort/src/core_op/resource_manager/periph_calculator.cpp b/hailort/libhailort/src/core_op/resource_manager/periph_calculator.cpp index b35878b..55aa4cf 100644 --- a/hailort/libhailort/src/core_op/resource_manager/periph_calculator.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/periph_calculator.cpp @@ -92,13 +92,12 @@ Expected PeriphCalculator::calculate_periph_registers_impl(const Laye // confgured periph registers will add / removed the extra padding if (is_core_hw_padding_config_in_dfc) { if (0 != (periph_frame_size % PERIPH_FRAME_ALIGNMENT)) { - auto max_periph_padding_payload = HefConfigurator::max_periph_padding_payload_value( - DeviceBase::hef_arch_to_device_arch(hw_arch)); - CHECK_EXPECTED(max_periph_padding_payload); + TRY(const auto max_periph_padding_payload, HefConfigurator::max_periph_padding_payload_value( + DeviceBase::hef_arch_to_device_arch(hw_arch))); // Currently case of payload larger than max periph padding payload value - not supported - CHECK_AS_EXPECTED(max_periph_padding_payload.value() > periph_frame_size, HAILO_INVALID_HEF, - "Error, padded frame size larger than {} Currently not supported", max_periph_padding_payload.value()); + CHECK_AS_EXPECTED(max_periph_padding_payload > periph_frame_size, HAILO_INVALID_HEF, + "Error, padded frame size larger than {} Currently not supported", max_periph_padding_payload); const auto padded_periph_frame_size = HailoRTCommon::align_to(periph_frame_size, static_cast(PERIPH_FRAME_ALIGNMENT)); @@ -138,9 +137,8 @@ Expected PeriphCalculator::calculate_periph_registers(const LayerInfo const uint32_t desc_page_size, const bool is_periph_calculated_in_hailort, const HEFHwArch &hw_arch, const bool is_core_hw_padding_config_in_dfc) { - auto max_periph_bytes_from_hef = HefConfigurator::max_periph_bytes_value(DeviceBase::hef_arch_to_device_arch(hw_arch)); - CHECK_EXPECTED(max_periph_bytes_from_hef); - const auto max_periph_bytes = std::min(max_periph_bytes_from_hef.value(), layer_info.max_shmifo_size); + TRY(const auto max_periph_bytes_from_hef, HefConfigurator::max_periph_bytes_value(DeviceBase::hef_arch_to_device_arch(hw_arch))); + const auto max_periph_bytes = std::min(max_periph_bytes_from_hef, layer_info.max_shmifo_size); // If extension for calculating periph values in hailort is false and core hw padding is not supported - copy values from // Core registers, otherwise calculate them according to shape and other layer information const bool hw_padding_supported = HefConfigurator::is_core_hw_padding_supported(layer_info, max_periph_bytes, diff --git a/hailort/libhailort/src/core_op/resource_manager/resource_manager.cpp b/hailort/libhailort/src/core_op/resource_manager/resource_manager.cpp index f34e48c..c36edaa 100644 --- a/hailort/libhailort/src/core_op/resource_manager/resource_manager.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/resource_manager.cpp @@ -27,12 +27,12 @@ Expected ContextResources::create(HailoRTDriver &driver, std::vector config_buffers; config_buffers.reserve(config_buffer_infos.size()); for (uint8_t config_stream_index = 0; config_stream_index < config_buffer_infos.size(); config_stream_index++) { - auto buffer_resource = ConfigBuffer::create(driver, config_channels_ids[config_stream_index], - config_buffer_infos.at(config_stream_index)); - CHECK_EXPECTED(buffer_resource); - config_buffers.emplace_back(buffer_resource.release()); + TRY(auto buffer_resource, ConfigBuffer::create(driver, config_channels_ids[config_stream_index], + config_buffer_infos.at(config_stream_index).bursts_sizes)); + config_buffers.emplace_back(std::move(buffer_resource)); - internal_buffer_manager->add_config_buffer_info(context_index, config_stream_index, config_buffer_infos.at(config_stream_index)); + internal_buffer_manager->add_config_buffer_info(context_index, config_stream_index, + config_buffer_infos.at(config_stream_index).bursts_sizes); } return ContextResources(driver, context_type, std::move(config_buffers), internal_buffer_manager); @@ -209,13 +209,10 @@ static Expected create_latency_meters_from_config_params( // Best effort for starting latency meter. auto networks_names = core_op_metadata->get_network_names(); for (auto &network_name : networks_names) { - auto layer_infos = core_op_metadata->get_all_layer_infos(network_name); - CHECK_EXPECTED(layer_infos); - auto latency_meter = create_hw_latency_meter(layer_infos.value()); - if (latency_meter) { - latency_meters_map.emplace(network_name, latency_meter.release()); - LOGGER__DEBUG("Starting hw latency measurement for network {}", network_name); - } + TRY(const auto layer_infos, core_op_metadata->get_all_layer_infos(network_name)); + TRY(auto latency_meter, create_hw_latency_meter(layer_infos)); + latency_meters_map.emplace(network_name, latency_meter); + LOGGER__DEBUG("Starting hw latency measurement for network {}", network_name); } } @@ -223,8 +220,8 @@ static Expected create_latency_meters_from_config_params( } Expected ResourcesManager::create(VdmaDevice &vdma_device, HailoRTDriver &driver, - const ConfigureNetworkParams &config_params, std::shared_ptr core_op_metadata, - uint8_t core_op_index) + const ConfigureNetworkParams &config_params, CacheManagerPtr cache_manager, + std::shared_ptr core_op_metadata, uint8_t core_op_index) { // Allocate config channels. In order to use the same channel ids for config channels in all contexts, // we allocate all of them here, and use in preliminary/dynamic context. @@ -235,32 +232,27 @@ Expected ResourcesManager::create(VdmaDevice &vdma_device, Hai for (uint8_t cfg_index = 0; cfg_index < config_channels_info.size(); cfg_index++) { const auto layer_identifier = std::make_tuple(LayerType::CFG, HAILO_H2D_STREAM, "", cfg_index); const auto engine_index = config_channels_info[cfg_index].engine_index; - auto channel_id = allocator.get_available_channel_id(layer_identifier, HailoRTDriver::DmaDirection::H2D, engine_index); - CHECK_EXPECTED(channel_id); - config_channels_ids.push_back(channel_id.release()); + TRY(const auto channel_id, + allocator.get_available_channel_id(layer_identifier, HailoRTDriver::DmaDirection::H2D, engine_index)); + config_channels_ids.push_back(channel_id); } - auto internal_buffer_manager = InternalBufferManager::create(driver, config_params); - CHECK_EXPECTED(internal_buffer_manager); - - auto action_list_buffer_builder = create_action_list_buffer_builder(core_op_metadata->dynamic_contexts().size(), - driver); - CHECK_EXPECTED(action_list_buffer_builder); - + TRY(auto internal_buffer_manager, InternalBufferManager::create(driver, config_params)); + TRY_V(auto action_list_buffer_builder, create_action_list_buffer_builder(core_op_metadata->dynamic_contexts().size(), + vdma_device)); + TRY(auto latency_meters, create_latency_meters_from_config_params(config_params, core_op_metadata)); auto network_index_map = core_op_metadata->get_network_names(); - auto latency_meters = create_latency_meters_from_config_params(config_params, core_op_metadata); - CHECK_EXPECTED(latency_meters); - ResourcesManager resources_manager(vdma_device, driver, std::move(allocator), config_params, - std::move(core_op_metadata), core_op_index, std::move(network_index_map), latency_meters.release(), - std::move(config_channels_ids), internal_buffer_manager.release(), action_list_buffer_builder.release()); + ResourcesManager resources_manager(vdma_device, driver, std::move(allocator), config_params, cache_manager, + std::move(core_op_metadata), core_op_index, std::move(network_index_map), std::move(latency_meters), + std::move(config_channels_ids), internal_buffer_manager, std::move(action_list_buffer_builder)); return resources_manager; } ResourcesManager::ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &driver, ChannelAllocator &&channel_allocator, const ConfigureNetworkParams config_params, - std::shared_ptr &&core_op_metadata, + CacheManagerPtr cache_manager, std::shared_ptr &&core_op_metadata, uint8_t core_op_index, const std::vector &&network_index_map, LatencyMetersMap &&latency_meters, std::vector &&config_channels_ids, @@ -271,6 +263,7 @@ ResourcesManager::ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &drive m_vdma_device(vdma_device), m_driver(driver), m_config_params(config_params), + m_cache_manager(cache_manager), m_intermediate_buffers(), m_core_op_metadata(std::move(core_op_metadata)), m_core_op_index(core_op_index), @@ -278,7 +271,6 @@ ResourcesManager::ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &drive m_total_context_count(0), m_network_index_map(std::move(network_index_map)), m_latency_meters(std::move(latency_meters)), - m_boundary_channels(), m_is_configured(false), m_is_activated(false), m_config_channels_ids(std::move(config_channels_ids)), @@ -293,6 +285,7 @@ ResourcesManager::ResourcesManager(ResourcesManager &&other) noexcept : m_vdma_device(other.m_vdma_device), m_driver(other.m_driver), m_config_params(other.m_config_params), + m_cache_manager(std::move(other.m_cache_manager)), m_intermediate_buffers(std::move(other.m_intermediate_buffers)), m_core_op_metadata(std::move(other.m_core_op_metadata)), m_core_op_index(other.m_core_op_index), @@ -340,9 +333,8 @@ hailo_status ResourcesManager::fill_network_batch_size(CONTROL_PROTOCOL__applica for (network_index = 0; network_index < m_network_index_map.size(); network_index++) { auto const network_name_from_map = m_network_index_map[network_index]; if (network_name_from_map == network_name_from_params) { - auto batch_size = get_network_batch_size(network_name_from_params); - CHECK_EXPECTED_AS_STATUS(batch_size); - app_header.batch_size[network_index] = batch_size.value(); + TRY(const auto batch_size, get_network_batch_size(network_name_from_params)); + app_header.batch_size[network_index] = batch_size; break; } } @@ -362,43 +354,6 @@ hailo_status ResourcesManager::fill_csm_buffer_size(CONTROL_PROTOCOL__applicatio return HAILO_SUCCESS; } -void ResourcesManager::process_interrupts(IrqData &&irq_data) -{ - assert(irq_data.channels_count <= ARRAY_ENTRIES(irq_data.channels_irq_data)); - for (uint8_t irq_index = 0; irq_index < irq_data.channels_count; irq_index++) { - const auto &channel_irq_data = irq_data.channels_irq_data[irq_index]; - auto boundary_channel = m_boundary_channels.find(channel_irq_data.channel_id); - if (std::end(m_boundary_channels) == boundary_channel) { - LOGGER__ERROR("Got interrupt for channel {}, but there is no such boundary channel", channel_irq_data.channel_id); - continue; - } - - if ((channel_irq_data.host_error != 0) || (channel_irq_data.device_error != 0)) { - LOGGER__CRITICAL("Got error on channel {} host_error=0x{:x} device_error=0x{:x}", - channel_irq_data.channel_id, channel_irq_data.host_error, channel_irq_data.device_error); - continue; - } - - if (!channel_irq_data.validation_success) { - LOGGER__CRITICAL("Got validation error on channel {}", channel_irq_data.channel_id); - continue; - } - - if (!channel_irq_data.is_active) { - LOGGER__CRITICAL("Channel {} was aborted by external source", channel_irq_data.channel_id); - continue; - } - - auto status = boundary_channel->second->trigger_channel_completion(channel_irq_data.desc_num_processed); - if ((status != HAILO_SUCCESS) && - (status != HAILO_STREAM_ABORT) && - (status != HAILO_STREAM_NOT_ACTIVATED)) { - // Log error and continue gracefully to process other interrupts - LOGGER__ERROR("Trigger channel completion failed on channel {} with status {}", channel_irq_data.channel_id, status); - } - } -} - Expected ResourcesManager::get_batch_size() const { uint16_t batch_size = UINT16_MAX; @@ -416,39 +371,65 @@ Expected ResourcesManager::get_batch_size() const } +std::pair ResourcesManager::calculate_transfer_queue_sizes(const vdma::DescriptorList &desc_list, + uint32_t transfer_size, uint32_t max_active_trans, bool use_latency_meter) +{ + // Calculate m_ongoing_transfers capacity - transfers that are already bound to the descriptor list + // Add desc for boundary channel because might need extra for non aligned async API + // We don't use get_max_aligned_transfers_in_desc_list because we want to include the option of a bounce buffer + static const auto INCLUDE_BOUNCE_BUFFER = true; + const size_t max_transfers_in_desc_list = desc_list.max_transfers(transfer_size, INCLUDE_BOUNCE_BUFFER); + + // Max capacity due to driver constraints (see HAILO_VDMA_MAX_ONGOING_TRANSFERS) + const size_t max_ongoing_transfers_capacity = (use_latency_meter ? + (ONGOING_TRANSFERS_SIZE / 2) : ONGOING_TRANSFERS_SIZE) - 1; + + const auto ongoing_transfers = std::min(max_transfers_in_desc_list, max_ongoing_transfers_capacity); + + // We want to allow max_active_trans transfers in m_pending_transfers + // * If the transfers can all fit in m_ongoing_transfers, we don't need to use m_pending_transfers so we set it + // to 0. In this case, all transfers will be handled via m_ongoing_transfers and each time launch_transfer is + // called, the transfer will be launched immediately. + // * Otherwise, we set it to max_active_trans. In this case, we will use m_pending_transfers to queue up + // transfers that can't fit in m_ongoing_transfers. We will then launch them as soon as there is room in + // m_ongoing_transfers, via the transfer launcher. + // TODO: Bring back commented out impl bellow (HRT-13644) + // Setting pending_transfers to zero, s.t. the pending transfer queue won't be used. + (void)max_active_trans; + const auto pending_transfers = 0; + // const auto pending_transfers = (max_active_trans > ongoing_transfers) ? max_active_trans : 0; + + return std::make_pair(ongoing_transfers, pending_transfers); +} + hailo_status ResourcesManager::create_boundary_vdma_channel(const LayerInfo &layer_info) { // TODO: put in layer info const auto channel_direction = layer_info.direction == HAILO_H2D_STREAM ? HailoRTDriver::DmaDirection::H2D : HailoRTDriver::DmaDirection::D2H; - const auto channel_id = get_available_channel_id(to_layer_identifier(layer_info), - channel_direction, layer_info.dma_engine_index); - CHECK_EXPECTED_AS_STATUS(channel_id); - - const auto network_batch_size = get_network_batch_size(layer_info.network_name); - CHECK_EXPECTED_AS_STATUS(network_batch_size); + TRY(const auto channel_id, get_available_channel_id(to_layer_identifier(layer_info), + channel_direction, layer_info.dma_engine_index)); + TRY(const auto network_batch_size, get_network_batch_size(layer_info.network_name)); const auto nms_max_detections_per_frame = - layer_info.nms_info.number_of_classes * layer_info.nms_info.max_bboxes_per_class * layer_info.nms_info.chunks_per_frame; + layer_info.nms_info.number_of_classes * layer_info.nms_info.max_bboxes_per_class * layer_info.nms_info.chunks_per_frame; const auto max_active_transfers_scale = (layer_info.format.order == HAILO_FORMAT_ORDER_HAILO_NMS) ? (nms_max_detections_per_frame * MAX_ACTIVE_TRANSFERS_SCALE) : MAX_ACTIVE_TRANSFERS_SCALE; - auto device_arch = m_vdma_device.get_architecture(); - CHECK_EXPECTED_AS_STATUS(device_arch); + TRY(const auto device_arch, m_vdma_device.get_architecture()); /* Add error in configure phase for invalid NMS parameters */ - if (layer_info.format.order == HAILO_FORMAT_ORDER_HAILO_NMS && - (device_arch.value() == HAILO_ARCH_HAILO15H || device_arch.value() == HAILO_ARCH_HAILO15M || device_arch.value() == HAILO_ARCH_PLUTO)) { - CHECK(layer_info.nms_info.number_of_classes * layer_info.nms_info.chunks_per_frame * network_batch_size.value() < HAILO15H_NMS_MAX_CLASSES, - HAILO_INVALID_ARGUMENT, "Invalid NMS parameters. Number of classes ({}) * division factor ({}) * batch size ({}) must be under {}", - layer_info.nms_info.number_of_classes, layer_info.nms_info.chunks_per_frame, network_batch_size.value(), HAILO15H_NMS_MAX_CLASSES); + if ((layer_info.format.order == HAILO_FORMAT_ORDER_HAILO_NMS) && (HailoRTCommon::is_hailo1x_device_type(device_arch))) { + CHECK(layer_info.nms_info.number_of_classes * layer_info.nms_info.chunks_per_frame * network_batch_size < HAILO15H_NMS_MAX_CLASSES, + HAILO_INVALID_ARGUMENT, "Invalid NMS parameters. Number of classes ({}) * division factor ({}) * batch size ({}) must be under {}", + layer_info.nms_info.number_of_classes, layer_info.nms_info.chunks_per_frame, network_batch_size, HAILO15H_NMS_MAX_CLASSES); } - const auto min_active_trans = MIN_ACTIVE_TRANSFERS_SCALE * network_batch_size.value(); + const auto min_active_trans = MIN_ACTIVE_TRANSFERS_SCALE * network_batch_size; const auto max_active_trans = (layer_info.format.order == HAILO_FORMAT_ORDER_HAILO_NMS) ? /* NMS Case - Value be be higher than UINT16_MAX. in this case we only limit to UART16_MAX with no error */ - std::min(static_cast(UINT16_MAX), max_active_transfers_scale * network_batch_size.value()) : - max_active_transfers_scale * network_batch_size.value(); + std::min(static_cast(UINT16_MAX), max_active_transfers_scale * network_batch_size) : + max_active_transfers_scale * network_batch_size; CHECK(IS_FIT_IN_UINT16(min_active_trans), HAILO_INVALID_ARGUMENT, "calculated min_active_trans for vdma descriptor list is out of UINT16 range"); @@ -475,42 +456,37 @@ hailo_status ResourcesManager::create_boundary_vdma_channel(const LayerInfo &lay DONT_FORCE_BATCH_SIZE, IS_VDMA_ALIGNED_BUFFER); if (HAILO_CANT_MEET_BUFFER_REQUIREMENTS == buffer_sizes_requirements.status()) { LOGGER__ERROR("Network shapes and batch size exceeds driver descriptors capabilities. " - "(A common cause for this error could be the batch size - which is {}).", network_batch_size.value()); + "(A common cause for this error could be the batch size - which is {}).", network_batch_size); } - CHECK_EXPECTED_AS_STATUS(buffer_sizes_requirements); + CHECK_EXPECTED_AS_STATUS(buffer_sizes_requirements); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here const auto page_size = buffer_sizes_requirements->desc_page_size(); const auto descs_count = (nullptr != std::getenv("HAILO_CONFIGURE_FOR_HW_INFER")) ? MAX_SG_DESCS_COUNT : buffer_sizes_requirements->descs_count(); - auto channel = vdma::BoundaryChannel::create(channel_id.value(), channel_direction, m_driver, descs_count, - page_size, layer_info.name, latency_meter); - CHECK_EXPECTED_AS_STATUS(channel); + const bool CIRCULAR = true; + TRY(auto desc_list, vdma::DescriptorList::create(descs_count, page_size, CIRCULAR, m_driver)); - m_boundary_channels.emplace(channel_id.value(), channel.release()); + size_t pending_transfers = 0, ongoing_transfers = 0; + std::tie(ongoing_transfers, pending_transfers) = calculate_transfer_queue_sizes(desc_list, transfer_size, + max_active_trans, (latency_meter != nullptr)); + + TRY(auto vdma_transfer_launcher, m_vdma_device.get_vdma_transfer_launcher()); + TRY(auto channel, vdma::BoundaryChannel::create(m_driver, channel_id, channel_direction, std::move(desc_list), + vdma_transfer_launcher.get(), ongoing_transfers, pending_transfers, layer_info.name, latency_meter)); + + m_boundary_channels.add_channel(std::move(channel)); return HAILO_SUCCESS; } Expected ResourcesManager::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) { - for (const auto &boundary_channel : m_boundary_channels) { - if (boundary_channel.second->stream_name() == stream_name) { - return vdma::BoundaryChannelPtr(boundary_channel.second); - } - } - - return make_unexpected(HAILO_NOT_FOUND); + return m_boundary_channels.get_by_name(stream_name); } Expected> ResourcesManager::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) const { - for (const auto &boundary_channel : m_boundary_channels) { - if (boundary_channel.second->stream_name() == stream_name) { - return std::shared_ptr(boundary_channel.second); - } - } - - return make_unexpected(HAILO_NOT_FOUND); + return const_cast(m_boundary_channels).get_by_name(stream_name); } hailo_power_mode_t ResourcesManager::get_power_mode() const @@ -525,12 +501,11 @@ ExpectedRef ResourcesManager::create_intermediate_buffer( auto edge_layer_key = std::make_pair(src_context_index, src_stream_index); TRY(auto buffer_info, m_internal_buffer_manager->get_intermediate_buffer(edge_layer_key)); - auto intermediate_buffer = IntermediateBuffer::create(m_driver, transfer_size, batch_size, d2h_channel_id, - streaming_type, std::move(buffer_info.buffer), buffer_info.offset); - CHECK_EXPECTED(intermediate_buffer); + TRY(auto intermediate_buffer, IntermediateBuffer::create(m_driver, transfer_size, batch_size, d2h_channel_id, + streaming_type, buffer_info.buffer, buffer_info.offset)); const auto key = std::make_pair(src_context_index, src_stream_index); - auto emplace_res = m_intermediate_buffers.emplace(key, intermediate_buffer.release()); + auto emplace_res = m_intermediate_buffers.emplace(key, std::move(intermediate_buffer)); return std::ref(emplace_res.first->second); } @@ -544,6 +519,23 @@ ExpectedRef ResourcesManager::get_intermediate_buffer(const return std::ref(buffer_it->second); } +ExpectedRef ResourcesManager::set_cache_input_channel(uint32_t cache_id, uint16_t batch_size, + vdma::ChannelId channel_id) +{ + return m_cache_manager->set_cache_input_channel(cache_id, batch_size, channel_id); +} + +ExpectedRef ResourcesManager::set_cache_output_channel(uint32_t cache_id, uint16_t batch_size, + vdma::ChannelId channel_id) +{ + return m_cache_manager->set_cache_output_channel(cache_id, batch_size, channel_id); +} + +std::unordered_map &ResourcesManager::get_cache_buffers() +{ + return m_cache_manager->get_cache_buffers(); +} + Expected ResourcesManager::get_control_core_op_header() { CONTROL_PROTOCOL__application_header_t app_header{}; @@ -572,11 +564,9 @@ Expected> ResourcesManager::add_new_con { CHECK_AS_EXPECTED(m_total_context_count < std::numeric_limits::max(), HAILO_INVALID_CONTEXT_COUNT); - auto context_resources = ContextResources::create(m_driver, context_type, context_index, - m_config_channels_ids, config_info, m_internal_buffer_manager); - CHECK_EXPECTED(context_resources); - - m_contexts_resources.emplace_back(context_resources.release()); + TRY(auto context_resources, ContextResources::create(m_driver, context_type, context_index, + m_config_channels_ids, config_info, m_internal_buffer_manager)); + m_contexts_resources.emplace_back(std::move(context_resources)); m_total_context_count++; if (CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC == context_type) { m_dynamic_context_count++; @@ -633,15 +623,33 @@ Expected ResourcesManager::read_intermediate_buffer(const IntermediateBu return intermediate_buffer_it->second.read(); } +Expected ResourcesManager::read_cache_buffer(uint32_t cache_id) +{ + auto &cache_buffers_map = m_cache_manager->get_cache_buffers(); + auto cache_buffer_it = cache_buffers_map.find(cache_id); + CHECK_AS_EXPECTED(std::end(cache_buffers_map) != cache_buffer_it, HAILO_NOT_FOUND, + "Failed to find cache buffer for cache_id {}", cache_id); + return cache_buffer_it->second.read_entire_cache(); +} + +Expected> ResourcesManager::read_cache_buffers() +{ + std::map result; + for (auto &cache_buffer : m_cache_manager->get_cache_buffers()) { + TRY(auto buffer, cache_buffer.second.read_entire_cache()); + result.emplace(cache_buffer.first, std::move(buffer)); + } + + return result; +} + hailo_status ResourcesManager::configure() { CHECK(!m_is_configured, HAILO_INTERNAL_FAILURE, "Can't configure the same core-op twice"); m_is_configured = true; - auto core_op_header = get_control_core_op_header(); - CHECK_EXPECTED_AS_STATUS(core_op_header); - - auto status = Control::context_switch_set_network_group_header(m_vdma_device, core_op_header.release()); + TRY(const auto core_op_header, get_control_core_op_header()); + auto status = Control::context_switch_set_network_group_header(m_vdma_device, core_op_header); CHECK_SUCCESS(status); // Only send controls to FW in case of control action list builder @@ -689,19 +697,8 @@ hailo_status ResourcesManager::start_vdma_interrupts_dispatcher() { CHECK(get_is_activated(), HAILO_INTERNAL_FAILURE, "Cannot call start_vdma_interrupts_dispatcher when core-op already deactivated"); - auto interrupts_dispatcher = m_vdma_device.get_vdma_interrupts_dispatcher(); - CHECK_EXPECTED_AS_STATUS(interrupts_dispatcher); - - ChannelsBitmap channels_bitmap{}; - for (const auto &boundary_channel : m_boundary_channels) { - const auto channel_id = boundary_channel.first; - channels_bitmap[channel_id.engine_index] |= (1 << channel_id.channel_index); - } - - const bool enable_timestamp_measure = !m_latency_meters.empty(); - return interrupts_dispatcher->get().start(channels_bitmap, enable_timestamp_measure, [this](IrqData &&irq_data){ - process_interrupts(std::move(irq_data)); - }); + TRY(auto interrupts_dispatcher, m_vdma_device.get_vdma_interrupts_dispatcher()); + return interrupts_dispatcher.get().start(m_boundary_channels); } hailo_status ResourcesManager::stop_vdma_interrupts_dispatcher() @@ -733,7 +730,8 @@ hailo_status ResourcesManager::stop_vdma_transfer_launcher() return HAILO_SUCCESS; } -Expected ResourcesManager::program_desc_for_hw_only_flow(std::shared_ptr desc_list, +Expected ResourcesManager::program_desc_for_hw_only_flow(vdma::DescriptorList &desc_list, + vdma::MappedBuffer &mapped_buffer, vdma::ChannelId channel_id, const uint32_t single_transfer_size, const uint16_t dynamic_batch_size, const uint16_t batch_count) { size_t acc_desc_offset = 0; @@ -741,10 +739,11 @@ Expected ResourcesManager::program_desc_for_hw_only_flow(std::shared_p for (uint16_t transfer_index = 0; transfer_index < dynamic_batch_size; transfer_index++) { const auto last_desc_interrupts_domain = ((dynamic_batch_size - 1) == transfer_index) ? InterruptsDomain::DEVICE : InterruptsDomain::NONE; - auto desc_count_local = desc_list->program_last_descriptor(single_transfer_size, - last_desc_interrupts_domain, acc_desc_offset); - CHECK_EXPECTED(desc_count_local, "Failed to program descs for inter context channels. Given max_batch_size is too big."); - acc_desc_offset += desc_count_local.value(); + const bool should_bind = false; + CHECK_SUCCESS(desc_list.program(mapped_buffer, single_transfer_size, + (acc_desc_offset * desc_list.desc_page_size()), channel_id, static_cast(acc_desc_offset), + should_bind, last_desc_interrupts_domain)); + acc_desc_offset += desc_list.descriptors_in_buffer(single_transfer_size); } } CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(acc_desc_offset), HAILO_INTERNAL_FAILURE, @@ -758,28 +757,27 @@ Expected> ResourcesManager::create_mapped_b { const auto total_frames_per_run = dynamic_batch_size * batch_count; - auto desc_list = boundary_channel_ptr->get_desc_list(); - const auto descs_per_transfer = desc_list->descriptors_in_buffer(single_transfer_size); + auto &desc_list = boundary_channel_ptr->get_desc_list(); + const auto descs_per_transfer = desc_list.descriptors_in_buffer(single_transfer_size); const auto total_desc_count = total_frames_per_run * descs_per_transfer; CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(total_desc_count), HAILO_INVALID_ARGUMENT, "calculated total_desc_count for vdma descriptor list is out of UINT16 range"); - auto mapped_buffer = vdma::MappedBuffer::create_shared_by_allocation( - total_desc_count * desc_list->desc_page_size(), m_driver, direction); - CHECK_EXPECTED(mapped_buffer); - m_hw_only_boundary_buffers.emplace_back(mapped_buffer.release()); + TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared_by_allocation( + total_desc_count * desc_list.desc_page_size(), m_driver, direction)); + m_hw_only_boundary_buffers.emplace_back(std::move(mapped_buffer)); static const auto DEFAULT_BUFFER_OFFSET = 0; - auto status = desc_list->configure_to_use_buffer(*m_hw_only_boundary_buffers.back(), + auto status = desc_list.program(*m_hw_only_boundary_buffers.back(), m_hw_only_boundary_buffers.back()->size(), DEFAULT_BUFFER_OFFSET, boundary_channel_ptr->get_channel_id()); CHECK_SUCCESS_AS_EXPECTED(status); - auto desc_programed = program_desc_for_hw_only_flow(desc_list, single_transfer_size, dynamic_batch_size, batch_count); - CHECK_EXPECTED(desc_programed); - assert(static_cast(total_desc_count) == desc_programed.value()); + TRY(auto desc_programed, + program_desc_for_hw_only_flow(desc_list, *m_hw_only_boundary_buffers.back(), boundary_channel_ptr->get_channel_id(), single_transfer_size, dynamic_batch_size, batch_count)); + assert(static_cast(total_desc_count) == desc_programed); - auto channel_info_pair = std::make_pair(boundary_channel_ptr->get_channel_id(), desc_programed.release()); + auto channel_info_pair = std::make_pair(boundary_channel_ptr->get_channel_id(), desc_programed); return channel_info_pair; } @@ -824,10 +822,8 @@ Expected ResourcesManager::calc_hw_infer_batch_count(uint16_t dynamic_ const auto &stream_infos = LayerInfoUtils::get_stream_infos_from_layer_info(layer_info); for (auto &stream_info : stream_infos) { uint32_t single_transfer_size = LayerInfoUtils::get_stream_transfer_size(stream_info, layer_info); - auto boundary_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(layer_info.name); - CHECK_EXPECTED(boundary_channel_ptr_exp); - auto boundary_channel_ptr = boundary_channel_ptr_exp.release(); - const auto max_batch_transfers = boundary_channel_ptr->get_desc_list()->max_transfers(single_transfer_size * dynamic_batch_size); + TRY(auto boundary_channel_ptr, get_boundary_vdma_channel_by_stream_name(layer_info.name)); + const auto max_batch_transfers = boundary_channel_ptr->get_desc_list().max_transfers(single_transfer_size * dynamic_batch_size); // infer batch count is the lowest number of "Max transfers" per descriptor list that for all given boundary channels. batch_count = MIN(batch_count, max_batch_transfers); } @@ -868,15 +864,11 @@ Expected ResourcesManager::run_hw_only_infer() channels_info.channel_count = 0; static constexpr auto INFER_TIMEOUT = std::chrono::milliseconds(120000); - auto batch_size = get_batch_size(); - CHECK_EXPECTED(batch_size); - - auto batch_count = calc_hw_infer_batch_count(*batch_size); - CHECK_EXPECTED(batch_count); + TRY(const auto batch_size, get_batch_size()); + TRY(const auto batch_count, calc_hw_infer_batch_count(batch_size)); for (const auto &layer_info : m_core_op_metadata->get_all_layer_infos()) { - auto boundary_channel_ptr = get_boundary_vdma_channel_by_stream_name(layer_info.name); - CHECK_EXPECTED(boundary_channel_ptr); + TRY(auto boundary_channel_ptr, get_boundary_vdma_channel_by_stream_name(layer_info.name)); const auto &stream_infos = LayerInfoUtils::get_stream_infos_from_layer_info(layer_info); for (auto &stream_info : stream_infos) { auto single_transfer_size = (HAILO_FORMAT_ORDER_HAILO_NMS == stream_info.format.order) ? @@ -884,11 +876,9 @@ Expected ResourcesManager::run_hw_only_infer() const auto direction = (layer_info.direction == HAILO_H2D_STREAM) ? HailoRTDriver::DmaDirection::H2D : HailoRTDriver::DmaDirection::D2H; - auto channel_info_pair = create_mapped_buffer_for_hw_only_infer(boundary_channel_ptr.release(), direction, - single_transfer_size, *batch_size, batch_count.value()); - CHECK_EXPECTED(channel_info_pair); - - add_channel_to_hw_infer_channel_info(channel_info_pair.release(), channels_info); + TRY(const auto channel_info_pair, create_mapped_buffer_for_hw_only_infer(std::move(boundary_channel_ptr), direction, + single_transfer_size, batch_size, batch_count)); + add_channel_to_hw_infer_channel_info(std::move(channel_info_pair), channels_info); } } @@ -899,8 +889,8 @@ Expected ResourcesManager::run_hw_only_infer() std::mutex mutex; std::unique_lock lock(mutex); - status = Control::start_hw_only_infer(m_vdma_device, m_core_op_index, *batch_size, - batch_count.value(), &channels_info); + status = Control::start_hw_only_infer(m_vdma_device, m_core_op_index, batch_size, + batch_count, &channels_info); CHECK_SUCCESS_AS_EXPECTED(status); infer_done_cond.wait_for(lock, INFER_TIMEOUT); @@ -908,13 +898,13 @@ Expected ResourcesManager::run_hw_only_infer() status = Control::stop_hw_only_infer(m_vdma_device, &fw_infer_results); CHECK_SUCCESS_AS_EXPECTED(status); - auto single_frame_transfer_size = m_core_op_metadata->get_total_transfer_size(); - CHECK_EXPECTED(single_frame_transfer_size); + TRY(const auto single_frame_transfer_size, m_core_op_metadata->get_total_transfer_size()); - return hw_infer_calc_stats(batch_count.value(), *batch_size, single_frame_transfer_size.release(), + return hw_infer_calc_stats(batch_count, batch_size, single_frame_transfer_size, fw_infer_results.infer_cycles); } + hailo_status ResourcesManager::fill_internal_buffers_info() { for (const auto &context_metadata : m_core_op_metadata->dynamic_contexts()) { @@ -946,19 +936,17 @@ bool ResourcesManager::should_use_ddr_action_list(size_t num_contexts, HailoRTDr } Expected> ResourcesManager::create_action_list_buffer_builder( - size_t num_dynamic_contexts, HailoRTDriver &driver) + size_t num_dynamic_contexts, VdmaDevice &vdma_device) { static const auto total_num_contexts = CONTROL_PROTOCOL__CONTEXT_SWITCH_NUMBER_OF_NON_DYNAMIC_CONTEXTS + num_dynamic_contexts; - if (should_use_ddr_action_list(total_num_contexts, driver.dma_type())) { - auto ddr_action_list_buffer_builder = DDRActionListBufferBuilder::create(total_num_contexts, driver); - CHECK_EXPECTED(ddr_action_list_buffer_builder); - return std::static_pointer_cast(ddr_action_list_buffer_builder.release()); + if (should_use_ddr_action_list(total_num_contexts, vdma_device.get_driver().dma_type())) { + TRY(auto ddr_action_list_buffer_builder, DDRActionListBufferBuilder::create(total_num_contexts, vdma_device)); + return std::static_pointer_cast(std::move(ddr_action_list_buffer_builder)); } else { - auto control_action_list_buffer_builder = ControlActionListBufferBuilder::create(); - CHECK_EXPECTED(control_action_list_buffer_builder); - return std::static_pointer_cast(control_action_list_buffer_builder.release()); + TRY(auto control_action_list_buffer_builder, ControlActionListBufferBuilder::create()); + return std::static_pointer_cast(std::move(control_action_list_buffer_builder)); } } diff --git a/hailort/libhailort/src/core_op/resource_manager/resource_manager.hpp b/hailort/libhailort/src/core_op/resource_manager/resource_manager.hpp index 9b89246..fc6204c 100644 --- a/hailort/libhailort/src/core_op/resource_manager/resource_manager.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/resource_manager.hpp @@ -29,6 +29,8 @@ #include "hailo/hailort.h" #include "core_op/resource_manager/intermediate_buffer.hpp" +#include "core_op/resource_manager/cache_buffer.hpp" +#include "core_op/resource_manager/cache_manager.hpp" #include "core_op/resource_manager/config_buffer.hpp" #include "core_op/resource_manager/channel_allocator.hpp" #include "core_op/resource_manager/action_list_buffer_builder/control_action_list_buffer_builder.hpp" @@ -134,8 +136,8 @@ class ResourcesManager final { public: static Expected create(VdmaDevice &vdma_device, HailoRTDriver &driver, - const ConfigureNetworkParams &config_params, std::shared_ptr core_op_metadata, - uint8_t core_op_index); + const ConfigureNetworkParams &config_params, CacheManagerPtr cache_manager, + std::shared_ptr core_op_metadata, uint8_t core_op_index); // TODO: HRT-9432 needs to call stop_vdma_interrupts_dispatcher and any other resource on dtor. ~ResourcesManager() = default; @@ -148,6 +150,9 @@ public: uint32_t transfer_size, uint16_t batch_size, uint8_t src_stream_index, uint16_t src_context_index, vdma::ChannelId d2h_channel_id, IntermediateBuffer::StreamingType streaming_type); ExpectedRef get_intermediate_buffer(const IntermediateBufferKey &key); + ExpectedRef set_cache_input_channel(uint32_t cache_id, uint16_t batch_size, vdma::ChannelId channel_id); + ExpectedRef set_cache_output_channel(uint32_t cache_id, uint16_t batch_size, vdma::ChannelId channel_id); + std::unordered_map &get_cache_buffers(); hailo_status create_boundary_vdma_channel(const LayerInfo &layer_info); Expected get_control_core_op_header(); @@ -180,11 +185,6 @@ public: return m_latency_meters; } - std::map get_boundary_vdma_channels() const - { - return m_boundary_channels; - } - std::shared_ptr& get_action_list_buffer_builder() { return m_action_list_buffer_builder; @@ -193,6 +193,8 @@ public: Expected get_default_streams_interface(); Expected read_intermediate_buffer(const IntermediateBufferKey &key); + Expected read_cache_buffer(uint32_t cache_id); + Expected> read_cache_buffers(); hailo_status configure(); hailo_status enable_state_machine(uint16_t dynamic_batch_size, @@ -206,7 +208,8 @@ public: Expected get_boundary_vdma_channel_by_stream_name(const std::string &stream_name); Expected> get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) const; hailo_power_mode_t get_power_mode() const; - Expected program_desc_for_hw_only_flow(std::shared_ptr desc_list, + Expected program_desc_for_hw_only_flow(vdma::DescriptorList &desc_list, + vdma::MappedBuffer &mapped_buffer, vdma::ChannelId channel_id, const uint32_t single_transfer_size, const uint16_t dynamic_batch_size, const uint16_t batch_count); Expected> create_mapped_buffer_for_hw_only_infer( vdma::BoundaryChannelPtr boundary_channel_ptr, const HailoRTDriver::DmaDirection direction, @@ -221,7 +224,7 @@ public: hailo_status fill_internal_buffers_info(); static bool should_use_ddr_action_list(size_t num_contexts, HailoRTDriver::DmaType dma_type); static Expected> create_action_list_buffer_builder( - size_t num_dynamic_contexts, HailoRTDriver &driver); + size_t num_dynamic_contexts, VdmaDevice &vdma_device); bool get_can_fast_batch_switch() { return m_core_op_metadata->get_can_fast_batch_switch(); @@ -242,14 +245,18 @@ private: hailo_status fill_validation_features(CONTROL_PROTOCOL__application_header_t &app_header); hailo_status fill_network_batch_size(CONTROL_PROTOCOL__application_header_t &app_header); hailo_status fill_csm_buffer_size(CONTROL_PROTOCOL__application_header_t &app_header); - void process_interrupts(IrqData &&irq_data); Expected get_batch_size() const; + // + static std::pair calculate_transfer_queue_sizes(const vdma::DescriptorList &desc_list, + uint32_t transfer_size, uint32_t max_active_trans, bool use_latency_meter); + std::vector m_contexts_resources; ChannelAllocator m_channel_allocator; VdmaDevice &m_vdma_device; HailoRTDriver &m_driver; const ConfigureNetworkParams m_config_params; + CacheManagerPtr m_cache_manager; std::map m_intermediate_buffers; std::shared_ptr m_core_op_metadata; uint8_t m_core_op_index; @@ -257,8 +264,7 @@ private: uint16_t m_total_context_count; const std::vector m_network_index_map; LatencyMetersMap m_latency_meters; // Latency meter per network - // TODO: HRT-9429 - fast access to channel by id, using array, using engine_index and channel_index. - std::map m_boundary_channels; + vdma::ChannelsGroup m_boundary_channels; bool m_is_configured; bool m_is_activated; // Config channels ids are shared between all context. The following vector contains the channel id for each @@ -271,6 +277,7 @@ private: ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &driver, ChannelAllocator &&channel_allocator, const ConfigureNetworkParams config_params, + CacheManagerPtr cache_manager, std::shared_ptr &&core_op_metadata, uint8_t core_op_index, const std::vector &&network_index_map, LatencyMetersMap &&latency_meters, std::vector &&config_channels_ids, diff --git a/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.cpp b/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.cpp index e86c141..0f8985f 100644 --- a/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.cpp @@ -11,6 +11,7 @@ #include "device_common/control.hpp" #include "periph_calculator.hpp" #include "hef/hef_internal.hpp" +#include "common/file_utils.hpp" namespace hailort { @@ -91,13 +92,11 @@ static Expected update_layer_info(const LayerInfo &original_layer_inf local_layer_info.nn_stream_config.is_periph_calculated_in_hailort = is_periph_calculated_in_hailort; local_layer_info.nn_stream_config.is_core_hw_padding_config_in_dfc = is_core_hw_padding_config_in_dfc; - auto updated_periph_layer_info = PeriphCalculator::calculate_periph_registers(local_layer_info, - buffer_info.desc_page_size, is_periph_calculated_in_hailort, hw_arch, is_core_hw_padding_config_in_dfc); - CHECK_EXPECTED(updated_periph_layer_info); + TRY(const auto updated_periph_layer_info, PeriphCalculator::calculate_periph_registers(local_layer_info, + buffer_info.desc_page_size, is_periph_calculated_in_hailort, hw_arch, is_core_hw_padding_config_in_dfc)); - auto updated_local_layer_info = calculate_credit_params(hw_consts, buffer_info.desc_page_size, should_optimize_credits, - updated_periph_layer_info.release()); - CHECK_EXPECTED(updated_local_layer_info); + TRY(auto updated_local_layer_info, calculate_credit_params(hw_consts, buffer_info.desc_page_size, should_optimize_credits, + updated_periph_layer_info)); return updated_local_layer_info; } @@ -108,18 +107,16 @@ static hailo_status fill_boundary_input_layer_impl(ContextResources &context_res { const auto transfer_size = LayerInfoUtils::get_layer_transfer_size(layer_info); - auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name); - CHECK_EXPECTED_AS_STATUS(vdma_channel); + TRY(const auto vdma_channel, resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name)); - const auto buffer_info = vdma_channel.value()->get_boundary_buffer_info(transfer_size); + const auto buffer_info = vdma_channel->get_boundary_buffer_info(transfer_size); const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort; const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc; - auto local_layer_info = update_layer_info(layer_info, buffer_info, hw_consts, hw_arch, should_optimize_credits, - is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc); - CHECK_EXPECTED_AS_STATUS(local_layer_info); + TRY(auto local_layer_info, update_layer_info(layer_info, buffer_info, hw_consts, hw_arch, should_optimize_credits, + is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc)); - const auto channel_id = vdma_channel.value()->get_channel_id(); - auto status = context_resources.add_edge_layer(local_layer_info.value(), channel_id, buffer_info, + const auto channel_id = vdma_channel->get_channel_id(); + auto status = context_resources.add_edge_layer(local_layer_info, channel_id, buffer_info, resources_manager.get_supported_features()); CHECK_SUCCESS(status); @@ -146,31 +143,28 @@ static hailo_status fill_inter_context_input_layer(ContextResources &context_res ResourcesManager &resources_manager, const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch, bool should_optimize_credits) { - const auto channel_id = resources_manager.get_available_channel_id(to_layer_identifier(layer_info), - HailoRTDriver::DmaDirection::H2D, layer_info.dma_engine_index); - CHECK_EXPECTED_AS_STATUS(channel_id); + TRY(const auto channel_id, resources_manager.get_available_channel_id(to_layer_identifier(layer_info), + HailoRTDriver::DmaDirection::H2D, layer_info.dma_engine_index)); /* Get inter context buffer previously created */ const auto &connected_context = layer_info.connected_context_info; auto intermediate_buffer_key = std::make_pair(connected_context.context_index, connected_context.stream_index); - auto inter_context_buffer_exp = resources_manager.get_intermediate_buffer(intermediate_buffer_key); - CHECK_EXPECTED_AS_STATUS(inter_context_buffer_exp, "Failed to find inter context buffer for src context {}, src_stream_index {}", + TRY(auto inter_context_buffer, resources_manager.get_intermediate_buffer(intermediate_buffer_key), + "Failed to find inter context buffer for src context {}, src_stream_index {}", connected_context.context_index, connected_context.stream_index); - auto &inter_context_buffer = inter_context_buffer_exp->get(); const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort; const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc; - auto local_layer_info = update_layer_info(layer_info, inter_context_buffer.get_host_buffer_info(), hw_consts, - hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc); - CHECK_EXPECTED_AS_STATUS(local_layer_info); + TRY(auto local_layer_info, update_layer_info(layer_info, inter_context_buffer.get().get_host_buffer_info(), hw_consts, + hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc)); - auto status = context_resources.add_edge_layer(local_layer_info.value(), channel_id.value(), - inter_context_buffer.get_host_buffer_info(), resources_manager.get_supported_features()); + auto status = context_resources.add_edge_layer(local_layer_info, channel_id, + inter_context_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features()); CHECK_SUCCESS(status); LOGGER__DEBUG("Intermediate edge key: {}:{} src_context:{}, dst_context: {}, h2d_channel {}.", connected_context.context_index, connected_context.stream_index, - layer_info.connected_context_info.context_index, layer_info.context_index, channel_id.value()); + layer_info.connected_context_info.context_index, layer_info.context_index, channel_id); return HAILO_SUCCESS; } @@ -181,18 +175,16 @@ static hailo_status fill_boundary_output_layer(ContextResources &context_resourc { const auto transfer_size = LayerInfoUtils::get_layer_transfer_size(layer_info); - auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name); - CHECK_EXPECTED_AS_STATUS(vdma_channel); + TRY(const auto vdma_channel, resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name)); - const auto buffer_info = vdma_channel.value()->get_boundary_buffer_info(transfer_size); + const auto buffer_info = vdma_channel->get_boundary_buffer_info(transfer_size); const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort; const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc; - auto local_layer_info = update_layer_info(layer_info, buffer_info, hw_consts, hw_arch, should_optimize_credits, - is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc); - CHECK_EXPECTED_AS_STATUS(local_layer_info); + TRY(auto local_layer_info, update_layer_info(layer_info, buffer_info, hw_consts, hw_arch, should_optimize_credits, + is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc)); - const auto channel_id = vdma_channel.value()->get_channel_id(); - auto status = context_resources.add_edge_layer(local_layer_info.value(), channel_id, buffer_info, + const auto channel_id = vdma_channel->get_channel_id(); + auto status = context_resources.add_edge_layer(local_layer_info, channel_id, buffer_info, resources_manager.get_supported_features()); CHECK_SUCCESS(status); @@ -204,33 +196,28 @@ static hailo_status fill_inter_context_output_layer(ContextResources &context_re ResourcesManager &resources_manager, const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch, bool should_optimize_credits) { - const auto channel_id = resources_manager.get_available_channel_id(to_layer_identifier(layer_info), - HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index); - CHECK_EXPECTED_AS_STATUS(channel_id); + TRY(const auto channel_id, resources_manager.get_available_channel_id(to_layer_identifier(layer_info), + HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index)); const auto frame_credits_in_bytes = LayerInfoUtils::get_layer_transfer_size(layer_info); - auto network_batch_size = resources_manager.get_network_batch_size(layer_info.network_name); - CHECK_EXPECTED_AS_STATUS(network_batch_size); + TRY(const auto network_batch_size, resources_manager.get_network_batch_size(layer_info.network_name)); - auto inter_context_buffer_exp = resources_manager.create_intermediate_buffer(frame_credits_in_bytes, - network_batch_size.value(), layer_info.stream_index, layer_info.context_index, - channel_id.value(), IntermediateBuffer::StreamingType::BURST); - CHECK_EXPECTED_AS_STATUS(inter_context_buffer_exp); - auto &inter_context_buffer = inter_context_buffer_exp->get(); + TRY(auto inter_context_buffer, resources_manager.create_intermediate_buffer(frame_credits_in_bytes, + network_batch_size, layer_info.stream_index, layer_info.context_index, + channel_id, IntermediateBuffer::StreamingType::BURST)); const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort; const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc; - auto local_layer_info = update_layer_info(layer_info, inter_context_buffer.get_host_buffer_info(), hw_consts, - hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc); - CHECK_EXPECTED_AS_STATUS(local_layer_info); + TRY(auto local_layer_info, update_layer_info(layer_info, inter_context_buffer.get().get_host_buffer_info(), hw_consts, + hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc)); - auto status = context_resources.add_edge_layer(local_layer_info.value(), channel_id.value(), - inter_context_buffer.get_host_buffer_info(), resources_manager.get_supported_features()); + auto status = context_resources.add_edge_layer(local_layer_info, channel_id, + inter_context_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features()); CHECK_SUCCESS(status); LOGGER__DEBUG("Inter-context output stream {}, src_context:{}, d2h_channel {}.", - layer_info.stream_index, layer_info.context_index, channel_id.value()); + layer_info.stream_index, layer_info.context_index, channel_id); return HAILO_SUCCESS; } @@ -247,16 +234,14 @@ static hailo_status fill_ddr_output_layer(ContextResources &context_resources, const auto h2d_stream_index = layer_info.connected_context_info.stream_index; const auto h2d_layer_identifier = std::make_tuple(LayerType::DDR, HAILO_H2D_STREAM, layer_info.name, h2d_stream_index); - const auto h2d_channel_id = resources_manager.get_available_channel_id(h2d_layer_identifier, - HailoRTDriver::DmaDirection::H2D, layer_info.connected_context_info.dma_engine_index); - CHECK_EXPECTED_AS_STATUS(h2d_channel_id); + TRY(const auto h2d_channel_id, resources_manager.get_available_channel_id(h2d_layer_identifier, + HailoRTDriver::DmaDirection::H2D, layer_info.connected_context_info.dma_engine_index)); const auto d2h_stream_index = layer_info.stream_index; const auto d2h_layer_identifier = std::make_tuple(LayerType::DDR, HAILO_D2H_STREAM, layer_info.name, d2h_stream_index); - const auto d2h_channel_id = resources_manager.get_available_channel_id(d2h_layer_identifier, - HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index); - CHECK_EXPECTED_AS_STATUS(d2h_channel_id); + TRY(const auto d2h_channel_id, resources_manager.get_available_channel_id(d2h_layer_identifier, + HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index)); // In DDR - always use core bytes per buffer as row size const auto row_size = static_cast(layer_info.nn_stream_config.core_bytes_per_buffer); @@ -265,21 +250,20 @@ static hailo_status fill_ddr_output_layer(ContextResources &context_resources, const auto min_buffered_rows = layer_info.ddr_info.min_buffered_rows; // Allocate the ddr buffer - auto ddr_buffer = resources_manager.create_intermediate_buffer(row_size, min_buffered_rows, - d2h_stream_index, layer_info.context_index, d2h_channel_id.value(), - IntermediateBuffer::StreamingType::CIRCULAR_CONTINUOS); - CHECK_EXPECTED_AS_STATUS(ddr_buffer); + TRY(auto ddr_buffer, resources_manager.create_intermediate_buffer(row_size, min_buffered_rows, + d2h_stream_index, layer_info.context_index, d2h_channel_id, + IntermediateBuffer::StreamingType::CIRCULAR_CONTINUOS)); DdrChannelsInfo ddr_pair_info{}; ddr_pair_info.h2d_stream_index = h2d_stream_index; ddr_pair_info.d2h_stream_index = d2h_stream_index; ddr_pair_info.network_index = layer_info.network_index; - ddr_pair_info.h2d_channel_id = h2d_channel_id.value(); - ddr_pair_info.d2h_channel_id = d2h_channel_id.value(); + ddr_pair_info.h2d_channel_id = h2d_channel_id; + ddr_pair_info.d2h_channel_id = d2h_channel_id; ddr_pair_info.row_size = row_size; ddr_pair_info.min_buffered_rows = min_buffered_rows; ddr_pair_info.total_buffers_per_frame = layer_info.ddr_info.total_buffers_per_frame; - ddr_pair_info.host_buffer_info = ddr_buffer->get().get_host_buffer_info(); + ddr_pair_info.host_buffer_info = ddr_buffer.get().get_host_buffer_info(); context_resources.add_ddr_channels_info(ddr_pair_info); // On ddr layers, we assume the periph credit size is aligned to the size of descriptor, so we don't want to @@ -287,12 +271,11 @@ static hailo_status fill_ddr_output_layer(ContextResources &context_resources, const bool should_optimize_credits = false; const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort; const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc; - auto local_layer_info = update_layer_info(layer_info, ddr_buffer->get().get_host_buffer_info(), hw_consts, - hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc); - CHECK_EXPECTED_AS_STATUS(local_layer_info); + TRY(auto local_layer_info, update_layer_info(layer_info, ddr_buffer.get().get_host_buffer_info(), hw_consts, + hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc)); - auto status = context_resources.add_edge_layer(local_layer_info.value(), ddr_pair_info.d2h_channel_id, - ddr_buffer->get().get_host_buffer_info(), resources_manager.get_supported_features()); + auto status = context_resources.add_edge_layer(local_layer_info, ddr_pair_info.d2h_channel_id, + ddr_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features()); CHECK_SUCCESS(status); return HAILO_SUCCESS; @@ -302,51 +285,96 @@ static hailo_status fill_ddr_input_layer(ContextResources &context_resources, Re const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch) { auto connected_stream_index = layer_info.connected_context_info.stream_index; - auto ddr_info = context_resources.get_ddr_channels_info(connected_stream_index); - CHECK_EXPECTED_AS_STATUS(ddr_info, "Matching DDR layer as not found for context {} src stream {}", - layer_info.context_index, connected_stream_index); + TRY(const auto ddr_info, context_resources.get_ddr_channels_info(connected_stream_index), + "Matching DDR layer as not found for context {} src stream {}", layer_info.context_index, connected_stream_index); LOGGER__DEBUG("DDR layer: input stream_index: {}, output stream_index: {}, h2d_channel {}, d2h_channel: {}.", - ddr_info->h2d_stream_index, ddr_info->d2h_stream_index, ddr_info->h2d_channel_id, ddr_info->d2h_channel_id); + ddr_info.h2d_stream_index, ddr_info.d2h_stream_index, ddr_info.h2d_channel_id, ddr_info.d2h_channel_id); - CHECK(layer_info.stream_index == ddr_info->h2d_stream_index, HAILO_INVALID_HEF, "DDR channel pair mismatch in h2d channel"); - CHECK(layer_info.connected_context_info.stream_index == ddr_info->d2h_stream_index, HAILO_INVALID_HEF, "DDR channel pair mismatch in d2h channel"); - CHECK(layer_info.network_index == ddr_info->network_index, HAILO_INVALID_HEF, "DDR channel pair mismatch network_index"); + CHECK(layer_info.stream_index == ddr_info.h2d_stream_index, HAILO_INVALID_HEF, "DDR channel pair mismatch in h2d channel"); + CHECK(layer_info.connected_context_info.stream_index == ddr_info.d2h_stream_index, HAILO_INVALID_HEF, "DDR channel pair mismatch in d2h channel"); + CHECK(layer_info.network_index == ddr_info.network_index, HAILO_INVALID_HEF, "DDR channel pair mismatch network_index"); // On ddr layers, we assume the periph credit size is aligned to the size of descriptor, so we don't want to // optimize the credits. const bool should_optimize_credits = false; const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort; const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc; - auto local_layer_info = update_layer_info(layer_info, ddr_info->host_buffer_info, hw_consts, - hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc); - CHECK_EXPECTED_AS_STATUS(local_layer_info); + TRY(auto local_layer_info, update_layer_info(layer_info, ddr_info.host_buffer_info, hw_consts, + hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc)); - auto status = context_resources.add_edge_layer(local_layer_info.value(), ddr_info->h2d_channel_id, - ddr_info->host_buffer_info, resources_manager.get_supported_features()); + auto status = context_resources.add_edge_layer(local_layer_info, ddr_info.h2d_channel_id, + ddr_info.host_buffer_info, resources_manager.get_supported_features()); CHECK_SUCCESS(status); return HAILO_SUCCESS; } +static hailo_status fill_cache_output_layer(ContextResources &context_resources, ResourcesManager &resources_manager, + const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch, + bool should_optimize_credits) +{ + TRY(const auto channel_id, resources_manager.get_available_channel_id(to_layer_identifier(layer_info), + HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index)); + + TRY(const auto network_batch_size, resources_manager.get_network_batch_size(layer_info.network_name)); + TRY(auto cache_buffer, resources_manager.set_cache_output_channel(layer_info.cache_info.id, + network_batch_size, channel_id)); + + const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort; + const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc; + TRY(auto local_layer_info, update_layer_info(layer_info, cache_buffer.get().get_host_buffer_info(), hw_consts, + hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc)); + + auto status = context_resources.add_edge_layer(local_layer_info, channel_id, + cache_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features()); + CHECK_SUCCESS(status); + + LOGGER__DEBUG("Cache id {}: output stream {}, d2h_channel {}, context {}", + layer_info.cache_info.id, layer_info.stream_index, channel_id, layer_info.context_index); + return HAILO_SUCCESS; +} + +static hailo_status fill_cache_input_layer(ContextResources &context_resources, ResourcesManager &resources_manager, + const LayerInfo &layer_info, const CONTROL_PROTOCOL__hw_consts_t &hw_consts, const HEFHwArch &hw_arch, bool should_optimize_credits) +{ + TRY(const auto channel_id, resources_manager.get_available_channel_id(to_layer_identifier(layer_info), + HailoRTDriver::DmaDirection::H2D, layer_info.dma_engine_index)); + + TRY(const auto network_batch_size, resources_manager.get_network_batch_size(layer_info.network_name)); + TRY(auto cache_buffer, resources_manager.set_cache_input_channel(layer_info.cache_info.id, network_batch_size, channel_id)); + + const bool is_periph_calculated_in_hailort = resources_manager.get_supported_features().periph_calculation_in_hailort; + const bool is_core_hw_padding_config_in_dfc = resources_manager.get_supported_features().core_hw_padding_config_in_dfc; + TRY(auto local_layer_info, update_layer_info(layer_info, cache_buffer.get().get_host_buffer_info(), hw_consts, + hw_arch, should_optimize_credits, is_periph_calculated_in_hailort, is_core_hw_padding_config_in_dfc)); + + auto status = context_resources.add_edge_layer(local_layer_info, channel_id, + cache_buffer.get().get_host_buffer_info(), resources_manager.get_supported_features()); + CHECK_SUCCESS(status); + + LOGGER__DEBUG("Cache id {}: input stream {}, h2d_channel {}, context {}", + layer_info.cache_info.id, layer_info.stream_index, channel_id, layer_info.context_index); + + return HAILO_SUCCESS; +} + static hailo_status add_ddr_buffers_info(std::vector &configuration_actions, const ContextResources &context_resources) { bool start_fw_ddr_buffer_task = false; for (const auto &ddr_info : context_resources.get_ddr_channels_infos()) { if (ddr_info.need_manual_credit_management()) { - auto ddr_pair_action = DdrPairInfoAction::create(ddr_info.h2d_channel_id, ddr_info.d2h_channel_id, - ddr_info.network_index, ddr_info.descriptors_per_frame(), ddr_info.descs_count()); - CHECK_EXPECTED_AS_STATUS(ddr_pair_action); - configuration_actions.emplace_back(ddr_pair_action.release()); + TRY(auto ddr_pair_action, DdrPairInfoAction::create(ddr_info.h2d_channel_id, ddr_info.d2h_channel_id, + ddr_info.network_index, ddr_info.descriptors_per_frame(), ddr_info.descs_count())); + configuration_actions.emplace_back(std::move(ddr_pair_action)); start_fw_ddr_buffer_task = true; } } if (start_fw_ddr_buffer_task) { - auto start_ddr_buffering_action = StartDdrBufferingTaskAction::create(); - CHECK_EXPECTED_AS_STATUS(start_ddr_buffering_action); - configuration_actions.emplace_back(start_ddr_buffering_action.release()); + TRY(auto start_ddr_buffering_action, StartDdrBufferingTaskAction::create()); + configuration_actions.emplace_back(std::move(start_ddr_buffering_action)); } return HAILO_SUCCESS; @@ -359,9 +387,8 @@ static hailo_status parse_and_fill_edge_layers_mapping( { hailo_status status = HAILO_UNINITIALIZED; - auto hw_consts = Control::get_hw_consts(resources_manager.get_device()); - CHECK_EXPECTED_AS_STATUS(hw_consts); - const bool should_optimize_credits = hw_consts->should_optimize_credits && + TRY(const auto hw_consts, Control::get_hw_consts(resources_manager.get_device())); + const bool should_optimize_credits = hw_consts.should_optimize_credits && (HAILO_POWER_MODE_PERFORMANCE == resources_manager.get_power_mode()); // Parse the edge layer by order - first output edge layers, then ddr inputs and only then the input edge layers @@ -369,41 +396,53 @@ static hailo_status parse_and_fill_edge_layers_mapping( // We parse ddr inputs before boundary/inter-context because otherwise on C2C mode we may lose some credit. for (const auto &output_layer_info : context_metadata.get_ddr_output_layers()) { - status = fill_ddr_output_layer(context_resources, resources_manager, output_layer_info, *hw_consts, hw_arch); + status = fill_ddr_output_layer(context_resources, resources_manager, output_layer_info, hw_consts, hw_arch); CHECK_SUCCESS(status); } for (const auto &output_layer_info : context_metadata.get_boundary_output_layers()) { status = fill_boundary_output_layer(context_resources, resources_manager, output_layer_info, - *hw_consts, hw_arch, should_optimize_credits); + hw_consts, hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } for (const auto &output_layer_info : context_metadata.get_inter_context_output_layers()) { status = fill_inter_context_output_layer(context_resources, resources_manager, output_layer_info, - *hw_consts, hw_arch, should_optimize_credits); + hw_consts, hw_arch, should_optimize_credits); + CHECK_SUCCESS(status); + } + + for (const auto &cache_layer_info : context_metadata.get_cache_output_layers()) { + status = fill_cache_output_layer(context_resources, resources_manager, cache_layer_info, hw_consts, + hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } for (const auto &input_layer_info : context_metadata.get_ddr_input_layers()) { - status = fill_ddr_input_layer(context_resources, resources_manager, input_layer_info, *hw_consts, hw_arch); + status = fill_ddr_input_layer(context_resources, resources_manager, input_layer_info, hw_consts, hw_arch); CHECK_SUCCESS(status); } for (const auto &input_layer_info : context_metadata.get_boundary_input_layers()) { status = fill_boundary_input_layer(context_resources, resources_manager, input_layer_info, - *hw_consts, hw_arch, should_optimize_credits); + hw_consts, hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } for (const auto &input_layer_info : context_metadata.get_inter_context_input_layers()) { status = fill_inter_context_input_layer(context_resources, resources_manager, input_layer_info, - *hw_consts, hw_arch, should_optimize_credits); + hw_consts, hw_arch, should_optimize_credits); + CHECK_SUCCESS(status); + } + + for (const auto &cache_layer_info : context_metadata.get_cache_input_layers()) { + status = fill_cache_input_layer(context_resources, resources_manager, cache_layer_info, hw_consts, + hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } - /* UN-Lock resources at the end of the context - - h2d inter-context, d2h inter-context and DDR buffer channels */ + // Unlock resources at the end of the context - + // Inter-context channels, DDR buffer channels and cache channels for (const auto &input_layer_info : context_metadata.get_inter_context_input_layers()) { status = resources_manager.free_channel_index(to_layer_identifier(input_layer_info)); CHECK_SUCCESS(status); @@ -426,6 +465,16 @@ static hailo_status parse_and_fill_edge_layers_mapping( CHECK_SUCCESS(status); } + for (const auto &input_layer_info : context_metadata.get_cache_input_layers()) { + status = resources_manager.free_channel_index(to_layer_identifier(input_layer_info)); + CHECK_SUCCESS(status); + } + + for (const auto &output_layer_info : context_metadata.get_cache_output_layers()) { + status = resources_manager.free_channel_index(to_layer_identifier(output_layer_info)); + CHECK_SUCCESS(status); + } + return HAILO_SUCCESS; } @@ -534,16 +583,12 @@ static hailo_status push_fetch_config_actions( std::vector &processed_configuration_actions) { if (support_pre_fetch) { - auto action = AddCcwBurstAction::create(config_stream_index, total_ccw_bursts); - CHECK_EXPECTED_AS_STATUS(action); - processed_configuration_actions.emplace_back(action.release()); + TRY(const auto action, AddCcwBurstAction::create(config_stream_index, total_ccw_bursts)); + processed_configuration_actions.emplace_back(std::move(action)); } else { - const auto desc_count = config_resources.program_descriptors(); - CHECK_EXPECTED_AS_STATUS(desc_count); - - auto action = FetchCfgChannelDescriptorsAction::create(config_resources.channel_id(), desc_count.value()); - CHECK_EXPECTED_AS_STATUS(action); - processed_configuration_actions.emplace_back(action.release()); + TRY(const auto desc_count, config_resources.program_descriptors()); + TRY(const auto action, FetchCfgChannelDescriptorsAction::create(config_resources.channel_id(), desc_count)); + processed_configuration_actions.emplace_back(std::move(action)); } return HAILO_SUCCESS; @@ -588,22 +633,19 @@ static hailo_status add_change_vdma_to_stream_mapping_impl(const HEFHwArch &hw_a ContextResources &context_resources, uint16_t context_index, std::vector &processed_configuration_actions) { - auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name); - CHECK_EXPECTED_AS_STATUS(vdma_channel); + TRY(auto vdma_channel, resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name)); - const auto channel_id = vdma_channel.value()->get_channel_id(); + const auto channel_id = vdma_channel->get_channel_id(); const bool is_dummy_stream = layer_info.context_index != context_index; uint8_t stream_index = layer_info.stream_index; if (is_dummy_stream) { - auto dummy_stream_index = find_dummy_stream(layer_info, context_resources, - HailoRTCommon::is_hailo1x_device_type(DeviceBase::hef_arch_to_device_arch(hw_arch))); - CHECK_EXPECTED_AS_STATUS(dummy_stream_index); - stream_index = *dummy_stream_index; + TRY(const auto dummy_stream_index, find_dummy_stream(layer_info, context_resources, + HailoRTCommon::is_hailo1x_device_type(DeviceBase::hef_arch_to_device_arch(hw_arch)))); + stream_index = dummy_stream_index; } - auto action = ChangeVdmaToStreamMapping::create(channel_id, stream_index, is_dummy_stream); - CHECK_EXPECTED_AS_STATUS(action); - processed_configuration_actions.emplace_back(action.release()); + TRY(const auto action, ChangeVdmaToStreamMapping::create(channel_id, stream_index, is_dummy_stream)); + processed_configuration_actions.emplace_back(std::move(action)); return HAILO_SUCCESS; } @@ -640,60 +682,75 @@ static hailo_status push_edge_layer_activation_actions( // We parse ddr inputs before boundary/inter-context because otherwise on C2C mode we may lose some credit. for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::DDR, HAILO_D2H_STREAM)) { - auto activate_action = ActivateDdrOutputChannelAction::create(edge_layer.channel_id, + TRY(const auto activate_action, ActivateDdrOutputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info, - edge_layer.layer_info.ddr_info.min_buffered_rows); - CHECK_EXPECTED_AS_STATUS(activate_action); - actions.emplace_back(activate_action.release()); + edge_layer.layer_info.ddr_info.min_buffered_rows)); + actions.emplace_back(std::move(activate_action)); } if (!push_internal_only) { for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_D2H_STREAM)) { - auto activate_action = ActivateBoundaryOutputChannelAction::create(edge_layer.channel_id, + TRY(const auto activate_action, ActivateBoundaryOutputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.network_index, - edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info); - CHECK_EXPECTED_AS_STATUS(activate_action); - actions.emplace_back(activate_action.release()); + edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info)); + actions.emplace_back(std::move(activate_action)); } } for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::INTER_CONTEXT, HAILO_D2H_STREAM)) { - auto activate_action = ActivateInterContextOutputChannelAction::create(edge_layer.channel_id, + TRY(auto activate_action, ActivateInterContextOutputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.network_index, - edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info); - CHECK_EXPECTED_AS_STATUS(activate_action); - actions.emplace_back(activate_action.release()); + edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info)); + actions.emplace_back(std::move(activate_action)); + } + + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::CACHE, HAILO_D2H_STREAM)) { + // TODO: Cache edge_layers' buffer_info isn't correct - total_desc_count and bytes_in_pattern are calculated + // according to the cache size, not the transfer size (since the edge layer is backed by a buffer of cache_size) + // Think of a way of holding the correct buffer_info for cache edge_layers (or maybe removing host buffer info all together) + // (HRT-13775) + TRY(const auto activate_action, ActivateCacheOutputChannelAction::create(edge_layer.channel_id, + edge_layer.layer_info.stream_index, edge_layer.layer_info.network_index, + edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info)); + actions.emplace_back(std::move(activate_action)); } for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::DDR, HAILO_H2D_STREAM)) { const auto d2h_stream_index = edge_layer.layer_info.connected_context_info.stream_index; - auto ddr_channels_info = context_resources.get_ddr_channels_info(d2h_stream_index); - CHECK_EXPECTED_AS_STATUS(ddr_channels_info); - const auto d2h_channel_id = ddr_channels_info->d2h_channel_id; + TRY(const auto ddr_channels_info, context_resources.get_ddr_channels_info(d2h_stream_index)); + const auto d2h_channel_id = ddr_channels_info.d2h_channel_id; - auto activate_action = ActivateDdrInputChannelAction::create(edge_layer.channel_id, + TRY(const auto activate_action, ActivateDdrInputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info, - edge_layer.layer_info.max_shmifo_size, d2h_channel_id); - CHECK_EXPECTED_AS_STATUS(activate_action); - actions.emplace_back(activate_action.release()); + edge_layer.layer_info.max_shmifo_size, d2h_channel_id)); + actions.emplace_back(std::move(activate_action)); } if (!push_internal_only) { for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) { - auto activate_action = ActivateBoundaryInputChannelAction::create(edge_layer.channel_id, + TRY(auto activate_action, ActivateBoundaryInputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info, - edge_layer.layer_info.max_shmifo_size); - CHECK_EXPECTED_AS_STATUS(activate_action); - actions.emplace_back(activate_action.release()); + edge_layer.layer_info.max_shmifo_size)); + actions.emplace_back(activate_action); } } for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::INTER_CONTEXT, HAILO_H2D_STREAM)) { - auto activate_action = ActivateInterContextInputChannelAction::create(edge_layer.channel_id, + TRY(const auto activate_action, ActivateInterContextInputChannelAction::create(edge_layer.channel_id, + edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info, + edge_layer.layer_info.max_shmifo_size)); + actions.emplace_back(std::move(activate_action)); + } + + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::CACHE, HAILO_H2D_STREAM)) { + // TODO: Cache edge_layers' buffer_info isn't correct - total_desc_count and bytes_in_pattern are calculated + // according to the cache size, not the transfer size (since the edge layer is backed by a buffer of cache_size) + // Think of a way of holding the correct buffer_info for cache edge_layers (or maybe removing host buffer info all together) + // (HRT-13775) + TRY(const auto activate_action, ActivateCacheInputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info, - edge_layer.layer_info.max_shmifo_size); - CHECK_EXPECTED_AS_STATUS(activate_action); - actions.emplace_back(activate_action.release()); + edge_layer.layer_info.max_shmifo_size)); + actions.emplace_back(std::move(activate_action)); } return HAILO_SUCCESS; @@ -728,9 +785,8 @@ static hailo_status proccess_trigger_new_data_input_action(const HEFHwArch &hw_a /* Open the boundary input channel */ for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) { - auto activate_action = ResumeVdmaChannel::create(edge_layer); - CHECK_EXPECTED_AS_STATUS(activate_action); - processed_configuration_actions.emplace_back(activate_action.release()); + TRY(const auto activate_action, ResumeVdmaChannel::create(edge_layer)); + processed_configuration_actions.emplace_back(std::move(activate_action)); } } @@ -742,9 +798,8 @@ static hailo_status proccess_trigger_new_data_input_action(const HEFHwArch &hw_a if (trigger_new_data_from_input_group_end == action_index) { auto boundary_input_edge_layers = context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM); if (boundary_input_edge_layers.size() > 0) { - auto start_burst_credits_task_action = StartBurstCreditsTaskAction::create(); - CHECK_EXPECTED_AS_STATUS(start_burst_credits_task_action); - processed_configuration_actions.emplace_back(start_burst_credits_task_action.release()); + TRY(const auto start_burst_credits_task_action, StartBurstCreditsTaskAction::create()); + processed_configuration_actions.emplace_back(std::move(start_burst_credits_task_action)); } } @@ -784,19 +839,17 @@ static hailo_status add_config_channel_activation_actions(std::vectorserialize(context_resources); - CHECK_EXPECTED_AS_STATUS(action_buffers); + TRY(auto action_buffers, action->serialize(context_resources)); - for (auto &action_buffer : action_buffers.value()) { - const bool last_action_buffer_in_context = (action_buffer == *(action_buffers.value().end() - 1)) && + for (auto &action_buffer : action_buffers) { + const bool last_action_buffer_in_context = (action_buffer == *(action_buffers.end() - 1)) && (action == *(actions.end() - 1)); builder->write_action(MemoryView(action_buffer), context_resources.get_context_type(), is_first_action_buffer_of_context, last_action_buffer_in_context); @@ -916,15 +967,14 @@ static hailo_status add_edge_layer_end_of_context_actions(const ContextResources auto action = should_validate ? ValidateChannelAction::create(edge_layer, is_batch_switch_context) : DeactivateChannelAction::create(edge_layer, is_batch_switch_context); - CHECK_EXPECTED_AS_STATUS(action); + CHECK_EXPECTED_AS_STATUS(action); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here actions.emplace_back(action.release()); } /* Pause the boundary input channel */ for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) { - auto activate_action = PauseVdmaChannel::create(edge_layer); - CHECK_EXPECTED_AS_STATUS(activate_action); - actions.emplace_back(activate_action.release()); + TRY(const auto activate_action, PauseVdmaChannel::create(edge_layer)); + actions.emplace_back(std::move(activate_action)); } return HAILO_SUCCESS; @@ -933,7 +983,7 @@ static hailo_status add_edge_layer_end_of_context_actions(const ContextResources static hailo_status fill_context_recipes_for_multi_context(const HEFHwArch &hw_arch, ContextResources &context_resources, ResourcesManager &resources_manager, uint16_t context_index, const CoreOpMetadata &core_op_metadata, const ContextMetadata &context_metadata, - bool is_single_context) + bool is_single_context, bool is_last_context, bool caches_in_use) { hailo_status status = HAILO_UNINITIALIZED; @@ -955,15 +1005,26 @@ static hailo_status fill_context_recipes_for_multi_context(const HEFHwArch &hw_a status = add_config_channel_activation_actions(actions, context_resources.get_config_buffers()); CHECK_SUCCESS(status); + // End of context actions - these must be last in their respective context if (is_single_context) { // Single context network must wait for network group change event after they finish the dynamic context. - auto wait_action = WaitForNetworkGroupChangeAction::create(); - CHECK_EXPECTED_AS_STATUS(wait_action); - actions.emplace_back(wait_action.release()); - } - else { + TRY(const auto wait_action, WaitForNetworkGroupChangeAction::create()); + actions.emplace_back(std::move(wait_action)); + } else { const bool NOT_BATCH_SWITCH_CONTEXT = false; status = add_edge_layer_end_of_context_actions(context_resources, actions, NOT_BATCH_SWITCH_CONTEXT); + + if (is_last_context && caches_in_use) { + const auto cache_offset_env_var = get_env_variable(HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR); + if (cache_offset_env_var.has_value() && + (cache_offset_env_var.value() == HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DISABLED)) { + LOGGER__INFO("Skipping cache offset updates"); + } else { + // If caches are in use, we'll wait for the caches to be updated at the end of the last context + TRY(const auto action, WaitForCacheUpdatedAction::create()); + actions.emplace_back(std::move(action)); + } + } } status = handle_repeated_actions(actions); @@ -993,33 +1054,31 @@ static hailo_status fill_activation_config_recepies_for_multi_context( ContextResources &context_resources, ResourcesManager &resources_manager, std::shared_ptr core_op_metadata, const HEFHwArch &hw_arch) { - auto hw_consts = Control::get_hw_consts(resources_manager.get_device()); - CHECK_EXPECTED_AS_STATUS(hw_consts); - const bool should_optimize_credits = hw_consts->should_optimize_credits && + TRY(const auto hw_consts, Control::get_hw_consts(resources_manager.get_device())); + const bool should_optimize_credits = hw_consts.should_optimize_credits && (HAILO_POWER_MODE_PERFORMANCE == resources_manager.get_power_mode()); for (const auto &layer_info : core_op_metadata->get_output_layer_infos()){ - auto status = fill_boundary_output_layer(context_resources, resources_manager, layer_info, *hw_consts, + auto status = fill_boundary_output_layer(context_resources, resources_manager, layer_info, hw_consts, hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } for (const auto &layer_info : core_op_metadata->get_input_layer_infos()) { - auto status = fill_boundary_input_layer(context_resources, resources_manager, layer_info, *hw_consts, + auto status = fill_boundary_input_layer(context_resources, resources_manager, layer_info, hw_consts, hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } std::vector actions; - auto reset_burst_task_action = ResetBurstCreditsTaskAction::create(); - CHECK_EXPECTED_AS_STATUS(reset_burst_task_action); - actions.emplace_back(reset_burst_task_action.release()); + TRY(const auto reset_burst_task_action, ResetBurstCreditsTaskAction::create()); + actions.emplace_back(std::move(reset_burst_task_action)); for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY)) { auto action = edge_layer.layer_info.direction == HAILO_H2D_STREAM ? OpenBoundaryInputChannelAction::create(edge_layer.channel_id, edge_layer.buffer_info) : OpenBoundaryOutputChannelAction::create(edge_layer.channel_id, edge_layer.buffer_info); - CHECK_EXPECTED_AS_STATUS(action); + CHECK_EXPECTED_AS_STATUS(action); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here actions.emplace_back(action.release()); } @@ -1039,23 +1098,22 @@ static Expected create_switch_lcu_batch_action(con (ContextSwitchConfigAction::Type::EnableLcuNonDefault == action->get_type()), HAILO_INVALID_ARGUMENT, "Invalid action type - must be enable lcu (default or non default) or switch lcu batch, Received type {}", action->get_type()); - const auto params_buffer = action->serialize_params(context_resources); - CHECK_EXPECTED(params_buffer); + TRY(const auto params_buffer, action->serialize_params(context_resources)); if (ContextSwitchConfigAction::Type::EnableLcuDefault == action->get_type()) { - const auto params = reinterpret_cast(params_buffer.value().data()); + const auto params = reinterpret_cast(params_buffer.data()); cluster_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_CLUSTER_INDEX_READ(params->packed_lcu_id); lcu_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_LCU_INDEX_READ(params->packed_lcu_id); network_index = params->network_index; kernel_done_count = CONTEXT_SWITCH_DEFS__ENABLE_LCU_DEFAULT_KERNEL_COUNT; } else if (ContextSwitchConfigAction::Type::EnableLcuNonDefault == action->get_type()) { - const auto params = reinterpret_cast(params_buffer.value().data()); + const auto params = reinterpret_cast(params_buffer.data()); cluster_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_CLUSTER_INDEX_READ(params->packed_lcu_id); lcu_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_LCU_INDEX_READ(params->packed_lcu_id); network_index = params->network_index; kernel_done_count = params->kernel_done_count; } else if (ContextSwitchConfigAction::Type::SwitchLcuBatch == action->get_type()) { - const auto params = reinterpret_cast(params_buffer.value().data()); + const auto params = reinterpret_cast(params_buffer.data()); cluster_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_CLUSTER_INDEX_READ(params->packed_lcu_id); lcu_index = CONTEXT_SWITCH_DEFS__PACKED_LCU_ID_LCU_INDEX_READ(params->packed_lcu_id); network_index = params->network_index; @@ -1068,36 +1126,35 @@ static Expected create_switch_lcu_batch_action(con static hailo_status fill_batch_switching_context_edge_layers(ContextResources &context_resources, const CoreOpMetadata &core_op_metadata, ResourcesManager &resources_manager, const HEFHwArch &hw_arch) { - auto hw_consts = Control::get_hw_consts(resources_manager.get_device()); - CHECK_EXPECTED_AS_STATUS(hw_consts); - const bool should_optimize_credits = hw_consts->should_optimize_credits && + TRY(const auto hw_consts, Control::get_hw_consts(resources_manager.get_device())); + const bool should_optimize_credits = hw_consts.should_optimize_credits && (HAILO_POWER_MODE_PERFORMANCE == resources_manager.get_power_mode()); for (const auto &output_layer_info : core_op_metadata.dynamic_contexts()[0].get_ddr_output_layers()) { - auto status = fill_ddr_output_layer(context_resources, resources_manager, output_layer_info, *hw_consts, hw_arch); + auto status = fill_ddr_output_layer(context_resources, resources_manager, output_layer_info, hw_consts, hw_arch); CHECK_SUCCESS(status); } for (const auto &output_layer_info : core_op_metadata.dynamic_contexts()[0].get_boundary_output_layers()) { auto status = fill_boundary_output_layer(context_resources, resources_manager, output_layer_info, - *hw_consts, hw_arch, should_optimize_credits); + hw_consts, hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } for (const auto &output_layer_info : core_op_metadata.dynamic_contexts()[0].get_inter_context_output_layers()) { auto status = fill_inter_context_output_layer(context_resources, resources_manager, output_layer_info, - *hw_consts, hw_arch, should_optimize_credits); + hw_consts, hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } for (const auto &input_layer_info : core_op_metadata.dynamic_contexts()[0].get_ddr_input_layers()) { - auto status = fill_ddr_input_layer(context_resources, resources_manager, input_layer_info, *hw_consts, hw_arch); + auto status = fill_ddr_input_layer(context_resources, resources_manager, input_layer_info, hw_consts, hw_arch); CHECK_SUCCESS(status); } for (const auto &input_layer_info : core_op_metadata.dynamic_contexts()[0].get_boundary_input_layers()) { auto status = fill_boundary_input_layer(context_resources, resources_manager, input_layer_info, - *hw_consts, hw_arch, should_optimize_credits); + hw_consts, hw_arch, should_optimize_credits); CHECK_SUCCESS(status); } @@ -1121,9 +1178,8 @@ static hailo_status add_lcu_actions_to_batch_switch_context(ContextResources &co const auto lcu_batch_switch_actions = core_op_metadata.preliminary_context().get_actions_of_type(ENABLE_LCU_ACTIONS); for (const auto &action : lcu_batch_switch_actions) { - auto switch_lcu_batch_action = create_switch_lcu_batch_action(action, context_resources); - CHECK_EXPECTED_AS_STATUS(switch_lcu_batch_action); - actions.insert(actions.end(), switch_lcu_batch_action.release()); + TRY(const auto switch_lcu_batch_action, create_switch_lcu_batch_action(action, context_resources)); + actions.insert(actions.end(), std::move(switch_lcu_batch_action)); } return HAILO_SUCCESS; @@ -1133,14 +1189,12 @@ static hailo_status create_change_boundary_input_batch_actions(const ContextReso std::vector &batch_switch_context_actions) { for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) { - auto change_boundary_input_batch_action = ChangeBoundaryInputBatchAction::create(edge_layer.channel_id); - CHECK_EXPECTED_AS_STATUS(change_boundary_input_batch_action); - batch_switch_context_actions.emplace_back(change_boundary_input_batch_action.release()); + TRY(const auto change_boundary_input_batch_action, ChangeBoundaryInputBatchAction::create(edge_layer.channel_id)); + batch_switch_context_actions.emplace_back(std::move(change_boundary_input_batch_action)); } - auto start_burst_credits_task_action = StartBurstCreditsTaskAction::create(); - CHECK_EXPECTED_AS_STATUS(start_burst_credits_task_action); - batch_switch_context_actions.emplace_back(start_burst_credits_task_action.release()); + TRY(const auto start_burst_credits_task_action, StartBurstCreditsTaskAction::create()); + batch_switch_context_actions.emplace_back(std::move(start_burst_credits_task_action)); return HAILO_SUCCESS; @@ -1156,16 +1210,14 @@ static hailo_status add_edge_layers_actions_to_batch_switch_context(ContextResou const auto BATCH_SWITCHING_CONTEXT = true; for (const auto &edge_layer : context_resources.get_edge_layers()) { if (edge_layer.layer_info.type != LayerType::BOUNDARY) { - auto action = DeactivateChannelAction::create(edge_layer, BATCH_SWITCHING_CONTEXT); - CHECK_EXPECTED_AS_STATUS(action); - actions.emplace_back(action.release()); + TRY(const auto action, DeactivateChannelAction::create(edge_layer, BATCH_SWITCHING_CONTEXT)); + actions.emplace_back(std::move(action)); } } // We need to reset the ddr buffering task when we change the batch_size (since it depends on the batch_size param) - auto reset_ddr_action = ResetDdrBufferingTaskAction::create(); - CHECK_EXPECTED_AS_STATUS(reset_ddr_action); - actions.emplace_back(reset_ddr_action.release()); + TRY(const auto reset_ddr_action, ResetDdrBufferingTaskAction::create()); + actions.emplace_back(std::move(reset_ddr_action)); // Now re-open all the internal channels const bool PUSH_INTERNAL_EDGE_LAYERS = true; @@ -1237,8 +1289,8 @@ static hailo_status fill_preliminary_config_recepies_for_multi_context(const HEF } Expected> ResourcesManagerBuilder::build(uint8_t current_core_op_index, VdmaDevice &device, - HailoRTDriver &driver, const ConfigureNetworkParams &config_params, - std::shared_ptr core_op_metadata, const HEFHwArch &hw_arch, std::shared_ptr shef_file_handle) + HailoRTDriver &driver, CacheManagerPtr cache_manager, const ConfigureNetworkParams &config_params, + std::shared_ptr core_op_metadata, const HEFHwArch &hw_arch) { const auto num_contexts = core_op_metadata->dynamic_contexts().size() + CONTROL_PROTOCOL__CONTEXT_SWITCH_NUMBER_OF_NON_DYNAMIC_CONTEXTS; @@ -1252,79 +1304,66 @@ Expected> ResourcesManagerBuilder::build(uint8 core_op_metadata->core_op_name(), network_params.first, HAILO_MAX_BATCH_SIZE); } - auto resources_manager = ResourcesManager::create(device, driver, config_params, core_op_metadata, - current_core_op_index); - CHECK_EXPECTED(resources_manager); + TRY(auto resources_manager, ResourcesManager::create(device, driver, config_params, cache_manager, + core_op_metadata, current_core_op_index)); // TODO: Use a new flag in config_params.stream_params_by_name to mark channels as async channels. // will also used to mark streams as async in ConfiguredNetworkGroupBase::create_in/output_stream_from_config_params // (HRT-9104) - auto status = create_boundary_channels(resources_manager.value(), *core_op_metadata); + auto status = create_boundary_channels(resources_manager, *core_op_metadata); CHECK_SUCCESS_AS_EXPECTED(status); - status = resources_manager->fill_internal_buffers_info(); + status = resources_manager.fill_internal_buffers_info(); CHECK_SUCCESS_AS_EXPECTED(status); // No allocation of edge layers in the activation context. No need for context index here auto INVLID_CONTEXT_INDEX = static_cast(UINT16_MAX); auto ACTIVATION_CONTEXT_INDEX = INVLID_CONTEXT_INDEX; - auto activation_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION, - ACTIVATION_CONTEXT_INDEX); - CHECK_EXPECTED(activation_context); - status = fill_activation_config_recepies_for_multi_context(activation_context.value().get(), - resources_manager.value(), core_op_metadata, hw_arch); + TRY(auto activation_context, resources_manager.add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION, + ACTIVATION_CONTEXT_INDEX)); + status = fill_activation_config_recepies_for_multi_context(activation_context.get(), + resources_manager, core_op_metadata, hw_arch); CHECK_SUCCESS_AS_EXPECTED(status); // No allocation of edge layers in the batch switching context. No need for context index here auto BATCH_SWITCH_CONTEXT_INDEX = INVLID_CONTEXT_INDEX; - auto batch_switching_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING, - BATCH_SWITCH_CONTEXT_INDEX); - CHECK_EXPECTED(batch_switching_context); - status = fill_batch_switching_context_config_recepies_for_multi_context(batch_switching_context.value().get(), - *core_op_metadata, resources_manager.value(), hw_arch); + TRY(auto batch_switching_context, resources_manager.add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING, + BATCH_SWITCH_CONTEXT_INDEX)); + status = fill_batch_switching_context_config_recepies_for_multi_context(batch_switching_context.get(), + *core_op_metadata, resources_manager, hw_arch); CHECK_SUCCESS_AS_EXPECTED(status); - const bool is_single_context = core_op_metadata->dynamic_contexts().size() == 1; - - if (nullptr != shef_file_handle) { - // We will start reading CCWs from the HEF file so we need to open it - status = shef_file_handle->open(); - CHECK_SUCCESS_AS_EXPECTED(status); - } - - auto PRELIMINARY_CONTEXT_INDEX = static_cast(0); - auto preliminary_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY, - PRELIMINARY_CONTEXT_INDEX, core_op_metadata->preliminary_context().config_buffers_info()); - CHECK_EXPECTED(preliminary_context); - status = fill_preliminary_config_recepies_for_multi_context(hw_arch, preliminary_context.value().get(), - resources_manager.value(), core_op_metadata, core_op_metadata->preliminary_context(), is_single_context); + static const uint16_t PRELIMINARY_CONTEXT_INDEX = 0; + static const uint16_t FIRST_DYNAMIC_CONTEXT_INDEX = 1; + const auto is_single_context = (core_op_metadata->dynamic_contexts().size() == 1); + TRY(auto preliminary_context, resources_manager.add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY, + PRELIMINARY_CONTEXT_INDEX, core_op_metadata->preliminary_context().config_buffers_info())); + status = fill_preliminary_config_recepies_for_multi_context(hw_arch, preliminary_context.get(), + resources_manager, core_op_metadata, core_op_metadata->preliminary_context(), is_single_context); CHECK_SUCCESS_AS_EXPECTED(status); - uint16_t context_index = 0; - auto FIRST_DYNAMIC_CONTEXT_INDEX = 1; - for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) { - auto new_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC, - static_cast(FIRST_DYNAMIC_CONTEXT_INDEX + context_index), context_metadata.config_buffers_info()); - CHECK_EXPECTED(new_context); + const auto caches_in_use = core_op_metadata->get_cache_layers_count() > 0; + CHECK_AS_EXPECTED(!caches_in_use || !is_single_context, HAILO_INVALID_ARGUMENT, + "Caches are in use but the network is single context"); - status = fill_context_recipes_for_multi_context(hw_arch, new_context.value().get(), resources_manager.value(), - context_index, *core_op_metadata, - context_metadata, is_single_context); - CHECK_SUCCESS_AS_EXPECTED(status); - - context_index++; - } + const auto num_dynamic_contexts = core_op_metadata->dynamic_contexts().size(); + for (size_t context_index = 0; context_index < num_dynamic_contexts; context_index++) { + const auto &context_metadata = core_op_metadata->dynamic_contexts()[context_index]; + TRY(auto new_context, resources_manager.add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC, + static_cast(FIRST_DYNAMIC_CONTEXT_INDEX + context_index), context_metadata.config_buffers_info())); - if (nullptr != shef_file_handle) { - status = shef_file_handle->close(); + const auto is_last_context = (context_index == (num_dynamic_contexts - 1)); + status = fill_context_recipes_for_multi_context(hw_arch, new_context.get(), resources_manager, + static_cast(context_index), *core_op_metadata, context_metadata, is_single_context, + is_last_context, caches_in_use); CHECK_SUCCESS_AS_EXPECTED(status); } - status = resources_manager->configure(); + status = resources_manager.configure(); CHECK_SUCCESS_AS_EXPECTED(status); - auto resources_manager_ptr = make_shared_nothrow(resources_manager.release()); + auto resources_manager_ptr = make_shared_nothrow(std::move(resources_manager)); CHECK_NOT_NULL_AS_EXPECTED(resources_manager_ptr, HAILO_OUT_OF_HOST_MEMORY); return resources_manager_ptr; diff --git a/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.hpp b/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.hpp index a97f955..8144f65 100644 --- a/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.hpp @@ -11,19 +11,20 @@ #define _HAILO_RESOURCE_MANAGER_BUILDER_HPP_ #include "core_op/resource_manager/resource_manager.hpp" +#include "core_op/resource_manager/cache_manager.hpp" namespace hailort { -class ShefFileHandle; +class Reader; class ResourcesManagerBuilder final { public: ResourcesManagerBuilder() = delete; static Expected> build(uint8_t net_group_index, VdmaDevice &device, - HailoRTDriver &driver, const ConfigureNetworkParams &config_params, - std::shared_ptr core_op, const HEFHwArch &hw_arch, std::shared_ptr shef_file_handle); + HailoRTDriver &driver, CacheManagerPtr cache_manager, const ConfigureNetworkParams &config_params, + std::shared_ptr core_op, const HEFHwArch &hw_arch); }; diff --git a/hailort/libhailort/src/device_common/control.cpp b/hailort/libhailort/src/device_common/control.cpp index ae9249b..16bbb11 100644 --- a/hailort/libhailort/src/device_common/control.cpp +++ b/hailort/libhailort/src/device_common/control.cpp @@ -109,8 +109,8 @@ Expected control__parse_identify_results(CONTROL_PROTOC return board_info; } -Expected control__parse_get_extended_device_information_results - (CONTROL_PROTOCOL__get_extended_device_information_response_t &get_extended_device_information_response) +Expected control__parse_get_extended_device_information_results( + const CONTROL_PROTOCOL__get_extended_device_information_response_t &get_extended_device_information_response) { uint8_t local_supported_features; hailo_extended_device_information_t device_info; @@ -2432,6 +2432,108 @@ exit: return status; } +hailo_status Control::context_switch_init_cache_info(Device &device, const CONTROL_PROTOCOL__context_switch_cache_info_t &cache_info) +{ + CONTROL_PROTOCOL__request_t request{}; + size_t request_size = 0; + uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; + size_t response_size = RESPONSE_MAX_BUFFER_SIZE; + CONTROL_PROTOCOL__response_header_t *header = NULL; + CONTROL_PROTOCOL__payload_t *payload = NULL; + + const auto common_status = CONTROL_PROTOCOL__pack_context_switch_init_cache_info_request(&request, &request_size, + device.get_control_sequence(), cache_info.cache_size, cache_info.current_read_offset, cache_info.write_offset_delta); + auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + CHECK_SUCCESS(status); + + status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size); + CHECK_SUCCESS(status); + + /* Parse response */ + status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload, + &request, device); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +Expected Control::context_switch_get_cache_info(Device &device) +{ + CONTROL_PROTOCOL__request_t request{}; + size_t request_size = 0; + uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; + size_t response_size = RESPONSE_MAX_BUFFER_SIZE; + CONTROL_PROTOCOL__response_header_t *header = NULL; + CONTROL_PROTOCOL__payload_t *payload = NULL; + + const auto common_status = CONTROL_PROTOCOL__pack_context_switch_get_cache_info_request(&request, &request_size, + device.get_control_sequence()); + auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + CHECK_SUCCESS_AS_EXPECTED(status); + + status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size); + CHECK_SUCCESS_AS_EXPECTED(status); + + /* Parse response */ + status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload, + &request, device); + CHECK_SUCCESS_AS_EXPECTED(status); + + CONTROL_PROTOCOL__context_switch_cache_info_t result{}; + result = *reinterpret_cast(payload->parameters); + return result; +} + +hailo_status Control::context_switch_update_cache_read_offset(Device &device, int32_t read_offset_delta) +{ + CONTROL_PROTOCOL__request_t request{}; + size_t request_size = 0; + uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; + size_t response_size = RESPONSE_MAX_BUFFER_SIZE; + CONTROL_PROTOCOL__response_header_t *header = NULL; + CONTROL_PROTOCOL__payload_t *payload = NULL; + + const auto common_status = CONTROL_PROTOCOL__pack_context_switch_update_cache_read_offset_request(&request, &request_size, + device.get_control_sequence(), read_offset_delta); + auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + CHECK_SUCCESS(status); + + status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size); + CHECK_SUCCESS(status); + + /* Parse response */ + status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload, + &request, device); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +hailo_status Control::context_switch_signal_cache_updated(Device &device) +{ + CONTROL_PROTOCOL__request_t request{}; + size_t request_size = 0; + uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; + size_t response_size = RESPONSE_MAX_BUFFER_SIZE; + CONTROL_PROTOCOL__response_header_t *header = NULL; + CONTROL_PROTOCOL__payload_t *payload = NULL; + + const auto common_status = CONTROL_PROTOCOL__pack_context_switch_signal_cache_updated_request(&request, &request_size, + device.get_control_sequence()); + auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + CHECK_SUCCESS(status); + + status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size); + CHECK_SUCCESS(status); + + /* Parse response */ + status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload, + &request, device); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + hailo_status Control::context_switch_set_context_info(Device &device, const std::vector &context_infos) { @@ -3097,17 +3199,15 @@ Expected Control::get_partial_clusters_layout_bitmap(Device &device) // Partial clusters layout is only relevant in HAILO_ARCH_HAILO8L and HAILO_ARCH_HAILO15M arch return Expected(PARTIAL_CLUSTERS_LAYOUT_IGNORE); } else { - auto extended_device_info_response = get_extended_device_info_response(device); - CHECK_EXPECTED(extended_device_info_response); - return BYTE_ORDER__ntohl(extended_device_info_response->partial_clusters_layout_bitmap); + TRY(const auto extended_device_info_response, get_extended_device_info_response(device)); + return BYTE_ORDER__ntohl(extended_device_info_response.partial_clusters_layout_bitmap); } } Expected Control::get_extended_device_information(Device &device) { - auto extended_device_info_response = get_extended_device_info_response(device); - CHECK_EXPECTED(extended_device_info_response); - return control__parse_get_extended_device_information_results(extended_device_info_response.value()); + TRY(const auto extended_device_info_response, get_extended_device_info_response(device)); + return control__parse_get_extended_device_information_results(extended_device_info_response); } Expected Control::get_health_information(Device &device) diff --git a/hailort/libhailort/src/device_common/control.hpp b/hailort/libhailort/src/device_common/control.hpp index 6b0ed3d..eb4f1ba 100644 --- a/hailort/libhailort/src/device_common/control.hpp +++ b/hailort/libhailort/src/device_common/control.hpp @@ -346,6 +346,10 @@ public: const std::vector &context_infos); static hailo_status context_switch_set_network_group_header(Device &device, const CONTROL_PROTOCOL__application_header_t &network_group_header); + static hailo_status context_switch_init_cache_info(Device &device, const CONTROL_PROTOCOL__context_switch_cache_info_t &cache_info); + static Expected context_switch_get_cache_info(Device &device); + static hailo_status context_switch_update_cache_read_offset(Device &device, int32_t read_offset_delta); + static hailo_status context_switch_signal_cache_updated(Device &device); static hailo_status wd_enable(Device &device, uint8_t cpu_id, bool should_enable); static hailo_status wd_config(Device &device, uint8_t cpu_id, uint32_t wd_cycles, CONTROL_PROTOCOL__WATCHDOG_MODE_t wd_mode); static hailo_status previous_system_state(Device &device, uint8_t cpu_id, CONTROL_PROTOCOL__system_state_t *system_state); diff --git a/hailort/libhailort/src/device_common/control_protocol.cpp b/hailort/libhailort/src/device_common/control_protocol.cpp index af0ad08..7d34146 100644 --- a/hailort/libhailort/src/device_common/control_protocol.cpp +++ b/hailort/libhailort/src/device_common/control_protocol.cpp @@ -1716,6 +1716,73 @@ exit: return status; } +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_init_cache_info_request( + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, + uint32_t cache_size, uint32_t current_read_offset, int32_t write_offset_delta) +{ + HAILO_COMMON_STATUS_t status = HAILO_COMMON_STATUS__UNINITIALIZED; + size_t local_request_size = 0; + + if ((NULL == request) || (NULL == request_size)) { + status = HAILO_STATUS__CONTROL_PROTOCOL__NULL_ARGUMENT_PASSED; + goto exit; + } + + /* Header */ + local_request_size = CONTROL_PROTOCOL__REQUEST_BASE_SIZE + sizeof(CONTROL_PROTOCOL__context_switch_init_cache_info_request_t); + control_protocol__pack_request_header(request, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_INIT_CACHE_INFO, 1); + + /* Body */ + request->parameters.context_switch_init_cache_info_request.cache_info_length = + BYTE_ORDER__htonl(sizeof(request->parameters.context_switch_init_cache_info_request.cache_info)); + request->parameters.context_switch_init_cache_info_request.cache_info.cache_size = cache_size; + request->parameters.context_switch_init_cache_info_request.cache_info.current_read_offset = current_read_offset; + request->parameters.context_switch_init_cache_info_request.cache_info.write_offset_delta = write_offset_delta; + + *request_size = local_request_size; + status = HAILO_COMMON_STATUS__SUCCESS; +exit: + return status; +} + +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_get_cache_info_request( + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence) +{ + return control_protocol__pack_empty_request(request, request_size, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_GET_CACHE_INFO); +} + +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_update_cache_read_offset_request( + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, int32_t read_offset_delta) +{ + HAILO_COMMON_STATUS_t status = HAILO_COMMON_STATUS__UNINITIALIZED; + size_t local_request_size = 0; + + if ((NULL == request) || (NULL == request_size)) { + status = HAILO_STATUS__CONTROL_PROTOCOL__NULL_ARGUMENT_PASSED; + goto exit; + } + + /* Header */ + local_request_size = CONTROL_PROTOCOL__REQUEST_BASE_SIZE + sizeof(CONTROL_PROTOCOL__context_switch_update_cache_read_offset_request_t); + control_protocol__pack_request_header(request, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_UPDATE_CACHE_READ_OFFSET, 1); + + /* read_offset_delta */ + request->parameters.context_switch_update_cache_read_offset_request.read_offset_delta_length = + BYTE_ORDER__htonl(sizeof(read_offset_delta)); + request->parameters.context_switch_update_cache_read_offset_request.read_offset_delta = read_offset_delta; + + *request_size = local_request_size; + status = HAILO_COMMON_STATUS__SUCCESS; +exit: + return status; +} + +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_signal_cache_updated_request( + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence) +{ + return control_protocol__pack_empty_request(request, request_size, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_SIGNAL_CACHE_UPDATED); +} + HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_idle_time_get_measuremment_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence) diff --git a/hailort/libhailort/src/device_common/control_protocol.hpp b/hailort/libhailort/src/device_common/control_protocol.hpp index 5fb914f..eff5a23 100644 --- a/hailort/libhailort/src/device_common/control_protocol.hpp +++ b/hailort/libhailort/src/device_common/control_protocol.hpp @@ -95,8 +95,16 @@ HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_sensor_get_sections_info_request(CO HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_set_network_group_header_request( CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, const CONTROL_PROTOCOL__application_header_t *network_group_header); +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_init_cache_info_request( + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, uint32_t cache_size, + uint32_t current_read_offset, int32_t write_offset_delta); +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_get_cache_info_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence); +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_update_cache_read_offset_request( + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, int32_t read_offset_delta); +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_signal_cache_updated_request( + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence); HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_set_context_info_request( - CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, const CONTROL_PROTOCOL__context_switch_context_info_chunk_t *context_info); HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_idle_time_set_measuremment_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, uint8_t measurement_enable); HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_idle_time_get_measuremment_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence); diff --git a/hailort/libhailort/src/device_common/d2h_events_parser.cpp b/hailort/libhailort/src/device_common/d2h_events_parser.cpp index e60f7c3..dda4121 100644 --- a/hailort/libhailort/src/device_common/d2h_events_parser.cpp +++ b/hailort/libhailort/src/device_common/d2h_events_parser.cpp @@ -49,6 +49,7 @@ static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_context_switch_breakpoint_reached static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_health_monitor_clock_changed_event_notification(D2H_EVENT_MESSAGE_t *d2h_notification_message); static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_hw_infer_manager_infer_done_notification(D2H_EVENT_MESSAGE_t *d2h_notification_message); static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_context_switch_run_time_error_notification(D2H_EVENT_MESSAGE_t *d2h_notification_message); +static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_start_update_cache_offset(D2H_EVENT_MESSAGE_t *d2h_notification_message); /********************************************************************** * Globals @@ -66,7 +67,8 @@ firmware_notifications_parser_t g_firmware_notifications_parser[D2H_EVENT_ID_COU D2H_EVENTS__parse_context_switch_breakpoint_reached, D2H_EVENTS__parse_health_monitor_clock_changed_event_notification, D2H_EVENTS__parse_hw_infer_manager_infer_done_notification, - D2H_EVENTS__parse_context_switch_run_time_error_notification + D2H_EVENTS__parse_context_switch_run_time_error_notification, + D2H_EVENTS__parse_start_update_cache_offset, }; /********************************************************************** * Internal Functions @@ -203,6 +205,24 @@ l_exit: return status; } +static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_start_update_cache_offset(D2H_EVENT_MESSAGE_t *d2h_notification_message) +{ + if (D2H_EVENT_START_UPDATE_CACHE_OFFSET_PARAMETER_COUNT != d2h_notification_message->header.parameter_count) { + LOGGER__ERROR("d2h notification invalid parameter count: {}", d2h_notification_message->header.parameter_count); + return HAILO_STATUS__D2H_EVENTS__INCORRECT_PARAMETER_COUNT; + } + + if(d2h_notification_message->header.payload_length != sizeof(d2h_notification_message->message_parameters.start_update_cache_offset_event)) { + LOGGER__ERROR("d2h notification invalid payload_length: {}", d2h_notification_message->header.payload_length); + return HAILO_STATUS__D2H_EVENTS__INCORRECT_PARAMETER_LENGTH; + } + + LOGGER__TRACE("Got can update cache offset notification - cache_id_bitmask={}", + d2h_notification_message->message_parameters.start_update_cache_offset_event.cache_id_bitmask); + + return HAILO_COMMON_STATUS__SUCCESS; +} + static HAILO_COMMON_STATUS_t D2H_EVENTS__parse_health_monitor_closed_streams_notification(D2H_EVENT_MESSAGE_t *d2h_notification_message) { HAILO_COMMON_STATUS_t status = HAILO_COMMON_STATUS__UNINITIALIZED; diff --git a/hailort/libhailort/src/device_common/device.cpp b/hailort/libhailort/src/device_common/device.cpp index 7122459..97c92a5 100644 --- a/hailort/libhailort/src/device_common/device.cpp +++ b/hailort/libhailort/src/device_common/device.cpp @@ -58,16 +58,14 @@ Expected> Device::scan() return std::vector{IntegratedDevice::DEVICE_ID}; } else { - auto pcie_device_infos = PcieDevice::scan(); - CHECK_EXPECTED(pcie_device_infos); + TRY(auto pcie_device_infos, PcieDevice::scan()); std::vector results; - results.reserve(pcie_device_infos->size()); + results.reserve(pcie_device_infos.size()); - for (const auto pcie_device_info : pcie_device_infos.release()) { - auto device_id = pcie_device_info_to_string(pcie_device_info); - CHECK_EXPECTED(device_id); - results.emplace_back(device_id.release()); + for (const auto pcie_device_info : pcie_device_infos) { + TRY(auto device_id, pcie_device_info_to_string(pcie_device_info)); + results.emplace_back(device_id); } return results; @@ -93,12 +91,12 @@ Expected> Device::scan_eth_by_host_address( Expected> Device::create() { - auto device_ids = scan(); - CHECK_EXPECTED(device_ids, "Failed scan devices"); - CHECK_AS_EXPECTED(device_ids->size() >= 1, HAILO_INVALID_OPERATION, "There is no hailo device on the system"); + TRY(const auto device_ids, scan(), "Failed scan devices"); + CHECK_AS_EXPECTED(device_ids.size() >= 1, HAILO_INVALID_OPERATION, + "There is no hailo device on the system"); // Choose the first device. - return Device::create(device_ids->at(0)); + return Device::create(device_ids.at(0)); } Expected> Device::create(const std::string &device_id) @@ -121,37 +119,33 @@ Expected> Device::create(const std::string &device_id) Expected> Device::create_pcie() { - auto pcie_device = PcieDevice::create(); - CHECK_EXPECTED(pcie_device); + TRY(auto pcie_device, PcieDevice::create()); // Upcasting to Device unique_ptr (from PcieDevice unique_ptr) - auto device = std::unique_ptr(pcie_device.release()); + auto device = std::unique_ptr(std::move(pcie_device)); return device; } Expected> Device::create_pcie(const hailo_pcie_device_info_t &device_info) { - auto pcie_device = PcieDevice::create(device_info); - CHECK_EXPECTED(pcie_device); + TRY(auto pcie_device, PcieDevice::create(device_info)); // Upcasting to Device unique_ptr (from PcieDevice unique_ptr) - auto device = std::unique_ptr(pcie_device.release()); + auto device = std::unique_ptr(std::move(pcie_device)); return device; } Expected> Device::create_eth(const hailo_eth_device_info_t &device_info) { - auto eth_device = EthernetDevice::create(device_info); - CHECK_EXPECTED(eth_device); + TRY(auto eth_device, EthernetDevice::create(device_info)); // Upcasting to Device unique_ptr (from EthernetDevice unique_ptr) - auto device = std::unique_ptr(eth_device.release()); + auto device = std::unique_ptr(std::move(eth_device)); return device; } Expected> Device::create_eth(const std::string &ip_addr) { - auto eth_device = EthernetDevice::create(ip_addr); - CHECK_EXPECTED(eth_device); + TRY(auto eth_device, EthernetDevice::create(ip_addr)); // Upcasting to Device unique_ptr (from EthernetDevice unique_ptr) - auto device = std::unique_ptr(eth_device.release()); + auto device = std::unique_ptr(std::move(eth_device)); return device; } @@ -403,6 +397,22 @@ hailo_status Device::dma_unmap(void *address, size_t size, hailo_dma_buffer_dire return HAILO_NOT_IMPLEMENTED; } +hailo_status Device::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + (void) dmabuf_fd; + (void) size; + (void) direction; + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status Device::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + (void) dmabuf_fd; + (void) size; + (void) direction; + return HAILO_NOT_IMPLEMENTED; +} + hailo_status Device::direct_write_memory(uint32_t address, const void *buffer, uint32_t size) { (void) address; @@ -442,9 +452,7 @@ Expected Device::get_extended_device_inform hailo_status Device::update_fw_state() { // Assuming FW is loaded, send identify - auto board_info_expected = Control::identify(*this); - CHECK_EXPECTED_AS_STATUS(board_info_expected); - hailo_device_identity_t board_info = board_info_expected.release(); + TRY(auto board_info, Control::identify(*this)); if ((FIRMWARE_VERSION_MAJOR == board_info.fw_version.major) && (FIRMWARE_VERSION_MINOR == board_info.fw_version.minor)) { @@ -537,27 +545,25 @@ Expected Device::download_context_action_list(uint32_t network_group_id, CHECK_ARG_NOT_NULL_AS_EXPECTED(batch_counter); // Allocate room for an action list of at most max_size bytes - auto action_list = Buffer::create(max_size); - CHECK_EXPECTED(action_list); + TRY(auto action_list, Buffer::create(max_size)); uint32_t base_address_local = 0; uint32_t batch_counter_local = 0; uint16_t actual_size = 0; const auto status = Control::download_context_action_list(*this, network_group_id, - (CONTROL_PROTOCOL__context_switch_context_type_t)context_type, context_index, action_list->size(), - &base_address_local, action_list->data(), &actual_size, &batch_counter_local); + (CONTROL_PROTOCOL__context_switch_context_type_t)context_type, context_index, action_list.size(), + &base_address_local, action_list.data(), &actual_size, &batch_counter_local); CHECK_SUCCESS_AS_EXPECTED(status); CHECK_AS_EXPECTED(actual_size <= max_size, HAILO_INTERNAL_FAILURE); // Create a copy of the list, truncating to the needed size - auto final_action_list = Buffer::create(action_list->data(), actual_size); - CHECK_EXPECTED(action_list); + TRY(auto final_action_list, Buffer::create(action_list.data(), actual_size)); // Transfer ownership of out params *base_address = base_address_local; *batch_counter = batch_counter_local; - return final_action_list.release(); + return final_action_list; } hailo_status Device::set_context_action_list_timestamp_batch(uint16_t batch_index) @@ -621,29 +627,46 @@ Expected Device::get_context_switch_breakpoint_status(uint8_t breakpoin return static_cast(breakpoint_status); } +// TODO: remove init_cache_info, get_cache_info and update_cache_read_offset (HRT-13896) +hailo_status Device::init_cache_info(const hailo_cache_info_t &cache_info) +{ + CONTROL_PROTOCOL__context_switch_cache_info_t control_cache_info = { + cache_info.cache_size, + cache_info.current_read_offset, + cache_info.write_offset_delta + }; + return Control::context_switch_init_cache_info(*this, control_cache_info); +} + +Expected Device::get_cache_info() +{ + TRY(const auto cache_info, Control::context_switch_get_cache_info(*this)); + return hailo_cache_info_t{cache_info.cache_size, cache_info.current_read_offset, cache_info.write_offset_delta}; +} + +hailo_status Device::update_cache_read_offset(int32_t read_offset_delta) +{ + return Control::context_switch_update_cache_read_offset(*this, read_offset_delta); +} + Expected> Device::create_core() { - auto integrated_device = IntegratedDevice::create(); - CHECK_EXPECTED(integrated_device); + TRY(auto integrated_device, IntegratedDevice::create()); // Upcasting to Device unique_ptr (from IntegratedDevice unique_ptr) - auto device = std::unique_ptr(integrated_device.release()); + auto device = std::unique_ptr(std::move(integrated_device)); return device; } Expected Device::create_configure_params(Hef &hef) const { - auto stream_interface = get_default_streams_interface(); - CHECK_EXPECTED(stream_interface, "Failed to get default streams interface"); - - return hef.create_configure_params(stream_interface.release()); + TRY(const auto stream_interface, get_default_streams_interface(), "Failed to get default streams interface"); + return hef.create_configure_params(stream_interface); } Expected Device::create_configure_params(Hef &hef, const std::string &network_group_name) const { - auto stream_interface = get_default_streams_interface(); - CHECK_EXPECTED(stream_interface, "Failed to get default streams interface"); - - return hef.create_configure_params(stream_interface.release(), network_group_name); + TRY(const auto stream_interface, get_default_streams_interface(), "Failed to get default streams interface"); + return hef.create_configure_params(stream_interface, network_group_name); } } /* namespace hailort */ diff --git a/hailort/libhailort/src/device_common/device_internal.cpp b/hailort/libhailort/src/device_common/device_internal.cpp index 1fa6d72..525abae 100644 --- a/hailort/libhailort/src/device_common/device_internal.cpp +++ b/hailort/libhailort/src/device_common/device_internal.cpp @@ -51,8 +51,7 @@ Expected DeviceBase::configure(Hef &hef, auto status = check_hef_is_compatible(hef); CHECK_SUCCESS_AS_EXPECTED(status); - auto network_groups = add_hef(hef, configure_params); - CHECK_EXPECTED(network_groups); + TRY(auto network_groups, add_hef(hef, configure_params)); auto elapsed_time_ms = std::chrono::duration(std::chrono::steady_clock::now() - start_time).count(); LOGGER__INFO("Configuring HEF took {} milliseconds", elapsed_time_ms); @@ -161,20 +160,19 @@ void DeviceBase::notification_fetch_thread(std::shared_ptr DeviceBase::get_fw_type() { firmware_type_t firmware_type; - const auto architecture = get_architecture(); - CHECK_EXPECTED(architecture); + TRY(const auto architecture, get_architecture()); - if ((architecture.value() == HAILO_ARCH_HAILO8) || (architecture.value() == HAILO_ARCH_HAILO8L)) { + if ((architecture == HAILO_ARCH_HAILO8) || (architecture == HAILO_ARCH_HAILO8L)) { firmware_type = FIRMWARE_TYPE_HAILO8; } - else if ((architecture.value() == HAILO_ARCH_HAILO15H ) || (architecture.value() == HAILO_ARCH_HAILO15M)) { + else if ((architecture == HAILO_ARCH_HAILO15H ) || (architecture == HAILO_ARCH_HAILO15M)) { firmware_type = FIRMWARE_TYPE_HAILO15; } - else if (architecture.value() == HAILO_ARCH_PLUTO) { + else if (architecture == HAILO_ARCH_PLUTO) { firmware_type = FIRMWARE_TYPE_PLUTO; } else { - LOGGER__ERROR("Invalid device arcitecture. {}", architecture.value()); + LOGGER__ERROR("Invalid device arcitecture. {}", architecture); return make_unexpected(HAILO_INVALID_DEVICE_ARCHITECTURE); } @@ -199,18 +197,16 @@ hailo_status DeviceBase::firmware_update(const MemoryView &firmware_binary, bool MD5_Update(&md5_ctx, firmware_binary.data(), firmware_binary.size()); MD5_Final(md5_sum, &md5_ctx); - const auto firmware_type = get_fw_type(); - CHECK_EXPECTED_AS_STATUS(firmware_type); + TRY(const auto firmware_type, get_fw_type()); - fw_header_status = FIRMWARE_HEADER_UTILS__validate_fw_headers((uintptr_t) firmware_binary.data(), static_cast(firmware_binary.size()), false, - &new_app_firmware_header, &new_core_firmware_header, NULL, firmware_type.value()); + fw_header_status = FIRMWARE_HEADER_UTILS__validate_fw_headers((uintptr_t)firmware_binary.data(), + static_cast(firmware_binary.size()), false, &new_app_firmware_header, + &new_core_firmware_header, NULL, firmware_type); CHECK(HAILO_COMMON_STATUS__SUCCESS == fw_header_status, HAILO_INVALID_FIRMWARE, "FW update validation failed with status {}", fw_header_status); // TODO: Are we ok with doing another identify here? - auto board_info_before_update_expected = Control::identify(*this); - CHECK_EXPECTED_AS_STATUS(board_info_before_update_expected); - hailo_device_identity_t board_info_before_update = board_info_before_update_expected.release(); + TRY(auto board_info_before_update, Control::identify(*this)); if (board_info_before_update.device_architecture != HAILO_ARCH_HAILO8_A0) { if ((new_app_firmware_header->firmware_major != new_core_firmware_header->firmware_major) || @@ -294,7 +290,7 @@ hailo_status DeviceBase::firmware_update(const MemoryView &firmware_binary, bool return HAILO_SUCCESS; } - CHECK_EXPECTED_AS_STATUS(board_info_after_install_expected); + CHECK_EXPECTED_AS_STATUS(board_info_after_install_expected); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here hailo_device_identity_t board_info_after_install = board_info_after_install_expected.release(); LOGGER__INFO("New App FW version: {}.{}.{}{}", board_info_after_install.fw_version.major, board_info_after_install.fw_version.minor, @@ -343,11 +339,10 @@ hailo_status DeviceBase::second_stage_update(uint8_t* second_stage_binary, uint3 MD5_Update(&md5_ctx, second_stage_binary, second_stage_binary_length); MD5_Final(md5_sum, &md5_ctx); - const auto firmware_type = get_fw_type(); - CHECK_EXPECTED_AS_STATUS(firmware_type); + TRY(const auto firmware_type, get_fw_type()); - second_stage_header_status = FIRMWARE_HEADER_UTILS__validate_second_stage_headers((uintptr_t) second_stage_binary, - second_stage_binary_length, &new_second_stage_header, firmware_type.value()); + second_stage_header_status = FIRMWARE_HEADER_UTILS__validate_second_stage_headers((uintptr_t)second_stage_binary, + second_stage_binary_length, &new_second_stage_header, firmware_type); CHECK(HAILO_COMMON_STATUS__SUCCESS == second_stage_header_status, HAILO_INVALID_SECOND_STAGE, "Second stage update validation failed with status {}", second_stage_header_status); @@ -392,30 +387,26 @@ hailo_status DeviceBase::store_sensor_config(uint32_t section_index, hailo_senso CHECK(sensor_type != HAILO_SENSOR_TYPES_HAILO8_ISP, HAILO_INVALID_ARGUMENT, "store_sensor_config intended only for sensor config, for ISP config use store_isp"); - auto control_buffers = SensorConfigUtils::read_config_file(config_file_path); - CHECK_EXPECTED_AS_STATUS(control_buffers, "Failed reading config file"); - - return store_sensor_control_buffers(control_buffers.value(), section_index, sensor_type, + TRY(auto control_buffers, SensorConfigUtils::read_config_file(config_file_path), "Failed reading config file"); + return store_sensor_control_buffers(control_buffers, section_index, sensor_type, reset_config_size, config_height, config_width, config_fps, config_name); } hailo_status DeviceBase::store_isp_config(uint32_t reset_config_size, uint16_t config_height, uint16_t config_width, uint16_t config_fps, const std::string &isp_static_config_file_path, const std::string &isp_runtime_config_file_path, const std::string &config_name) { - auto control_buffers = SensorConfigUtils::read_isp_config_file(isp_static_config_file_path, isp_runtime_config_file_path); - CHECK_EXPECTED_AS_STATUS(control_buffers, "Failed reading ISP config file"); - - return store_sensor_control_buffers(control_buffers.value(), SENSOR_CONFIG__ISP_SECTION_INDEX, HAILO_SENSOR_TYPES_HAILO8_ISP, + TRY(auto control_buffers, SensorConfigUtils::read_isp_config_file(isp_static_config_file_path, isp_runtime_config_file_path), + "Failed reading ISP config file"); + return store_sensor_control_buffers(control_buffers, SENSOR_CONFIG__ISP_SECTION_INDEX, HAILO_SENSOR_TYPES_HAILO8_ISP, reset_config_size, config_height, config_width, config_fps, config_name); } Expected DeviceBase::sensor_get_sections_info() { - auto buffer = Buffer::create(SENSOR_SECTIONS_INFO_SIZE); - CHECK_EXPECTED(buffer); + TRY(auto buffer, Buffer::create(SENSOR_SECTIONS_INFO_SIZE)); - hailo_status status = Control::sensor_get_sections_info(*this, buffer->data()); + hailo_status status = Control::sensor_get_sections_info(*this, buffer.data()); CHECK_SUCCESS_AS_EXPECTED(status); return buffer; @@ -424,16 +415,14 @@ Expected DeviceBase::sensor_get_sections_info() hailo_status DeviceBase::sensor_dump_config(uint32_t section_index, const std::string &config_file_path) { CHECK(SENSOR_CONFIG__TOTAL_SECTIONS_BLOCK_COUNT > section_index, HAILO_INVALID_ARGUMENT, "Section {} is invalid. Section index must be in the range [0 - {}]", section_index, (SENSOR_CONFIG__TOTAL_SECTIONS_BLOCK_COUNT - 1)); - auto sections_info_buffer = sensor_get_sections_info(); - CHECK_EXPECTED_AS_STATUS(sections_info_buffer); + TRY(auto sections_info_buffer, sensor_get_sections_info()); - SENSOR_CONFIG__section_info_t *section_info_ptr = &((SENSOR_CONFIG__section_info_t *)sections_info_buffer->data())[section_index]; + SENSOR_CONFIG__section_info_t *section_info_ptr = &((SENSOR_CONFIG__section_info_t *)sections_info_buffer.data())[section_index]; CHECK(section_info_ptr->is_free == 0, HAILO_NOT_FOUND, "Section {} is not active", section_index); CHECK(0 == (section_info_ptr->config_size % sizeof(SENSOR_CONFIG__operation_cfg_t)), HAILO_INVALID_OPERATION, "Section config size is invalid."); /* Read config data from device */ - auto operation_cfg = Buffer::create(section_info_ptr->config_size); - CHECK_EXPECTED_AS_STATUS(operation_cfg); + TRY(auto operation_cfg, Buffer::create(section_info_ptr->config_size)); size_t read_full_buffer_count = (section_info_ptr->config_size / MAX_CONFIG_ENTRIES_DATA_SIZE); uint32_t residue_to_read = static_cast(section_info_ptr->config_size - (read_full_buffer_count * MAX_CONFIG_ENTRIES_DATA_SIZE)); @@ -442,16 +431,17 @@ hailo_status DeviceBase::sensor_dump_config(uint32_t section_index, const std::s hailo_status status = HAILO_UNINITIALIZED; for (uint32_t i = 0; i < read_full_buffer_count; i++) { - status = Control::sensor_get_config(*this, section_index, offset, (uint32_t)MAX_CONFIG_ENTRIES_DATA_SIZE, (operation_cfg->data() + offset)); + status = Control::sensor_get_config(*this, section_index, offset, + (uint32_t)MAX_CONFIG_ENTRIES_DATA_SIZE, (operation_cfg.data() + offset)); CHECK_SUCCESS(status); offset += static_cast(MAX_CONFIG_ENTRIES_DATA_SIZE); } if (0 < residue_to_read) { - status = Control::sensor_get_config(*this, section_index, offset, residue_to_read, (operation_cfg->data() + offset)); + status = Control::sensor_get_config(*this, section_index, offset, residue_to_read, (operation_cfg.data() + offset)); CHECK_SUCCESS(status); } - status = SensorConfigUtils::dump_config_to_csv((SENSOR_CONFIG__operation_cfg_t*)operation_cfg->data(), config_file_path, entries_count); + status = SensorConfigUtils::dump_config_to_csv((SENSOR_CONFIG__operation_cfg_t*)operation_cfg.data(), config_file_path, entries_count); CHECK_SUCCESS(status); return HAILO_SUCCESS; @@ -486,13 +476,11 @@ hailo_status DeviceBase::sensor_set_generic_i2c_slave(uint16_t slave_address, ui Expected DeviceBase::read_board_config() { - auto result = Buffer::create(BOARD_CONFIG_SIZE, 0); - CHECK_EXPECTED(result); - - auto status = Control::read_board_config(*this, result->data(), static_cast(result->size())); + TRY(auto result, Buffer::create(BOARD_CONFIG_SIZE, 0)); + auto status = Control::read_board_config(*this, result.data(), static_cast(result.size())); CHECK_SUCCESS_AS_EXPECTED(status); - return result.release(); + return result; } hailo_status DeviceBase::write_board_config(const MemoryView &buffer) @@ -511,16 +499,13 @@ Expected DeviceBase::examine_user_config() Expected DeviceBase::read_user_config() { - auto user_config_info = examine_user_config(); - CHECK_EXPECTED(user_config_info, "Failed to examine user config"); + TRY(auto user_config_info, examine_user_config(), "Failed to examine user config"); + TRY(auto result, Buffer::create(user_config_info.total_size, 0)); - auto result = Buffer::create(user_config_info->total_size, 0); - CHECK_EXPECTED(result); - - auto status = Control::read_user_config(*this, result->data(), static_cast(result->size())); + auto status = Control::read_user_config(*this, result.data(), static_cast(result.size())); CHECK_SUCCESS_AS_EXPECTED(status); - return result.release(); + return result; } hailo_status DeviceBase::write_user_config(const MemoryView &buffer) @@ -607,11 +592,10 @@ void DeviceBase::d2h_notification_thread_main(const std::string &device_id) hailo_status DeviceBase::check_hef_is_compatible(Hef &hef) { - const auto device_arch = get_architecture(); - CHECK_EXPECTED_AS_STATUS(device_arch, "Can't get device architecture (is the FW loaded?)"); + TRY(const auto device_arch, get_architecture(), "Can't get device architecture (is the FW loaded?)"); - if (!is_hef_compatible(device_arch.value(), static_cast(hef.pimpl->get_device_arch()))) { - auto device_arch_str = HailoRTCommon::get_device_arch_str(device_arch.value()); + if (!is_hef_compatible(device_arch, static_cast(hef.pimpl->get_device_arch()))) { + auto device_arch_str = HailoRTCommon::get_device_arch_str(device_arch); auto hef_arch_str = HailoRTCommon::get_device_arch_str(hef_arch_to_device_arch(static_cast(hef.pimpl->get_device_arch()))); @@ -621,20 +605,18 @@ hailo_status DeviceBase::check_hef_is_compatible(Hef &hef) } // TODO: MSW-227 check clock rate for hailo15 as well. - if ((HAILO_ARCH_HAILO8 == device_arch.value()) || (HAILO_ARCH_HAILO8L == device_arch.value())) { - auto extended_device_info_expected = Control::get_extended_device_information(*this); - CHECK_EXPECTED_AS_STATUS(extended_device_info_expected, "Can't get device extended info"); - hailo_extended_device_information_t extended_device_information = extended_device_info_expected.release(); + if ((HAILO_ARCH_HAILO8 == device_arch) || (HAILO_ARCH_HAILO8L == device_arch)) { + TRY(auto extended_device_information, Control::get_extended_device_information(*this), "Can't get device extended info"); check_clock_rate_for_hailo8(extended_device_information.neural_network_core_clock_rate, static_cast(hef.pimpl->get_device_arch())); } if ((static_cast(HEFHwArch::HW_ARCH__HAILO8L) == hef.pimpl->get_device_arch()) && - (HAILO_ARCH_HAILO8 == device_arch.value())) { + (HAILO_ARCH_HAILO8 == device_arch)) { LOGGER__WARNING("HEF was compiled for Hailo8L device, while the device itself is Hailo8. " \ "This will result in lower performance."); } else if ((static_cast(HEFHwArch::HW_ARCH__HAILO15M) == hef.pimpl->get_device_arch()) && - (HAILO_ARCH_HAILO15H == device_arch.value())) { + (HAILO_ARCH_HAILO15H == device_arch)) { LOGGER__WARNING("HEF was compiled for Hailo15M device, while the device itself is Hailo15H. " \ "This will result in lower performance."); } @@ -687,6 +669,9 @@ hailo_status DeviceBase::fw_notification_id_to_hailo(D2H_EVENT_ID_t fw_notificat case CONTEXT_SWITCH_RUN_TIME_ERROR: *hailo_notification_id = HAILO_NOTIFICATION_ID_CONTEXT_SWITCH_RUN_TIME_ERROR_EVENT; break; + case START_UPDATE_CACHE_OFFSET_ID: + *hailo_notification_id = HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET; + break; default: status = HAILO_INVALID_ARGUMENT; goto l_exit; @@ -736,9 +721,10 @@ bool DeviceBase::is_hef_compatible(hailo_device_architecture_t device_arch, HEFH case HAILO_ARCH_HAILO8L: return (hef_arch == HEFHwArch::HW_ARCH__HAILO8L); case HAILO_ARCH_HAILO15H: + case HAILO_ARCH_HAILO10H: // Compare with HW_ARCH__LAVENDER and HW_ARCH__GINGER to support hefs compiled for them return (hef_arch == HEFHwArch::HW_ARCH__GINGER) || (hef_arch == HEFHwArch::HW_ARCH__LAVENDER) || - (hef_arch == HEFHwArch::HW_ARCH__HAILO15H) || (hef_arch == HEFHwArch::HW_ARCH__HAILO15M); + (hef_arch == HEFHwArch::HW_ARCH__HAILO15H) || (hef_arch == HEFHwArch::HW_ARCH__HAILO15M) || (hef_arch == HEFHwArch::HW_ARCH__HAILO10H); case HAILO_ARCH_PLUTO: return (hef_arch == HEFHwArch::HW_ARCH__PLUTO); case HAILO_ARCH_HAILO15M: @@ -769,6 +755,8 @@ hailo_device_architecture_t DeviceBase::hef_arch_to_device_arch(HEFHwArch hef_ar return HAILO_ARCH_PLUTO; case HEFHwArch::HW_ARCH__HAILO15M: return HAILO_ARCH_HAILO15M; + case HEFHwArch::HW_ARCH__HAILO10H: + return HAILO_ARCH_HAILO10H; default: return HAILO_ARCH_MAX_ENUM; diff --git a/hailort/libhailort/src/device_common/device_internal.hpp b/hailort/libhailort/src/device_common/device_internal.hpp index 29ac623..7a689ee 100644 --- a/hailort/libhailort/src/device_common/device_internal.hpp +++ b/hailort/libhailort/src/device_common/device_internal.hpp @@ -50,6 +50,7 @@ enum class HEFHwArch // Must be aligned to ProtoHEFHwArch HW_ARCH__HAILO8L = 3, HW_ARCH__HAILO15H = 103, HW_ARCH__HAILO15M = 4, + HW_ARCH__HAILO10H = 5, HW_ARCH__SAGE_A0 = 100, HW_ARCH__SAGE_B0 = 101, diff --git a/hailort/libhailort/src/eth/eth_device.cpp b/hailort/libhailort/src/eth/eth_device.cpp index c7685a5..f3f3ecb 100644 --- a/hailort/libhailort/src/eth/eth_device.cpp +++ b/hailort/libhailort/src/eth/eth_device.cpp @@ -57,14 +57,13 @@ hailo_status EthernetDevice::wait_for_wakeup() CONTROL_PROTOCOL__payload_t *payload = NULL; /* Create udp socket */ - auto udp = Udp::create(m_device_info.device_address.sin_addr, m_device_info.device_address.sin_port, - m_device_info.host_address.sin_addr, m_device_info.host_address.sin_port); - CHECK_EXPECTED_AS_STATUS(udp); + TRY(auto udp, Udp::create(m_device_info.device_address.sin_addr, m_device_info.device_address.sin_port, + m_device_info.host_address.sin_addr, m_device_info.host_address.sin_port)); - status = udp->set_timeout(std::chrono::milliseconds(WAIT_FOR_DEVICE_WAKEUP_TIMEOUT)); + status = udp.set_timeout(std::chrono::milliseconds(WAIT_FOR_DEVICE_WAKEUP_TIMEOUT)); CHECK_SUCCESS(status); - status = udp->set_max_number_of_attempts(WAIT_FOR_DEVICE_WAKEUP_MAX_ATTEMPTS); + status = udp.set_max_number_of_attempts(WAIT_FOR_DEVICE_WAKEUP_MAX_ATTEMPTS); CHECK_SUCCESS(status); /* Create and send identify-control until it runs successfully */ @@ -72,7 +71,7 @@ hailo_status EthernetDevice::wait_for_wakeup() status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS(status); - status = udp->fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size, + status = udp.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size, m_control_sequence); // Always increment sequence @@ -89,17 +88,13 @@ Expected> EthernetDevice::create(const hailo_eth hailo_status status = HAILO_UNINITIALIZED; // Creates control socket - auto udp = Udp::create(device_info.device_address.sin_addr, device_info.device_address.sin_port, - device_info.host_address.sin_addr, device_info.host_address.sin_port); - CHECK_EXPECTED(udp, "Failed to init control socket."); + TRY(auto udp, Udp::create(device_info.device_address.sin_addr, device_info.device_address.sin_port, + device_info.host_address.sin_addr, device_info.host_address.sin_port), "Failed to init control socket."); - auto device = std::unique_ptr(new (std::nothrow) EthernetDevice(device_info, udp.release(), status)); + auto device = std::unique_ptr(new (std::nothrow) EthernetDevice(device_info, std::move(udp), status)); CHECK_AS_EXPECTED((nullptr != device), HAILO_OUT_OF_HOST_MEMORY); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed creating EthernetDevice"); - return make_unexpected(status); - } + CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating EthernetDevice"); return device; } @@ -107,9 +102,9 @@ Expected> EthernetDevice::create(const hailo_eth Expected> EthernetDevice::create(const std::string &ip_addr) { const bool LOG_ON_FAILURE = true; - auto device_info = parse_eth_device_info(ip_addr, LOG_ON_FAILURE); - CHECK_EXPECTED(device_info, "Failed to parse ip address {}", ip_addr); - return create(device_info.release()); + TRY(const auto device_info, parse_eth_device_info(ip_addr, LOG_ON_FAILURE), + "Failed to parse ip address {}", ip_addr); + return create(device_info); } EthernetDevice::EthernetDevice(const hailo_eth_device_info_t &device_info, Udp &&control_udp, hailo_status &status) : @@ -215,10 +210,8 @@ Expected> EthernetDevice::scan(const std::s std::chrono::milliseconds timeout) { // Convert interface name to IP address - auto interface_ip_address = EthernetUtils::get_ip_from_interface(interface_name); - CHECK_EXPECTED(interface_ip_address); - - return scan_by_host_address(*interface_ip_address, timeout); + TRY(const auto interface_ip_address, EthernetUtils::get_ip_from_interface(interface_name)); + return scan_by_host_address(interface_ip_address, timeout); } hailo_status get_udp_broadcast_params(const char *host_address, struct in_addr &interface_ip_address, @@ -249,9 +242,8 @@ Expected> EthernetDevice::scan_by_host_addr CHECK_SUCCESS_AS_EXPECTED(status); /* Create broadcast udp object */ - auto udp_broadcast = Udp::create(broadcast_ip_address, HAILO_DEFAULT_ETH_CONTROL_PORT, interface_ip_address, 0); - CHECK_EXPECTED(udp_broadcast); - status = udp_broadcast->set_timeout(timeout); + TRY(auto udp_broadcast, Udp::create(broadcast_ip_address, HAILO_DEFAULT_ETH_CONTROL_PORT, interface_ip_address, 0)); + status = udp_broadcast.set_timeout(timeout); CHECK_SUCCESS_AS_EXPECTED(status); /* Build identify request */ @@ -260,11 +252,11 @@ Expected> EthernetDevice::scan_by_host_addr CHECK_SUCCESS_AS_EXPECTED(status); /* Send broadcast identify request */ - status = udp_broadcast->send((uint8_t *)&request, &request_size, false, MAX_UDP_PAYLOAD_SIZE); + status = udp_broadcast.send((uint8_t *)&request, &request_size, false, MAX_UDP_PAYLOAD_SIZE); CHECK_SUCCESS_AS_EXPECTED(status); /* Receive all responses */ - return eth_device__receive_responses(*udp_broadcast); + return eth_device__receive_responses(udp_broadcast); } Expected EthernetDevice::parse_eth_device_info(const std::string &ip_addr, @@ -398,17 +390,14 @@ Expected EthernetDevice::add_hef(Hef &hef, const N auto status = Control::reset_context_switch_state_machine(*this); CHECK_SUCCESS_AS_EXPECTED(status); - auto added_network_groups = create_networks_group_vector(hef, configure_params); - CHECK_EXPECTED(added_network_groups); + TRY(auto added_network_groups, create_networks_group_vector(hef, configure_params)); return added_network_groups; } Expected EthernetDevice::create_networks_group_vector(Hef &hef, const NetworkGroupsParamsMap &configure_params) { - auto partial_clusters_layout_bitmap_exp = Control::get_partial_clusters_layout_bitmap(*this); - CHECK_EXPECTED(partial_clusters_layout_bitmap_exp); - auto partial_clusters_layout_bitmap = partial_clusters_layout_bitmap_exp.release(); + TRY(auto partial_clusters_layout_bitmap, Control::get_partial_clusters_layout_bitmap(*this)); auto &hef_network_groups = hef.pimpl->network_groups(); auto configure_params_copy = configure_params; @@ -426,27 +415,22 @@ Expected EthernetDevice::create_networks_group_vec config_params = configure_params_copy.at(network_group_name); configure_params_copy.erase(network_group_name); } else if (configure_params.empty()) { - auto config_params_exp = create_configure_params(hef, network_group_name); - CHECK_EXPECTED(config_params_exp); - config_params = config_params_exp.release(); + TRY(config_params, create_configure_params(hef, network_group_name)); } else { continue; } - auto net_group_config = create_core_op_metadata(hef, network_group_name, partial_clusters_layout_bitmap); - CHECK_EXPECTED(net_group_config); + TRY(auto net_group_config, create_core_op_metadata(hef, network_group_name, partial_clusters_layout_bitmap)); // TODO: move to func, support multiple core ops std::vector> core_ops_ptrs; - auto core_op_metadata = hef.pimpl->get_core_op_metadata(network_group_name); - CHECK_EXPECTED(core_op_metadata); - auto core_op_metadata_ptr = core_op_metadata.release(); + TRY(auto core_op_metadata_ptr, hef.pimpl->get_core_op_metadata(network_group_name)); auto metadata = hef.pimpl->network_group_metadata(core_op_metadata_ptr->core_op_name()); auto status = HAILO_UNINITIALIZED; - auto single_context_app = HcpConfigCoreOp(*this, m_active_core_op_holder, net_group_config.release(), + auto single_context_app = HcpConfigCoreOp(*this, m_active_core_op_holder, std::move(net_group_config), config_params, core_op_metadata_ptr, status); CHECK_SUCCESS_AS_EXPECTED(status); @@ -463,9 +447,8 @@ Expected EthernetDevice::create_networks_group_vec m_core_ops.push_back(core_op_ptr); core_ops_ptrs.push_back(core_op_ptr); - auto net_group_expected = ConfiguredNetworkGroupBase::create(config_params, std::move(core_ops_ptrs), std::move(metadata)); - CHECK_EXPECTED(net_group_expected); - auto net_group_ptr = net_group_expected.release(); + TRY(auto net_group_ptr, ConfiguredNetworkGroupBase::create(config_params, + std::move(core_ops_ptrs), std::move(metadata))); added_network_groups.emplace_back(net_group_ptr); m_network_groups.push_back(net_group_ptr); @@ -484,19 +467,15 @@ Expected EthernetDevice::create_networks_group_vec Expected> EthernetDevice::create_core_op_metadata(Hef &hef, const std::string &core_op_name, uint32_t partial_clusters_layout_bitmap) { - auto device_arch_exp = get_architecture(); - CHECK_EXPECTED(device_arch_exp); - auto device_arch = device_arch_exp.release(); + TRY(const auto device_arch, get_architecture()); auto hef_arch = hef.pimpl->get_device_arch(); auto &hef_core_ops = hef.pimpl->core_ops(core_op_name); assert(1 == hef_core_ops.size()); const auto &core_op = hef_core_ops[0]; - auto expected_partial_core_op = Hef::Impl::get_core_op_per_arch(core_op, hef_arch, device_arch, - partial_clusters_layout_bitmap); - CHECK_EXPECTED(expected_partial_core_op); - auto partial_core_op = expected_partial_core_op.release(); + TRY(auto partial_core_op, Hef::Impl::get_core_op_per_arch(core_op, hef_arch, device_arch, + partial_clusters_layout_bitmap)); // TODO: decide about core_op names - align with the Compiler @@ -512,8 +491,7 @@ Expected> EthernetDevice::create_core_op_metadata(H /* Update preliminary_config and dynamic_contexts recepies */ auto &proto_preliminary_config = partial_core_op->preliminary_config; - auto core_op_config = Hef::Impl::create_single_context_core_op_config(proto_preliminary_config); - CHECK_EXPECTED(core_op_config); + TRY(auto core_op_config, Hef::Impl::create_single_context_core_op_config(proto_preliminary_config)); return core_op_config; } diff --git a/hailort/libhailort/src/eth/eth_stream.cpp b/hailort/libhailort/src/eth/eth_stream.cpp index b84db33..92d80eb 100644 --- a/hailort/libhailort/src/eth/eth_stream.cpp +++ b/hailort/libhailort/src/eth/eth_stream.cpp @@ -190,22 +190,15 @@ hailo_status EthernetInputStream::eth_stream__write_with_remainder(const void *b while (offset < offset_end_without_remainder) { transfer_size = offset_end_without_remainder - offset; - auto expected_bytes_written = sync_write_raw_buffer(MemoryView::create_const(static_cast(buffer) + offset, transfer_size)); - if (HAILO_STREAM_ABORT == expected_bytes_written.status()) { - LOGGER__INFO("sync_write_raw_buffer was aborted!"); - return expected_bytes_written.status(); - } - CHECK_EXPECTED_AS_STATUS(expected_bytes_written); - offset += expected_bytes_written.release(); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto bytes_written, + sync_write_raw_buffer(MemoryView::create_const(static_cast(buffer) + offset, transfer_size))); + offset += bytes_written; } if (0 < remainder_size) { - auto expected_bytes_written = sync_write_raw_buffer(MemoryView::create_const(static_cast(buffer) + offset, remainder_size)); - if (HAILO_STREAM_ABORT == expected_bytes_written.status()) { - LOGGER__INFO("sync_write_raw_buffer was aborted!"); - return expected_bytes_written.status(); - } - CHECK_EXPECTED_AS_STATUS(expected_bytes_written); - assert(expected_bytes_written.value() == remainder_size); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto bytes_written, + sync_write_raw_buffer(MemoryView::create_const(static_cast(buffer) + offset, remainder_size))); + (void)bytes_written; + assert(bytes_written == remainder_size); } return HAILO_SUCCESS; @@ -235,26 +228,19 @@ hailo_status TokenBucketEthernetInputStream::eth_stream__write_with_remainder(co (void)token_bucket.consumeWithBorrowAndWait(MAX_CONSUME_SIZE, rate_bytes_per_sec, BURST_SIZE); transfer_size = offset_end_without_remainder - offset; - auto expected_bytes_written = sync_write_raw_buffer(MemoryView::create_const(static_cast(buffer) + offset, transfer_size)); - if (HAILO_STREAM_ABORT == expected_bytes_written.status()) { - LOGGER__INFO("sync_write_raw_buffer was aborted!"); - return expected_bytes_written.status(); - } - CHECK_EXPECTED_AS_STATUS(expected_bytes_written); - offset += expected_bytes_written.release(); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto bytes_written, + sync_write_raw_buffer(MemoryView::create_const(static_cast(buffer) + offset, transfer_size))); + offset += bytes_written; } if (0 < remainder_size) { // We don't static_assert that "remainder_size <= BURST_SIZE", so the call could fail in theory. // However, since remainder_size is modulo MAX_UDP_PAYLOAD_SIZE and BURST_SIZE == MAX_UDP_PAYLOAD_SIZE, it should be smaller. (void)token_bucket.consumeWithBorrowAndWait(static_cast(remainder_size), rate_bytes_per_sec, BURST_SIZE); - - auto expected_bytes_written = sync_write_raw_buffer(MemoryView::create_const(static_cast(buffer) + offset, remainder_size)); - if (HAILO_STREAM_ABORT == expected_bytes_written.status()) { - LOGGER__INFO("sync_write_raw_buffer was aborted!"); - return expected_bytes_written.status(); - } - CHECK_EXPECTED_AS_STATUS(expected_bytes_written); - assert(expected_bytes_written.value() == remainder_size); + + TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto bytes_written, + sync_write_raw_buffer(MemoryView::create_const(static_cast(buffer) + offset, remainder_size))); + (void)bytes_written; + assert(bytes_written == remainder_size); } return HAILO_SUCCESS; @@ -264,19 +250,17 @@ hailo_status TokenBucketEthernetInputStream::eth_stream__write_with_remainder(co Expected> TrafficControlEthernetInputStream::create( Device &device, Udp &&udp, EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info) { - auto board_ip = get_interface_address(&udp.m_device_address.sin_addr); - CHECK_EXPECTED(board_ip, "get_interface_address failed with status {}", board_ip.status()); - + TRY(const auto board_ip, get_interface_address(&udp.m_device_address.sin_addr)); const auto board_port = BYTE_ORDER__ntohs(udp.m_device_address.sin_port); - auto tc = TrafficControl::create(board_ip.value(), board_port, rate_bytes_per_sec); - CHECK_EXPECTED(tc, "Creating traffic control at rate {} failed with error {}", rate_bytes_per_sec, tc.status()); + TRY(auto tc, TrafficControl::create(board_ip, board_port, rate_bytes_per_sec), + "Creating traffic control at rate {} failed", rate_bytes_per_sec); auto status = HAILO_UNINITIALIZED; // Note: we don't use make_unique because TrafficControlEthernetInputStream's ctor is private auto tc_ptr = std::unique_ptr(new (std::nothrow) TrafficControlEthernetInputStream(device, std::move(udp), std::move(core_op_activated_event), rate_bytes_per_sec, - tc.release(), layer_info, status)); + std::move(tc), layer_info, status)); CHECK_AS_EXPECTED(nullptr != tc_ptr, HAILO_OUT_OF_HOST_MEMORY); CHECK_SUCCESS_AS_EXPECTED(status); return tc_ptr; @@ -284,13 +268,12 @@ Expected> TrafficControlEther Expected TrafficControlEthernetInputStream::get_interface_address(const struct in_addr *addr) { - auto ip = Buffer::create(IPV4_STRING_MAX_LENGTH, 0); - CHECK_EXPECTED(ip); + TRY(const auto ip, Buffer::create(IPV4_STRING_MAX_LENGTH, 0)); - const auto result = Socket::ntop(AF_INET, addr, ip->as_pointer(), EthernetUtils::MAX_INTERFACE_SIZE); + const auto result = Socket::ntop(AF_INET, addr, ip.as_pointer(), EthernetUtils::MAX_INTERFACE_SIZE); CHECK_SUCCESS_AS_EXPECTED(result, "Failed parsing IP to string with status {}", result); - return ip->to_string(); + return ip.to_string(); } TrafficControlEthernetInputStream::TrafficControlEthernetInputStream(Device &device, Udp &&udp, @@ -370,34 +353,30 @@ Expected> EthernetInputStream::create(Devic std::unique_ptr local_stream; auto stream_index = edge_layer.stream_index; - auto udp = eth_stream__create_udp(eth_device, params.host_address, stream_index, params.device_port, true); - CHECK_EXPECTED(udp); + TRY(auto udp, eth_stream__create_udp(eth_device, params.host_address, stream_index, params.device_port, true)); if (params.rate_limit_bytes_per_sec == 0) { local_stream = std::unique_ptr( - new (std::nothrow) EthernetInputStream(device, udp.release(), std::move(core_op_activated_event), edge_layer, status)); + new (std::nothrow) EthernetInputStream(device, std::move(udp), std::move(core_op_activated_event), edge_layer, status)); CHECK_SUCCESS_AS_EXPECTED(status); } else { #ifdef _MSC_VER // TODO: Add factory class local_stream = std::unique_ptr( - new (std::nothrow) TokenBucketEthernetInputStream(device, udp.release(), + new (std::nothrow) TokenBucketEthernetInputStream(device, std::move(udp), std::move(core_op_activated_event), params.rate_limit_bytes_per_sec, edge_layer, status)); CHECK_SUCCESS_AS_EXPECTED(status); #else - auto stream_expected = TrafficControlEthernetInputStream::create(device, udp.release(), - std::move(core_op_activated_event), params.rate_limit_bytes_per_sec, edge_layer); - CHECK_EXPECTED(stream_expected); - local_stream = stream_expected.release(); + TRY(local_stream, TrafficControlEthernetInputStream::create(device, std::move(udp), + std::move(core_op_activated_event), params.rate_limit_bytes_per_sec, edge_layer)); #endif } CHECK_AS_EXPECTED((nullptr != local_stream), HAILO_OUT_OF_HOST_MEMORY); local_stream->m_is_stream_activated = false; - auto device_architecture = eth_device->get_architecture(); - CHECK_EXPECTED(device_architecture); - if ((HAILO_ARCH_HAILO8 == device_architecture.value()) || (HAILO_ARCH_HAILO8L == device_architecture.value())) { + TRY(const auto device_architecture, eth_device->get_architecture()); + if ((HAILO_ARCH_HAILO8 == device_architecture) || (HAILO_ARCH_HAILO8L == device_architecture)) { local_stream->configuration.use_dataflow_padding = true; } else { @@ -514,13 +493,8 @@ hailo_status EthernetOutputStream::read_all_no_sync(void *buffer, size_t offset, while (offset < offset_end) { transfer_size = offset_end - offset; MemoryView buffer_view(static_cast(buffer) + offset, transfer_size); - auto expected_bytes_read = this->sync_read_raw_buffer(buffer_view); - if (HAILO_STREAM_ABORT == expected_bytes_read.status()) { - LOGGER__INFO("sync_read_raw_buffer was aborted!"); - return expected_bytes_read.status(); - } - CHECK_EXPECTED_AS_STATUS(expected_bytes_read); - offset += expected_bytes_read.release(); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, auto bytes_read, this->sync_read_raw_buffer(buffer_view)); + offset += bytes_read; } return HAILO_SUCCESS; @@ -608,7 +582,7 @@ hailo_status EthernetOutputStream::get_last_sync() { MemoryView leftover_buffer_view(this->leftover_buffer, last_packet_size); auto expected_bytes_read = sync_read_raw_buffer(leftover_buffer_view); CHECK(HAILO_TIMEOUT != expected_bytes_read.status(), HAILO_INVALID_FRAME, "Got timeout on last sync, marking last frame as invalid"); - CHECK_EXPECTED_AS_STATUS(expected_bytes_read, "Recv error"); + CHECK_EXPECTED_AS_STATUS(expected_bytes_read, "Recv error"); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here last_packet_size = expected_bytes_read.release(); if (is_sync_packet(this->leftover_buffer, 0, last_packet_size)) { @@ -710,11 +684,9 @@ Expected> EthernetOutputStream::create(Dev auto eth_device = reinterpret_cast(&device); const auto stream_index = edge_layer.stream_index; - auto udp = eth_stream__create_udp(eth_device, params.host_address, stream_index, params.device_port, false); - CHECK_EXPECTED(udp); + TRY(auto udp, eth_stream__create_udp(eth_device, params.host_address, stream_index, params.device_port, false)); local_stream = std::unique_ptr(new (std::nothrow) EthernetOutputStream(device, - edge_layer, - udp.release(), std::move(core_op_activated_event), status)); + edge_layer, std::move(udp), std::move(core_op_activated_event), status)); CHECK((nullptr != local_stream), make_unexpected(HAILO_OUT_OF_HOST_MEMORY)); CHECK_SUCCESS_AS_EXPECTED(status); diff --git a/hailort/libhailort/src/eth/hcp_config_core_op.cpp b/hailort/libhailort/src/eth/hcp_config_core_op.cpp index c410890..4a325a0 100644 --- a/hailort/libhailort/src/eth/hcp_config_core_op.cpp +++ b/hailort/libhailort/src/eth/hcp_config_core_op.cpp @@ -69,6 +69,45 @@ Expected HcpConfigCoreOp::run_hw_infer_estimator() return make_unexpected(HAILO_INVALID_OPERATION); } +bool HcpConfigCoreOp::has_caches() const +{ + return false; +} + +Expected HcpConfigCoreOp::get_cache_read_size() const +{ + LOGGER__ERROR("get_cache_read_size function is not supported on ETH core-ops"); + return make_unexpected(HAILO_INVALID_OPERATION); +} + +Expected HcpConfigCoreOp::get_cache_write_size() const +{ + LOGGER__ERROR("get_cache_write_size function is not supported on ETH core-ops"); + return make_unexpected(HAILO_INVALID_OPERATION); +} + + +hailo_status HcpConfigCoreOp::init_cache(uint32_t read_offset, int32_t write_offset_delta) +{ + (void) read_offset; + (void) write_offset_delta; + LOGGER__ERROR("init_cache function is not supported on ETH core-ops"); + return HAILO_INVALID_OPERATION; +} + +Expected HcpConfigCoreOp::get_cache_info() const +{ + LOGGER__ERROR("get_cache_info function is not supported on ETH core-ops"); + return make_unexpected(HAILO_INVALID_OPERATION); +} + +hailo_status HcpConfigCoreOp::update_cache_offset(int32_t offset_delta_bytes) +{ + (void) offset_delta_bytes; + LOGGER__ERROR("update_cache_offset function is not supported on ETH core-ops"); + return HAILO_INVALID_OPERATION; +} + hailo_status HcpConfigCoreOp::activate_impl(uint16_t /* dynamic_batch_size */) { // Close older dataflows diff --git a/hailort/libhailort/src/eth/hcp_config_core_op.hpp b/hailort/libhailort/src/eth/hcp_config_core_op.hpp index a273e96..13f9849 100644 --- a/hailort/libhailort/src/eth/hcp_config_core_op.hpp +++ b/hailort/libhailort/src/eth/hcp_config_core_op.hpp @@ -53,6 +53,12 @@ public: virtual hailo_status deactivate_impl() override; virtual hailo_status shutdown() override; virtual Expected run_hw_infer_estimator() override; + virtual bool has_caches() const override; + virtual Expected get_cache_read_size() const override; + virtual Expected get_cache_write_size() const override; + virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override; + virtual Expected get_cache_info() const override; + virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override; virtual ~HcpConfigCoreOp() = default; HcpConfigCoreOp(const HcpConfigCoreOp &other) = delete; diff --git a/hailort/libhailort/src/eth/network_rate_calculator.cpp b/hailort/libhailort/src/eth/network_rate_calculator.cpp index 76937d8..97b00e1 100644 --- a/hailort/libhailort/src/eth/network_rate_calculator.cpp +++ b/hailort/libhailort/src/eth/network_rate_calculator.cpp @@ -27,11 +27,10 @@ Expected NetworkUdpRateCalculator::get_streams_from_hef(Hef* h { assert(nullptr != hef); - auto all_streams_infos = hef->get_all_stream_infos(network_group_name); - CHECK_EXPECTED(all_streams_infos); + TRY(auto all_streams_infos, hef->get_all_stream_infos(network_group_name)); // We expect to have two or more streams (atleast one for input and one for output) - if (all_streams_infos->size() < 2) { + if (all_streams_infos.size() < 2) { return make_unexpected(HAILO_INVALID_HEF); } @@ -135,8 +134,7 @@ Expected> NetworkUdpRateCalculator::calculate_in Expected> NetworkUdpRateCalculator::get_udp_ports_rates_dict( std::vector> &udp_input_streams, uint32_t fps, uint32_t max_supported_bandwidth) { - auto rates_per_name = calculate_inputs_bandwith(fps, max_supported_bandwidth); - CHECK_EXPECTED(rates_per_name); + TRY(const auto rates_per_name, calculate_inputs_bandwith(fps, max_supported_bandwidth)); std::map results = {}; for (const auto &input_stream : udp_input_streams) { @@ -145,7 +143,7 @@ Expected> NetworkUdpRateCalculator::get_udp_ports_r "Invalid stream index {}", stream_index); const uint16_t remote_port = static_cast(stream_index + HailoRTCommon::ETH_INPUT_BASE_PORT); results.insert(std::make_pair(remote_port, - rates_per_name->at(input_stream.get().name()))); + rates_per_name.at(input_stream.get().name()))); } return results; @@ -154,9 +152,8 @@ Expected> NetworkUdpRateCalculator::get_udp_ports_r hailo_status NetworkUdpRateCalculator::set_rate_limit(const std::string &ip, uint16_t port, uint32_t rate_bytes_per_sec) { #if defined(__GNUC__) - auto tc = TrafficControlUtil::create(ip, port, rate_bytes_per_sec); - CHECK_EXPECTED_AS_STATUS(tc); - CHECK_SUCCESS(tc->set_rate_limit()); + TRY(auto tc, TrafficControlUtil::create(ip, port, rate_bytes_per_sec)); + CHECK_SUCCESS(tc.set_rate_limit()); return HAILO_SUCCESS; #else @@ -171,9 +168,8 @@ hailo_status NetworkUdpRateCalculator::set_rate_limit(const std::string &ip, uin hailo_status NetworkUdpRateCalculator::reset_rate_limit(const std::string &ip, uint16_t port) { #if defined(__GNUC__) - auto tc = TrafficControlUtil::create(ip, port, 0); - CHECK_EXPECTED_AS_STATUS(tc); - CHECK_SUCCESS(tc->reset_rate_limit()); + TRY(auto tc, TrafficControlUtil::create(ip, port, 0)); + CHECK_SUCCESS(tc.reset_rate_limit()); return HAILO_SUCCESS; #else diff --git a/hailort/libhailort/src/eth/udp.cpp b/hailort/libhailort/src/eth/udp.cpp index 1d820e4..8a21675 100644 --- a/hailort/libhailort/src/eth/udp.cpp +++ b/hailort/libhailort/src/eth/udp.cpp @@ -48,9 +48,8 @@ Expected Udp::create(struct in_addr device_ip, uint16_t device_port, struct uint16_t host_port) { auto status = HAILO_UNINITIALIZED; - auto socket = Socket::create(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - CHECK_EXPECTED(socket); - auto object = Udp(device_ip, device_port, host_ip, host_port, socket.release(), status); + TRY(auto socket, Socket::create(AF_INET, SOCK_DGRAM, IPPROTO_UDP)); + auto object = Udp(device_ip, device_port, host_ip, host_port, std::move(socket), status); CHECK_SUCCESS_AS_EXPECTED(status); return object; diff --git a/hailort/libhailort/src/hailort.cpp b/hailort/libhailort/src/hailort.cpp index 96b1ce2..2874da9 100644 --- a/hailort/libhailort/src/hailort.cpp +++ b/hailort/libhailort/src/hailort.cpp @@ -1116,7 +1116,7 @@ hailo_status hailo_free_buffer(void *buffer) // TODO: hailo_device_dma_map_buffer/hailo_device_dma_unmap_buffer aren't thread safe when crossed with // hailo_allocate_buffer/hailo_free_buffer (HRT-10669) -hailo_status hailo_device_dma_map_buffer(hailo_device device,void *address, size_t size, hailo_dma_buffer_direction_t direction) +hailo_status hailo_device_dma_map_buffer(hailo_device device, void *address, size_t size, hailo_dma_buffer_direction_t direction) { CHECK_ARG_NOT_NULL(device); CHECK_ARG_NOT_NULL(address); @@ -1130,7 +1130,7 @@ hailo_status hailo_device_dma_unmap_buffer(hailo_device device, void *address, s return reinterpret_cast(device)->dma_unmap(address, size, direction); } -hailo_status hailo_vdevice_dma_map_buffer(hailo_vdevice vdevice,void *address, size_t size, hailo_dma_buffer_direction_t direction) +hailo_status hailo_vdevice_dma_map_buffer(hailo_vdevice vdevice, void *address, size_t size, hailo_dma_buffer_direction_t direction) { CHECK_ARG_NOT_NULL(vdevice); CHECK_ARG_NOT_NULL(address); @@ -1144,6 +1144,30 @@ hailo_status hailo_vdevice_dma_unmap_buffer(hailo_vdevice vdevice, void *address return reinterpret_cast(vdevice)->dma_unmap(address, size, direction); } +hailo_status hailo_device_dma_map_dmabuf(hailo_device device, int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + CHECK_ARG_NOT_NULL(device); + return reinterpret_cast(device)->dma_map_dmabuf(dmabuf_fd, size, direction); +} + +hailo_status hailo_device_dma_unmap_dmabuf(hailo_device device, int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + CHECK_ARG_NOT_NULL(device); + return reinterpret_cast(device)->dma_unmap_dmabuf(dmabuf_fd, size, direction); +} + +hailo_status hailo_vdevice_dma_map_dmabuf(hailo_vdevice vdevice, int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + CHECK_ARG_NOT_NULL(vdevice); + return reinterpret_cast(vdevice)->dma_map_dmabuf(dmabuf_fd, size, direction); +} + +hailo_status hailo_vdevice_dma_unmap_dmabuf(hailo_vdevice vdevice, int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + CHECK_ARG_NOT_NULL(vdevice); + return reinterpret_cast(vdevice)->dma_unmap_dmabuf(dmabuf_fd, size, direction); +} + hailo_status hailo_calculate_eth_input_rate_limits(hailo_hef hef, const char *network_group_name, uint32_t fps, hailo_rate_limit_t *rates, size_t *rates_length) { @@ -1228,6 +1252,9 @@ hailo_status hailo_stream_read_raw_buffer(hailo_output_stream stream, void *buff MemoryView buffer_view(buffer, size); auto status = (reinterpret_cast(stream))->read(buffer_view); + if (HAILO_STREAM_ABORT == status) { + return status; + } CHECK_SUCCESS(status); return HAILO_SUCCESS; } @@ -1238,6 +1265,9 @@ hailo_status hailo_stream_write_raw_buffer(hailo_input_stream stream, const void CHECK_ARG_NOT_NULL(buffer); auto status = (reinterpret_cast(stream))->write(MemoryView::create_const(buffer, size)); + if (HAILO_STREAM_ABORT == status) { + return status; + } CHECK_SUCCESS(status); return HAILO_SUCCESS; } diff --git a/hailort/libhailort/src/hef/context_switch_actions.cpp b/hailort/libhailort/src/hef/context_switch_actions.cpp index db6e9c2..42c85c3 100644 --- a/hailort/libhailort/src/hef/context_switch_actions.cpp +++ b/hailort/libhailort/src/hef/context_switch_actions.cpp @@ -43,20 +43,15 @@ Expected> ContextSwitchConfigAction::serialize(const Context CHECK_AS_EXPECTED(m_action_list_type < CONTEXT_SWITCH_DEFS__ACTION_TYPE_COUNT, HAILO_INTERNAL_FAILURE, "Action cannot be serialized"); - auto header = serialize_header(); - CHECK_EXPECTED(header); + TRY(auto header, serialize_header()); + TRY(auto params, serialize_params(context_resources)); + TRY(auto serialized_action, Buffer::create(header.size() + params.size())); - auto params = serialize_params(context_resources); - CHECK_EXPECTED(params); - - auto serialized_action = Buffer::create(header->size() + params->size()); - CHECK_EXPECTED(serialized_action); - - std::copy(header->begin(), header->end(), serialized_action->data()); - std::copy(params->begin(), params->end(), serialized_action->data() + header->size()); + std::copy(header.begin(), header.end(), serialized_action.data()); + std::copy(params.begin(), params.end(), serialized_action.data() + header.size()); std::vector buffers; - buffers.emplace_back(serialized_action.release()); + buffers.emplace_back(std::move(serialized_action)); return buffers; } @@ -195,37 +190,34 @@ hailo_status WriteDataCcwActionByBuffer::write_to_config_buffer(ConfigBuffer& co CHECK_SUCCESS(status); if (should_support_pre_fetch && is_last_write) { - auto desc_count = config_buffer.program_descriptors(); - CHECK_EXPECTED_AS_STATUS(desc_count); + TRY(const auto desc_count, config_buffer.program_descriptors()); + (void)desc_count; } return HAILO_SUCCESS; } -Expected WriteDataCcwAction::create(uint32_t offset, size_t size, uint8_t config_stream_index, - size_t total_ccw_burst, std::shared_ptr shef_file_handle) +Expected WriteDataCcwAction::create(std::vector &&ccw_write_ptrs, uint8_t config_stream_index, + uint16_t total_ccw_burst, std::shared_ptr hef_reader) { - CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(total_ccw_burst), HAILO_INVALID_HEF, - "Too many ccw burst {} (must fit in uint16)", total_ccw_burst); auto result = ContextSwitchConfigActionPtr(new (std::nothrow) WriteDataCcwAction( - offset, size, config_stream_index, static_cast(total_ccw_burst), shef_file_handle)); + std::move(ccw_write_ptrs), config_stream_index, total_ccw_burst, hef_reader)); CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); return result; } WriteDataCcwActionByBuffer::WriteDataCcwActionByBuffer(Buffer &&data, uint8_t config_stream_index, uint16_t total_ccw_burst) : - WriteDataCcwAction(0, 0, config_stream_index, total_ccw_burst, nullptr), + WriteDataCcwAction({}, config_stream_index, total_ccw_burst, nullptr), m_data(std::move(data)) {} -WriteDataCcwAction::WriteDataCcwAction(uint32_t offset, size_t size, uint8_t config_stream_index, uint16_t total_ccw_burst, - std::shared_ptr shef_file_handle) : +WriteDataCcwAction::WriteDataCcwAction(std::vector &&ccw_write_ptrs, uint8_t config_stream_index, + uint16_t total_ccw_burst, std::shared_ptr hef_reader) : ContextSwitchConfigAction(Type::WriteDataCcw), - m_offset(offset), - m_size(size), + m_ccw_write_ptrs(std::move(ccw_write_ptrs)), m_config_stream_index(config_stream_index), m_total_ccw_burst(total_ccw_burst), - m_shef_file_handle(shef_file_handle) + m_hef_reader(hef_reader) {} Expected> WriteDataCcwAction::serialize(const ContextResources &) const @@ -248,17 +240,36 @@ Expected WriteDataCcwAction::serialize_params(const ContextResources &) hailo_status WriteDataCcwAction::write_to_config_buffer(ConfigBuffer& config_buffer, bool should_support_pre_fetch) { - bool is_last_write = config_buffer.size_left() == size(); + uint64_t total_ccw_size = 0; + for (const auto &ccw_write_ptr : m_ccw_write_ptrs) { + total_ccw_size += ccw_write_ptr.size; + } + + bool is_last_write = config_buffer.size_left() == total_ccw_size; + if (should_support_pre_fetch && is_last_write) { + auto status = config_buffer.pad_with_nops(); + CHECK_SUCCESS(status); + } + + auto status = m_hef_reader->open(); + CHECK_SUCCESS(status); - auto buffer = m_shef_file_handle->read(m_offset, m_size); - CHECK_EXPECTED_AS_STATUS(buffer); + for (const auto &ccw_write_ptr : m_ccw_write_ptrs) { + TRY(auto buffer, Buffer::create_shared(ccw_write_ptr.size)); + MemoryView mem_view(buffer->data(), buffer->size()); + assert(ccw_write_ptr.offset <= SIZE_MAX); + status = m_hef_reader->read_from_offset(static_cast(ccw_write_ptr.offset), mem_view, ccw_write_ptr.size); + CHECK_SUCCESS(status); + status = config_buffer.write(mem_view); + CHECK_SUCCESS(status); + } - auto status = config_buffer.write(MemoryView(buffer.value())); + status = m_hef_reader->close(); CHECK_SUCCESS(status); if (should_support_pre_fetch && is_last_write) { - auto desc_count = config_buffer.program_descriptors(); - CHECK_EXPECTED_AS_STATUS(desc_count); + TRY(const auto desc_count, config_buffer.program_descriptors()); + (void)desc_count; } return HAILO_SUCCESS; @@ -366,6 +377,29 @@ Expected ResetBurstCreditsTaskAction::serialize_params(const ContextReso return Buffer::create(0); } +Expected WaitForCacheUpdatedAction::create() +{ + auto result = ContextSwitchConfigActionPtr(new (std::nothrow) WaitForCacheUpdatedAction()); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + return result; +} + +WaitForCacheUpdatedAction::WaitForCacheUpdatedAction() : + ContextSwitchConfigAction(Type::WaitForCacheUpdated, CONTEXT_SWITCH_DEFS__ACTION_TYPE_WAIT_FOR_CACHE_UPDATED) +{} + +bool WaitForCacheUpdatedAction::supports_repeated_block() const +{ + // We don't support repeated blocks for this action, since only one is added per group of consecutive + // TriggerNewDataFromDataInput actions. + return false; +} + +Expected WaitForCacheUpdatedAction::serialize_params(const ContextResources &) const +{ + return Buffer::create(0); +} + Expected WaitForNetworkGroupChangeAction::create() { auto result = ContextSwitchConfigActionPtr(new (std::nothrow) WaitForNetworkGroupChangeAction()); @@ -434,14 +468,14 @@ Expected> RepeatedAction::serialize(const ContextResources & auto repeated_header = ContextSwitchConfigAction::serialize(context_resources); CHECK_EXPECTED(repeated_header); - CHECK_AS_EXPECTED(repeated_header->size() == 1, HAILO_INTERNAL_FAILURE, "Repeated action header should contain one buffer"); + CHECK_AS_EXPECTED(repeated_header->size() == 1, HAILO_INTERNAL_FAILURE, + "Repeated action header should contain one buffer"); buffers.emplace_back(std::move(repeated_header->at(0))); for (const auto &action : m_actions) { assert(action->get_action_list_type() == m_sub_action_type); - auto action_buffer = action->serialize_params(context_resources); - CHECK_EXPECTED(action_buffer); - buffers.emplace_back(action_buffer.release()); + TRY(auto action_buffer, action->serialize_params(context_resources)); + buffers.emplace_back(std::move(action_buffer)); } return buffers; @@ -651,27 +685,28 @@ bool AllowInputDataflowAction::supports_repeated_block() const Expected AllowInputDataflowAction::serialize_params(const ContextResources &context_resources) const { // H2D direction because it is Input actions - const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_H2D_STREAM); - CHECK_EXPECTED(edge_layer); + TRY(const auto edge_layer, + context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_H2D_STREAM)); CONTEXT_SWITCH_DEFS__fetch_data_action_data_t params{}; - params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id); + params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id); params.stream_index = m_stream_index; - params.network_index = edge_layer->layer_info.network_index; - params.host_buffer_type = static_cast(edge_layer->buffer_info.buffer_type); + params.network_index = edge_layer.layer_info.network_index; + params.host_buffer_type = static_cast(edge_layer.buffer_info.buffer_type); - switch (edge_layer->layer_info.type) { + switch (edge_layer.layer_info.type) { case LayerType::BOUNDARY: params.credit_type = CONTEXT_SWITCH_DEFS__CREDIT_IN_BYTES; - params.frame_periph_size = edge_layer->layer_info.nn_stream_config.periph_bytes_per_buffer * - edge_layer->layer_info.nn_stream_config.periph_buffers_per_frame; + params.frame_periph_size = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer * + edge_layer.layer_info.nn_stream_config.periph_buffers_per_frame; break; case LayerType::INTER_CONTEXT: + case LayerType::CACHE: params.credit_type = CONTEXT_SWITCH_DEFS__CREDIT_IN_DESCRIPTORS; - params.frame_periph_size = ((edge_layer->buffer_info.bytes_in_pattern - 1) / (edge_layer->buffer_info.desc_page_size)) + 1; + params.frame_periph_size = ((edge_layer.buffer_info.bytes_in_pattern - 1) / (edge_layer.buffer_info.desc_page_size)) + 1; break; default: - LOGGER__ERROR("Invalid layer type {} for stream {}", static_cast(edge_layer->layer_info.type), m_stream_index); + LOGGER__ERROR("Invalid layer type {} for stream {}", static_cast(edge_layer.layer_info.type), m_stream_index); return make_unexpected(HAILO_INTERNAL_FAILURE); } @@ -866,15 +901,15 @@ bool WaitOutputTransferDoneAction::supports_repeated_block() const Expected WaitOutputTransferDoneAction::serialize_params(const ContextResources &context_resources) const { // D2H direction because it is output action - const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_D2H_STREAM); - CHECK_EXPECTED(edge_layer); + TRY(const auto edge_layer, + context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_D2H_STREAM)); CONTEXT_SWITCH_DEFS__vdma_dataflow_interrupt_data_t params{}; - params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id); + params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id); params.stream_index = m_stream_index; - params.network_index = edge_layer->layer_info.network_index; - params.is_inter_context = static_cast(LayerType::INTER_CONTEXT == edge_layer->layer_info.type); - params.host_buffer_type = static_cast(edge_layer->buffer_info.buffer_type); + params.network_index = edge_layer.layer_info.network_index; + params.is_inter_context = static_cast(LayerType::INTER_CONTEXT == edge_layer.layer_info.type); + params.host_buffer_type = static_cast(edge_layer.buffer_info.buffer_type); return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } @@ -906,16 +941,15 @@ Expected OpenBoundaryInputChannelAction::serialize_params(const ContextR CONTEXT_SWITCH_DEFS__open_boundary_input_channel_data_t params{}; // H2D direction because it is Input actions - const auto edge_layer = context_resources.get_edge_layer_by_channel_id(m_channel_id); - CHECK_EXPECTED(edge_layer); + TRY(const auto edge_layer, context_resources.get_edge_layer_by_channel_id(m_channel_id)); - params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id); + params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id); params.host_buffer_info = m_host_buffer_info; - params.stream_index = edge_layer->layer_info.stream_index; - params.network_index = edge_layer->layer_info.network_index; - params.periph_bytes_per_buffer = edge_layer->layer_info.nn_stream_config.periph_bytes_per_buffer; - params.frame_periph_size = edge_layer->layer_info.nn_stream_config.periph_bytes_per_buffer * - edge_layer->layer_info.nn_stream_config.periph_buffers_per_frame; + params.stream_index = edge_layer.layer_info.stream_index; + params.network_index = edge_layer.layer_info.network_index; + params.periph_bytes_per_buffer = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer; + params.frame_periph_size = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer * + edge_layer.layer_info.nn_stream_config.periph_buffers_per_frame; return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } @@ -1205,6 +1239,84 @@ Expected ActivateDdrOutputChannelAction::serialize_params(const ContextR return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } +Expected ActivateCacheInputChannelAction::create(const vdma::ChannelId &channel_id, + uint8_t stream_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, + const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info, uint32_t initial_credit_size) +{ + auto result = ContextSwitchConfigActionPtr(new (std::nothrow) ActivateCacheInputChannelAction(channel_id, + stream_index, nn_stream_config, host_buffer_info, initial_credit_size)); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + return result; +} + +ActivateCacheInputChannelAction::ActivateCacheInputChannelAction(const vdma::ChannelId &channel_id, + uint8_t stream_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, + const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info, uint32_t initial_credit_size) : + ContextSwitchConfigAction(ContextSwitchConfigAction::Type::ActivateCacheInputChannel, + CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_INPUT), + m_channel_id(channel_id), + m_stream_index(stream_index), + m_nn_stream_config(nn_stream_config), + m_host_buffer_info(host_buffer_info), + m_initial_credit_size(initial_credit_size) +{} + +bool ActivateCacheInputChannelAction::supports_repeated_block() const +{ + // Activate actions shouldn't be repeated (for easier debugging). + return false; +} + +Expected ActivateCacheInputChannelAction::serialize_params(const ContextResources &) const +{ + CONTEXT_SWITCH_DEFS__activate_cache_input_data_t params{}; + params.packed_vdma_channel_id = pack_vdma_channel_id(m_channel_id); + params.stream_index = m_stream_index; + params.stream_reg_info = parse_nn_config(m_nn_stream_config); + params.host_buffer_info = m_host_buffer_info; + params.initial_credit_size = m_initial_credit_size; + return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); +} + +Expected ActivateCacheOutputChannelAction::create(const vdma::ChannelId &channel_id, + uint8_t stream_index, uint8_t network_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, + const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info) +{ + auto result = ContextSwitchConfigActionPtr(new (std::nothrow) ActivateCacheOutputChannelAction(channel_id, + stream_index, network_index, nn_stream_config, host_buffer_info)); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + return result; +} + +ActivateCacheOutputChannelAction::ActivateCacheOutputChannelAction(const vdma::ChannelId &channel_id, + uint8_t stream_index, uint8_t network_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, + const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info) : + ContextSwitchConfigAction(ContextSwitchConfigAction::Type::ActivateCacheOutputChannel, + CONTEXT_SWITCH_DEFS__ACTION_TYPE_ACTIVATE_CACHE_OUTPUT), + m_channel_id(channel_id), + m_stream_index(stream_index), + m_network_index(network_index), + m_nn_stream_config(nn_stream_config), + m_host_buffer_info(host_buffer_info) +{} + +bool ActivateCacheOutputChannelAction::supports_repeated_block() const +{ + // Activate actions shouldn't be repeated (for easier debugging). + return false; +} + +Expected ActivateCacheOutputChannelAction::serialize_params(const ContextResources &) const +{ + CONTEXT_SWITCH_DEFS__activate_cache_output_data_t params{}; + params.packed_vdma_channel_id = pack_vdma_channel_id(m_channel_id); + params.stream_index = m_stream_index; + params.network_index = m_network_index; + params.stream_reg_info = parse_nn_config(m_nn_stream_config); + params.host_buffer_info = m_host_buffer_info; + return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); +} + Expected ValidateChannelAction::create(const EdgeLayer &edge_layer, const bool is_batch_switch_context) { @@ -1390,14 +1502,14 @@ bool WaitDmaIdleAction::supports_repeated_block() const Expected WaitDmaIdleAction::serialize_params(const ContextResources &context_resources) const { // D2H direction because it is output action - const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_D2H_STREAM); - CHECK_EXPECTED(edge_layer); + TRY(const auto edge_layer, + context_resources.get_edge_layer_by_stream_index(m_stream_index, HAILO_D2H_STREAM)); CONTEXT_SWITCH_DEFS__wait_dma_idle_data_t params{}; - params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id); - params.is_inter_context = static_cast(LayerType::INTER_CONTEXT == edge_layer->layer_info.type); + params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id); + params.is_inter_context = static_cast(LayerType::INTER_CONTEXT == edge_layer.layer_info.type); params.stream_index = m_stream_index; - params.host_buffer_type = static_cast(edge_layer->buffer_info.buffer_type); + params.host_buffer_type = static_cast(edge_layer.buffer_info.buffer_type); return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } diff --git a/hailort/libhailort/src/hef/context_switch_actions.hpp b/hailort/libhailort/src/hef/context_switch_actions.hpp index 31ab7d4..fab2375 100644 --- a/hailort/libhailort/src/hef/context_switch_actions.hpp +++ b/hailort/libhailort/src/hef/context_switch_actions.hpp @@ -13,6 +13,7 @@ #include "hailo/expected.hpp" #include "hailo/buffer.hpp" +#include "common/file_utils.hpp" #include "vdma/channel/channel_id.hpp" #include "hef/layer_info.hpp" @@ -21,16 +22,22 @@ #include "context_switch_defs.h" #include "core_op/resource_manager/config_buffer.hpp" - namespace hailort { class ContextResources; struct EdgeLayer; +#pragma pack(push, 1) +typedef struct { + uint64_t offset; + uint32_t size; +} ccw_write_ptr_t; +#pragma pack(pop) class ContextSwitchConfigAction; using ContextSwitchConfigActionPtr = std::shared_ptr; + class ContextSwitchConfigAction { public: @@ -68,6 +75,8 @@ public: ActivateInterContextOutputChannel, ActivateDdrInputChannel, ActivateDdrOutputChannel, + ActivateCacheInputChannel, + ActivateCacheOutputChannel, ValidateChannel, DeactivateChannel, WaitDmaIdle, @@ -78,6 +87,7 @@ public: ChangeBoundaryInputBatchAction, PauseVdmaChannel, ResumeVdmaChannel, + WaitForCacheUpdated, }; ContextSwitchConfigAction(ContextSwitchConfigAction &&) = default; @@ -156,12 +166,11 @@ private: const vdma::ChannelId m_channel_id; }; -class ShefFileHandle; class WriteDataCcwAction : public ContextSwitchConfigAction { public: - static Expected create(uint32_t offset, size_t size, uint8_t config_stream_index, - size_t total_ccw_burst, std::shared_ptr shef_file_handle); + static Expected create(std::vector &&ccw_write_ptrs, uint8_t config_stream_index, + uint16_t total_ccw_burst, std::shared_ptr hef_reader); WriteDataCcwAction(WriteDataCcwAction &&) = default; WriteDataCcwAction(const WriteDataCcwAction &) = delete; WriteDataCcwAction &operator=(WriteDataCcwAction &&) = delete; @@ -172,20 +181,20 @@ public: virtual bool supports_repeated_block() const override; virtual Expected serialize_params(const ContextResources &context_resources) const override; - uint8_t config_stream_index() const { return m_config_stream_index; } - uint16_t total_ccw_burst() const { return m_total_ccw_burst; } virtual size_t size() const { return m_size; } + virtual uint8_t config_stream_index() const { return m_config_stream_index; } virtual hailo_status write_to_config_buffer(ConfigBuffer& config_buffer, bool should_support_pre_fetch); + uint16_t total_ccw_burst() const { return m_total_ccw_burst; } protected: - WriteDataCcwAction(uint32_t offset, size_t size, uint8_t config_stream_index, - uint16_t total_ccw_burst, std::shared_ptr shef_file_handle); + WriteDataCcwAction(std::vector &&ccw_write_ptrs, uint8_t config_stream_index, + uint16_t total_ccw_burst, std::shared_ptr hef_reader); - uint32_t m_offset; + const std::vector m_ccw_write_ptrs; size_t m_size; const uint8_t m_config_stream_index; - const uint16_t m_total_ccw_burst; - std::shared_ptr m_shef_file_handle; + uint16_t m_total_ccw_burst; + std::shared_ptr m_hef_reader; }; class WriteDataCcwActionByBuffer : public WriteDataCcwAction @@ -277,6 +286,23 @@ private: ResetBurstCreditsTaskAction(); }; +class WaitForCacheUpdatedAction : public ContextSwitchConfigAction +{ +public: + static Expected create(); + + WaitForCacheUpdatedAction(WaitForCacheUpdatedAction &&) = default; + WaitForCacheUpdatedAction(const WaitForCacheUpdatedAction &) = delete; + WaitForCacheUpdatedAction &operator=(WaitForCacheUpdatedAction &&) = delete; + WaitForCacheUpdatedAction &operator=(const WaitForCacheUpdatedAction &) = delete; + virtual ~WaitForCacheUpdatedAction() = default; + virtual bool supports_repeated_block() const override; + virtual Expected serialize_params(const ContextResources &context_resources) const override; + +private: + WaitForCacheUpdatedAction(); +}; + class WaitForNetworkGroupChangeAction : public ContextSwitchConfigAction { public: @@ -735,6 +761,51 @@ private: const uint32_t m_buffered_rows_count; }; +class ActivateCacheInputChannelAction : public ContextSwitchConfigAction +{ +public: + static Expected create(const vdma::ChannelId &channel_id, + uint8_t stream_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, + const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info, uint32_t initial_credit_size); + + virtual bool supports_repeated_block() const override; + virtual Expected serialize_params(const ContextResources &context_resources) const override; + +private: + ActivateCacheInputChannelAction(const vdma::ChannelId &channel_id, + uint8_t stream_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, + const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info, + uint32_t initial_credit_size); + + const vdma::ChannelId m_channel_id; + const uint8_t m_stream_index; + const CONTROL_PROTOCOL__nn_stream_config_t m_nn_stream_config; + const CONTROL_PROTOCOL__host_buffer_info_t m_host_buffer_info; + const uint32_t m_initial_credit_size; +}; + +class ActivateCacheOutputChannelAction : public ContextSwitchConfigAction +{ +public: + static Expected create(const vdma::ChannelId &channel_id, uint8_t stream_index, + uint8_t network_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, + const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info); + + virtual bool supports_repeated_block() const override; + virtual Expected serialize_params(const ContextResources &context_resources) const override; + +private: + ActivateCacheOutputChannelAction(const vdma::ChannelId &channel_id, uint8_t stream_index, + uint8_t network_index, const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, + const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info); + + const vdma::ChannelId m_channel_id; + const uint8_t m_stream_index; + const uint8_t m_network_index; + const CONTROL_PROTOCOL__nn_stream_config_t m_nn_stream_config; + const CONTROL_PROTOCOL__host_buffer_info_t m_host_buffer_info; +}; + class ValidateChannelAction : public ContextSwitchConfigAction { public: diff --git a/hailort/libhailort/src/hef/core_op_metadata.cpp b/hailort/libhailort/src/hef/core_op_metadata.cpp index b700dd8..c8f2677 100644 --- a/hailort/libhailort/src/hef/core_op_metadata.cpp +++ b/hailort/libhailort/src/hef/core_op_metadata.cpp @@ -111,6 +111,15 @@ void ContextMetadata::add_ddr_layer(const LayerInfo &layer_info) } } +void ContextMetadata::add_cache_layer(const LayerInfo &layer_info) +{ + if (HAILO_H2D_STREAM == layer_info.direction) { + m_cache_input_layers.push_back(layer_info); + } else { + m_cache_output_layers.push_back(layer_info); + } +} + const std::vector &ContextMetadata::get_boundary_input_layers() const { return m_boundary_input_layers; @@ -141,13 +150,22 @@ const std::vector &ContextMetadata::get_ddr_output_layers() const return m_ddr_output_layers; } +const std::vector &ContextMetadata::get_cache_input_layers() const +{ + return m_cache_input_layers; +} + +const std::vector &ContextMetadata::get_cache_output_layers() const +{ + return m_cache_output_layers; +} + Expected ContextMetadata::get_layers_transfer_size(const std::vector &layer_infos) const { size_t total_transfer_size = 0; for (const auto &layer_info : layer_infos) { - auto transfer_size = LayerInfoUtils::get_transfer_size(layer_info); - CHECK_EXPECTED(transfer_size); - total_transfer_size += transfer_size.release(); + TRY(const auto transfer_size, LayerInfoUtils::get_transfer_size(layer_info)); + total_transfer_size += transfer_size; } return total_transfer_size; } @@ -158,27 +176,22 @@ Expected ContextMetadata::get_context_transfer_size() const // Calc config buffers for (const auto &config_buffer_sizes : m_config_buffers_info) { - total_transfer_size += std::accumulate(config_buffer_sizes.second.begin(), config_buffer_sizes.second.end(), 0); + total_transfer_size += std::accumulate(config_buffer_sizes.second.bursts_sizes.begin(), + config_buffer_sizes.second.bursts_sizes.end(), 0); } // Calc all edge layers - auto boundary_input_transfer_size = get_layers_transfer_size(m_boundary_input_layers); - CHECK_EXPECTED(boundary_input_transfer_size); - auto boundary_output_transfer_size = get_layers_transfer_size(m_boundary_output_layers); - CHECK_EXPECTED(boundary_output_transfer_size); - auto ddr_input_transfer_size = get_layers_transfer_size(m_ddr_input_layers); - CHECK_EXPECTED(ddr_input_transfer_size); - auto ddr_output_transfer_size = get_layers_transfer_size(m_ddr_output_layers); - CHECK_EXPECTED(ddr_output_transfer_size); - auto inter_context_input_transfer_size = get_layers_transfer_size(m_inter_context_input_layers); - CHECK_EXPECTED(inter_context_input_transfer_size); - auto inter_context_output_transfer_size = get_layers_transfer_size(m_inter_context_output_layers); - CHECK_EXPECTED(inter_context_output_transfer_size); + TRY(const auto boundary_input_transfer_size, get_layers_transfer_size(m_boundary_input_layers)); + TRY(const auto boundary_output_transfer_size, get_layers_transfer_size(m_boundary_output_layers)); + TRY(const auto ddr_input_transfer_size, get_layers_transfer_size(m_ddr_input_layers)); + TRY(const auto ddr_output_transfer_size, get_layers_transfer_size(m_ddr_output_layers)); + TRY(const auto inter_context_input_transfer_size, get_layers_transfer_size(m_inter_context_input_layers)); + TRY(const auto inter_context_output_transfer_size, get_layers_transfer_size(m_inter_context_output_layers)); total_transfer_size += - boundary_input_transfer_size.release() + boundary_output_transfer_size.release() + - ddr_input_transfer_size.release() + ddr_output_transfer_size.release() + - inter_context_input_transfer_size.release() + inter_context_output_transfer_size.release(); + boundary_input_transfer_size + boundary_output_transfer_size + + ddr_input_transfer_size + ddr_output_transfer_size + + inter_context_input_transfer_size + inter_context_output_transfer_size; return total_transfer_size; } @@ -282,26 +295,31 @@ const std::vector &CoreOpMetadata::config_channels_info() con Expected> CoreOpMetadata::get_all_layer_infos(const std::string &network_name) const { - auto input_layer_infos = get_input_layer_infos(network_name); - CHECK_EXPECTED(input_layer_infos); - - auto output_layer_infos = get_output_layer_infos(network_name); - CHECK_EXPECTED(output_layer_infos); + TRY(const auto input_layer_infos, get_input_layer_infos(network_name)); + TRY(const auto output_layer_infos, get_output_layer_infos(network_name)); std::vector res; - res.reserve(input_layer_infos->size() + output_layer_infos->size()); - res.insert(res.end(), input_layer_infos->begin(), input_layer_infos->end()); - res.insert(res.end(), output_layer_infos->begin(), output_layer_infos->end()); + res.reserve(input_layer_infos.size() + output_layer_infos.size()); + res.insert(res.end(), input_layer_infos.begin(), input_layer_infos.end()); + res.insert(res.end(), output_layer_infos.begin(), output_layer_infos.end()); return res; } +size_t CoreOpMetadata::get_cache_layers_count() const +{ + size_t cache_layers_count = 0; + for (const auto &context : m_dynamic_contexts) { + cache_layers_count += context.get_cache_input_layers().size() + context.get_cache_output_layers().size(); + } + return cache_layers_count; +} + Expected> CoreOpMetadata::get_input_stream_infos(const std::string &network_name) const { std::vector res; - auto input_layers = get_input_layer_infos(network_name); - CHECK_EXPECTED(input_layers); - for (auto &layer_info : input_layers.value()) { + TRY(const auto input_layers, get_input_layer_infos(network_name)); + for (auto &layer_info : input_layers) { const auto &stream_infos = LayerInfoUtils::get_stream_infos_from_layer_info(layer_info); res.insert(res.end(), stream_infos.begin(), stream_infos.end()); } @@ -311,9 +329,8 @@ Expected> CoreOpMetadata::get_input_stream_info Expected> CoreOpMetadata::get_output_stream_infos(const std::string &network_name) const { std::vector res; - auto output_layers = get_output_layer_infos(network_name); - CHECK_EXPECTED(output_layers); - for (auto &layer_info : output_layers.value()) { + TRY(const auto output_layers, get_output_layer_infos(network_name)); + for (auto &layer_info : output_layers) { const auto &stream_infos = LayerInfoUtils::get_stream_infos_from_layer_info(layer_info); res.insert(res.end(), stream_infos.begin(), stream_infos.end()); } @@ -322,16 +339,13 @@ Expected> CoreOpMetadata::get_output_stream_inf Expected> CoreOpMetadata::get_all_stream_infos(const std::string &network_name) const { - auto input_stream_infos = get_input_stream_infos(network_name); - CHECK_EXPECTED(input_stream_infos); - - auto output_stream_infos = get_output_stream_infos(network_name); - CHECK_EXPECTED(output_stream_infos); + TRY(const auto input_stream_infos, get_input_stream_infos(network_name)); + TRY(const auto output_stream_infos, get_output_stream_infos(network_name)); std::vector res; - res.reserve(input_stream_infos->size() + output_stream_infos->size()); - res.insert(res.end(), input_stream_infos->begin(), input_stream_infos->end()); - res.insert(res.end(), output_stream_infos->begin(), output_stream_infos->end()); + res.reserve(input_stream_infos.size() + output_stream_infos.size()); + res.insert(res.end(), input_stream_infos.begin(), input_stream_infos.end()); + res.insert(res.end(), output_stream_infos.begin(), output_stream_infos.end()); return res; } @@ -350,9 +364,8 @@ Expected CoreOpMetadata::get_total_transfer_size() { size_t total_transfer_size = 0; for (const auto &dynamic_context : m_dynamic_contexts) { - auto context_size = dynamic_context.get_context_transfer_size(); - CHECK_EXPECTED(context_size); - total_transfer_size += context_size.release(); + TRY(const auto context_size, dynamic_context.get_context_transfer_size()); + total_transfer_size += context_size; } return total_transfer_size; } @@ -392,44 +405,38 @@ Expected NetworkGroupMetadata::get_core_op_metadata() const so should be same across all clusters layouts */ { CHECK_AS_EXPECTED(1 == m_core_ops_metadata_per_arch.size(), HAILO_INTERNAL_FAILURE); - auto core_op_metadata_exp = m_core_ops_metadata_per_arch.begin()->second.get_metadata(PARTIAL_CLUSTERS_LAYOUT_IGNORE); - CHECK_EXPECTED(core_op_metadata_exp); + TRY(auto core_op_metadata, m_core_ops_metadata_per_arch.begin()->second.get_metadata(PARTIAL_CLUSTERS_LAYOUT_IGNORE)); - auto core_op_metadata = core_op_metadata_exp.release(); return core_op_metadata; } Expected> NetworkGroupMetadata::get_all_layer_infos() const { - auto core_op_metadata = get_core_op_metadata(); - CHECK_EXPECTED(core_op_metadata); + TRY(const auto core_op_metadata, get_core_op_metadata()); - return core_op_metadata.value()->get_all_layer_infos(); + return core_op_metadata->get_all_layer_infos(); } Expected> NetworkGroupMetadata::get_input_layer_infos(const std::string &network_name) const { - auto core_op_metadata = get_core_op_metadata(); - CHECK_EXPECTED(core_op_metadata); + TRY(const auto core_op_metadata, get_core_op_metadata()); - return core_op_metadata.value()->get_input_layer_infos(network_name); + return core_op_metadata->get_input_layer_infos(network_name); } Expected> NetworkGroupMetadata::get_output_layer_infos(const std::string &network_name) const { - auto core_op_metadata = get_core_op_metadata(); - CHECK_EXPECTED(core_op_metadata); + TRY(const auto core_op_metadata, get_core_op_metadata()); - return core_op_metadata.value()->get_output_layer_infos(network_name); + return core_op_metadata->get_output_layer_infos(network_name); } Expected> NetworkGroupMetadata::get_input_vstream_infos(const std::string &network_name) const { - auto input_layer_infos = get_input_layer_infos(network_name); - CHECK_EXPECTED(input_layer_infos); + TRY(const auto input_layer_infos, get_input_layer_infos(network_name)); std::vector input_vstream_infos; - for (auto &layer_info : input_layer_infos.value()) { + for (auto &layer_info : input_layer_infos) { auto vstreams_info = LayerInfoUtils::get_vstream_infos_from_layer_info(layer_info); input_vstream_infos.insert(input_vstream_infos.end(), std::make_move_iterator(vstreams_info.begin()), std::make_move_iterator(vstreams_info.end())); @@ -441,11 +448,10 @@ Expected> NetworkGroupMetadata::get_input_vstr Expected> NetworkGroupMetadata::get_output_vstream_infos(const std::string &network_name) const { - auto output_layer_infos = get_output_layer_infos(network_name); - CHECK_EXPECTED(output_layer_infos); + TRY(const auto output_layer_infos, get_output_layer_infos(network_name)); std::vector output_vstream_infos; - for (auto &layer_info : output_layer_infos.value()) { + for (auto &layer_info : output_layer_infos) { if (std::any_of(m_ops_metadata.begin(), m_ops_metadata.end(), [&layer_info](auto &op_metadata) { return contains(op_metadata->get_input_names(), layer_info.name); })) { continue; // all output_vstream_infos that relates to the op are coming from the op itself instead of layer_infos @@ -460,9 +466,8 @@ Expected> NetworkGroupMetadata::get_output_vst } } for (auto &metadata : m_ops_metadata) { - auto vstream_info = metadata->get_output_vstream_info(); - CHECK_EXPECTED(vstream_info); - output_vstream_infos.push_back(vstream_info.release()); + TRY(const auto vstream_info, metadata->get_output_vstream_info()); + output_vstream_infos.push_back(std::move(vstream_info)); } // Sort vstream infos by sorted_output_names @@ -496,16 +501,13 @@ Expected> NetworkGroupMetadata::get_output_vst Expected> NetworkGroupMetadata::get_all_vstream_infos(const std::string &network_name) const { - auto input_vstream_infos = get_input_vstream_infos(network_name); - CHECK_EXPECTED(input_vstream_infos); - - auto output_vstream_infos = get_output_vstream_infos(network_name); - CHECK_EXPECTED(output_vstream_infos); + TRY(const auto input_vstream_infos, get_input_vstream_infos(network_name)); + TRY(const auto output_vstream_infos, get_output_vstream_infos(network_name)); std::vector res; - res.reserve(input_vstream_infos->size() + output_vstream_infos->size()); - res.insert(res.end(), input_vstream_infos->begin(), input_vstream_infos->end()); - res.insert(res.end(), output_vstream_infos->begin(), output_vstream_infos->end()); + res.reserve(input_vstream_infos.size() + output_vstream_infos.size()); + res.insert(res.end(), input_vstream_infos.begin(), input_vstream_infos.end()); + res.insert(res.end(), output_vstream_infos.begin(), output_vstream_infos.end()); return res; } @@ -522,9 +524,8 @@ Expected> NetworkGroupMetadata::get_vstream_names_from_ } } - auto all_layers_infos = get_all_layer_infos(); - CHECK_EXPECTED(all_layers_infos); - for (auto &layer_info : all_layers_infos.release()) { + TRY(const auto all_layers_infos, get_all_layer_infos()); + for (const auto &layer_info : all_layers_infos) { if (layer_info.is_multi_planar) { for (auto &plane : layer_info.planes) { if (stream_name == plane.name) { @@ -557,9 +558,8 @@ Expected> NetworkGroupMetadata::get_stream_names_from_v } } - auto all_layers_infos = get_all_layer_infos(); - CHECK_EXPECTED(all_layers_infos); - for (auto &layer_info : all_layers_infos.value()) { + TRY(const auto all_layers_infos, get_all_layer_infos()); + for (const auto &layer_info : all_layers_infos) { if (layer_info.is_mux) { if (is_edge_under_mux(layer_info, vstream_name)) { // vstream_name is a demux of the layer info diff --git a/hailort/libhailort/src/hef/core_op_metadata.hpp b/hailort/libhailort/src/hef/core_op_metadata.hpp index d00ca89..84ef904 100644 --- a/hailort/libhailort/src/hef/core_op_metadata.hpp +++ b/hailort/libhailort/src/hef/core_op_metadata.hpp @@ -34,8 +34,21 @@ struct SupportedFeatures { bool batch_register_config = false; }; +struct ConfigBufferInfo { + /** + * Sizes of all the successive ccw's (ccw burst). + * When working with descriptors list, each burst is programed independently to its descriptors. + */ + std::vector bursts_sizes; + /** + * Default offset = 0. In case of continuous pre-allocated buffer, + * we use this var to get the config buffer offset from the beginning of the hef user address. + */ + uint64_t offset_from_hef_base = 0; +}; + // For each config_stream_index we store vector of all ccw write length. The vector is used to build the config buffer.g -using ConfigBufferInfoMap = std::unordered_map>; +using ConfigBufferInfoMap = std::unordered_map; class ContextMetadata final { @@ -52,6 +65,7 @@ public: void add_boundary_layer(const LayerInfo &layer_info); void add_inter_context_layer(const LayerInfo &layer_info); void add_ddr_layer(const LayerInfo &layer_info); + void add_cache_layer(const LayerInfo &layer_info); const std::vector &get_boundary_input_layers() const; const std::vector &get_boundary_output_layers() const; @@ -59,6 +73,8 @@ public: const std::vector &get_inter_context_output_layers() const; const std::vector &get_ddr_input_layers() const; const std::vector &get_ddr_output_layers() const; + const std::vector &get_cache_input_layers() const; + const std::vector &get_cache_output_layers() const; Expected get_layers_transfer_size(const std::vector &layer_infos) const; Expected get_context_transfer_size() const; @@ -75,6 +91,8 @@ private: std::vector m_inter_context_output_layers; std::vector m_ddr_input_layers; std::vector m_ddr_output_layers; + std::vector m_cache_input_layers; + std::vector m_cache_output_layers; }; struct ConfigChannelInfo { @@ -98,6 +116,7 @@ public: Expected> get_input_layer_infos(const std::string &network_name) const; Expected> get_output_layer_infos(const std::string &network_name) const; Expected> get_all_layer_infos(const std::string &network_name) const; + size_t get_cache_layers_count() const; const ContextMetadata &preliminary_context() const; const std::vector &dynamic_contexts() const; diff --git a/hailort/libhailort/src/hef/hef.cpp b/hailort/libhailort/src/hef/hef.cpp index 389af5b..c7b6315 100644 --- a/hailort/libhailort/src/hef/hef.cpp +++ b/hailort/libhailort/src/hef/hef.cpp @@ -20,7 +20,6 @@ #include "common/string_utils.hpp" #include "common/utils.hpp" #include "common/logger_macros.hpp" -#include "common/file_utils.hpp" #include "net_flow/ops/nms_post_process.hpp" #include "net_flow/ops/yolov5_post_process.hpp" @@ -31,6 +30,7 @@ #include "net_flow/ops/softmax_post_process.hpp" #include "net_flow/ops/yolov5_seg_post_process.hpp" #include "net_flow/ops/yolov8_post_process.hpp" +#include "net_flow/ops/yolov8_bbox_only_post_process.hpp" #include "hef/hef_internal.hpp" #include "vdma/pcie/pcie_device.hpp" #include "vdma/vdma_config_manager.hpp" @@ -146,7 +146,6 @@ bool ConfigureNetworkParams::operator!=(const ConfigureNetworkParams &other) con return !(*this == other); } - // Note: Can't add the definition in the header. This will lead to the following error: // /usr/include/c++/7/bits/unique_ptr.h: In instantiation of 'void std::default_delete<_Tp>::operator()(_Tp*) const [with _Tp = Hef::Impl]': // /usr/include/c++/7/bits/unique_ptr.h:263:17: required from 'std::unique_ptr<_Tp, _Dp>::~unique_ptr() [with _Tp = Hef::Impl; _Dp = std::default_delete]' @@ -160,20 +159,18 @@ Hef &Hef::operator=(Hef &&) = default; Expected Hef::create(const std::string &hef_path) { - auto impl = Hef::Impl::create(hef_path); - CHECK_EXPECTED(impl); + TRY(auto impl, Hef::Impl::create(hef_path)); // TODO: can we do this without the copy ctor here (i.e. make the impl as a unique_ptr to begin with) - return Hef(make_unique_nothrow(impl.release())); + return Hef(make_unique_nothrow(std::move(impl))); } Expected Hef::create(const MemoryView &hef_buffer) { - auto impl = Hef::Impl::create(hef_buffer); - CHECK_EXPECTED(impl); + TRY(auto impl, Hef::Impl::create(hef_buffer)); // TODO: can we do this without the copy ctor here (i.e. make the impl as a unique_ptr to begin with) - return Hef(make_unique_nothrow(impl.release())); + return Hef(make_unique_nothrow(std::move(impl))); } Hef::Hef(std::unique_ptr pimpl) : @@ -182,97 +179,77 @@ Hef::Hef(std::unique_ptr pimpl) : Expected> Hef::get_input_stream_infos(const std::string &name) const { - auto network_pair = pimpl->get_network_group_and_network_name(name); - CHECK_EXPECTED(network_pair); - - return pimpl->get_input_stream_infos(network_pair.value().first, network_pair.value().second); + TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name)); + return pimpl->get_input_stream_infos(network_pair.first, network_pair.second); } Expected> Hef::get_output_stream_infos(const std::string &name) const { - auto network_pair = pimpl->get_network_group_and_network_name(name); - CHECK_EXPECTED(network_pair); - - return pimpl->get_output_stream_infos(network_pair.value().first, network_pair.value().second); + TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name)); + return pimpl->get_output_stream_infos(network_pair.first, network_pair.second); } Expected> Hef::get_all_stream_infos(const std::string &name) const { - auto network_pair = pimpl->get_network_group_and_network_name(name); - CHECK_EXPECTED(network_pair); - - return pimpl->get_all_stream_infos(network_pair.value().first, network_pair.value().second); + TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name)); + return pimpl->get_all_stream_infos(network_pair.first, network_pair.second); } Expected> Hef::get_network_infos(const std::string &net_group_name) const { - auto names_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(names_pair); - return pimpl->get_network_infos(names_pair->first); + TRY(const auto names_pair, pimpl->get_network_group_and_network_name(net_group_name)); + return pimpl->get_network_infos(names_pair.first); } Expected Hef::get_stream_info_by_name(const std::string &stream_name, hailo_stream_direction_t stream_direction, const std::string &net_group_name) const { // Addressing the situation where net_group_name == "" - auto net_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(net_group_name_pair); - auto net_group_name_str = net_group_name_pair->first; + TRY(const auto net_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name)); + const auto &net_group_name_str = net_group_name_pair.first; return pimpl->get_stream_info_by_name(stream_name, stream_direction, net_group_name_str); } Expected> Hef::get_input_vstream_infos(const std::string &name) const { - auto network_pair = pimpl->get_network_group_and_network_name(name); - CHECK_EXPECTED(network_pair); - - return pimpl->get_input_vstream_infos(network_pair.value().first, network_pair.value().second); + TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name)); + return pimpl->get_input_vstream_infos(network_pair.first, network_pair.second); } Expected> Hef::get_output_vstream_infos(const std::string &name) const { - auto network_pair = pimpl->get_network_group_and_network_name(name); - CHECK_EXPECTED(network_pair); - - return pimpl->get_output_vstream_infos(network_pair.value().first, network_pair.value().second); + TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name)); + return pimpl->get_output_vstream_infos(network_pair.first, network_pair.second); } Expected> Hef::get_all_vstream_infos(const std::string &name) const { - auto network_pair = pimpl->get_network_group_and_network_name(name); - CHECK_EXPECTED(network_pair); - - return pimpl->get_all_vstream_infos(network_pair.value().first, network_pair.value().second); + TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name)); + return pimpl->get_all_vstream_infos(network_pair.first, network_pair.second); } Expected> Hef::get_sorted_output_names(const std::string &net_group_name) const { // Addressing the situation where net_group_name == "" - auto net_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(net_group_name_pair); - auto net_group_name_str = net_group_name_pair->first; - + TRY(const auto net_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name)); + const auto &net_group_name_str = net_group_name_pair.first; return pimpl->get_sorted_output_names(net_group_name_str); } Expected Hef::get_number_of_input_streams(const std::string &net_group_name) const { // Addressing the situation where net_group_name == "" - auto net_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(net_group_name_pair); - auto net_group_name_str = net_group_name_pair->first; - + TRY(const auto net_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name)); + const auto &net_group_name_str = net_group_name_pair.first; return pimpl->get_number_of_input_streams(net_group_name_str); } Expected Hef::get_number_of_output_streams(const std::string &net_group_name) const { // Addressing the situation where net_group_name == "" - auto net_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(net_group_name_pair); - auto net_group_name_str = net_group_name_pair->first; - + TRY(const auto net_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name)); + const auto &net_group_name_str = net_group_name_pair.first; return pimpl->get_number_of_output_streams(net_group_name_str); } @@ -307,54 +284,19 @@ Expected> Hef::get_original_names_from_vstream_name(con Expected> Hef::get_stream_names_from_vstream_name(const std::string &vstream_name, const std::string &net_group_name) const { - auto network_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(network_group_name_pair); - auto net_group_name_str = network_group_name_pair->first; - + TRY(const auto network_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name)); + const auto &net_group_name_str = network_group_name_pair.first; return pimpl->get_stream_names_from_vstream_name(vstream_name, net_group_name_str); } Expected> Hef::get_vstream_names_from_stream_name(const std::string &stream_name, const std::string &net_group_name) const { - auto network_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(network_group_name_pair); - auto net_group_name_str = network_group_name_pair->first; - + TRY(const auto network_group_name_pair, pimpl->get_network_group_and_network_name(net_group_name)); + const auto &net_group_name_str = network_group_name_pair.first; return pimpl->get_vstream_names_from_stream_name(stream_name, net_group_name_str); } -ShefFileHandle::ShefFileHandle(const std::string &hef_path, uint32_t ccws_buffer_offset) - : m_hef_path(hef_path), m_ccws_buffer_offset(ccws_buffer_offset) {} - -hailo_status ShefFileHandle::open() -{ - m_hef_file = std::ifstream(m_hef_path, std::ios::in | std::ios::binary); - CHECK(m_hef_file.is_open(), HAILO_OPEN_FILE_FAILURE, "Failed to open HEF file \"{}\". errno: {}", m_hef_path, errno); - return HAILO_SUCCESS; -} - -Expected ShefFileHandle::read(uint32_t offset, size_t size) -{ - auto buffer = Buffer::create(size); - CHECK_EXPECTED(buffer); - - m_hef_file.seekg(m_ccws_buffer_offset + offset, m_hef_file.beg); - CHECK_AS_EXPECTED(m_hef_file.good(), HAILO_FILE_OPERATION_FAILURE, "Seeking in file failed"); - - m_hef_file.read(reinterpret_cast(buffer->data()), size); - CHECK_AS_EXPECTED(m_hef_file.good(), HAILO_FILE_OPERATION_FAILURE, "Failed reading ccw"); - - return buffer; -} - -hailo_status ShefFileHandle::close() -{ - m_hef_file.close(); - CHECK_AS_EXPECTED(m_hef_file.good(), HAILO_CLOSE_FAILURE, "Closing file failed"); - return HAILO_SUCCESS; -} - Expected Hef::Impl::create(const std::string &hef_path) { hailo_status status = HAILO_UNINITIALIZED; @@ -381,6 +323,26 @@ Expected Hef::Impl::create(const MemoryView &hef_buffer) return hef; } +Expected calc_hef_residue_size(std::shared_ptr hef_reader, uint32_t version) +{ + TRY(auto total_size, hef_reader->get_size()); + if (HEADER_VERSION_0 == version) { + return total_size - HEF_HEADER_SIZE_V0; + } else { // version is 1 + return total_size - HEF_HEADER_SIZE_V1; + } +} + +static hailo_status calc_buffer_md5(const uint8_t *buffer, const size_t buffer_size, MD5_SUM_t &calculated_md5) +{ + MD5_CTX md5 = {}; + MD5_Init(&md5); + MD5_Update(&md5, buffer, buffer_size); + MD5_Final(calculated_md5, &md5); + + return HAILO_SUCCESS; +} + static hailo_status calc_istream_md5(std::ifstream &s, MD5_SUM_t &calculated_md5) { char md5_buffer[HEF__MD5_BUFFER_SIZE] = {}; @@ -409,18 +371,35 @@ hailo_status Hef::Impl::validate_hef_header(const hef__header_t &header, MD5_SUM CHECK(HEADER_MAGIC == header.magic, HAILO_INVALID_HEF, "HEF magic does not match. detected magic - {:x}", header.magic); - auto version = header.version; - CHECK((HEADER_VERSION_0 == version) || (HEADER_VERSION_1 == version), HAILO_INVALID_HEF, "HEF version does not match"); + CHECK((HEADER_VERSION_0 == header.version) , HAILO_INTERNAL_FAILURE, + "HEF version does not match. Should be {} but detected {}", HEADER_VERSION_0, header.version); - CHECK(hef_file_residue_size == header.hef_proto_size + header.ccws_size, HAILO_INVALID_HEF, + CHECK(hef_file_residue_size == header.hef_proto_size, HAILO_INVALID_HEF, "HEF file length does not match"); - CHECK(0 == memcmp(&calculated_md5, &header.expected_md5, sizeof(MD5_SUM_t)), HAILO_INVALID_HEF, + CHECK(0 == memcmp(&calculated_md5, &header.distinct.v0.expected_md5, sizeof(MD5_SUM_t)), HAILO_INVALID_HEF, "HEF md5 does not match"); return HAILO_SUCCESS; } +hailo_status Hef::Impl::validate_hef_header(const hef__header_t &header, const uint32_t &crc_32, size_t hef_file_residue_size) +{ + CHECK(HEADER_MAGIC == header.magic, HAILO_INVALID_HEF, + "HEF magic does not match. Should be {:x} but detected magic - {:x}", HEADER_MAGIC, header.magic); + + CHECK((HEADER_VERSION_1 == header.version), HAILO_INTERNAL_FAILURE, + "HEF version does not match. Should be {} but detected {}", HEADER_VERSION_1, header.version); + + CHECK(hef_file_residue_size == header.hef_proto_size + header.distinct.v1.ccws_size, HAILO_INVALID_HEF, + "HEF file length does not match"); + + CHECK(0 == memcmp(&crc_32, &header.distinct.v1.crc, sizeof(crc_32)), HAILO_INVALID_HEF, + "HEF crc does not match"); + + return HAILO_SUCCESS; +} + hailo_status Hef::Impl::validate_hef_extensions() { std::vector unsupported_extensions; @@ -442,6 +421,16 @@ void Hef::Impl::init_md5(MD5_SUM_t &calculated_md5) memcpy(m_md5, calculated_md5, sizeof(m_md5)); } +void Hef::Impl::init_crc(uint32_t crc_32) +{ + memcpy(&m_crc, &crc_32, sizeof(crc_32)); +} + +void Hef::Impl::init_hef_version(uint32_t version) +{ + m_hef_version = version; +} + void Hef::Impl::clear_hef_buffer() { #ifdef HAILO_SUPPORT_MULTI_PROCESS @@ -449,111 +438,183 @@ void Hef::Impl::clear_hef_buffer() #endif // HAILO_SUPPORT_MULTI_PROCESS } +Expected Hef::Impl::parse_hef_header_before_distinct(std::shared_ptr hef_reader) +{ + hef__header_t hef_header = {}; + auto status = hef_reader->read(reinterpret_cast(&hef_header), HEF_COMMON_SIZE); + CHECK_SUCCESS_AS_EXPECTED(status); + + hef_header.magic = BYTE_ORDER__htonl(hef_header.magic); + hef_header.version = BYTE_ORDER__htonl(hef_header.version); + hef_header.hef_proto_size = BYTE_ORDER__htonl(hef_header.hef_proto_size); + + return hef_header; +} + +hailo_status Hef::Impl::fill_v1_hef_header(hef__header_t &hef_header, std::shared_ptr hef_reader) +{ + auto status = hef_reader->read(reinterpret_cast(&hef_header.distinct), sizeof(hef__header_distinct_t::v1)); + CHECK_SUCCESS(status); + + hef_header.distinct.v1.ccws_size = BYTE_ORDER__htonll(hef_header.distinct.v1.ccws_size); + hef_header.distinct.v1.crc = BYTE_ORDER__htonl(hef_header.distinct.v1.crc); + + return HAILO_SUCCESS; +} + +hailo_status Hef::Impl::fill_core_ops_and_networks_metadata(uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset) +{ + fill_core_ops(); + + auto status = fill_networks_metadata(hef_version, hef_reader, ccws_offset); + CHECK_SUCCESS(status); + + // Must be called after fill_networks_metadata + status = validate_hef_extensions(); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +// TODO HRT-13920: remove duplications between parse_hef_file and parse_hef_memview hailo_status Hef::Impl::parse_hef_file(const std::string &hef_path) { #ifdef HAILO_SUPPORT_MULTI_PROCESS - auto hef_buffer = read_binary_file(hef_path); - CHECK_EXPECTED_AS_STATUS(hef_buffer); - m_hef_buffer = hef_buffer.release(); + TRY(m_hef_buffer, read_binary_file(hef_path)); #endif // HAILO_SUPPORT_MULTI_PROCESS - auto hef_file = std::ifstream(hef_path, std::ios::in | std::ios::binary); - CHECK(hef_file.is_open(), HAILO_OPEN_FILE_FAILURE, "Failed to open HEF file \"{}\". errno: {}", hef_path, errno); + TRY(auto hef_reader, SeekableBytesReader::create_reader(hef_path)); + auto status = hef_reader->open(); + CHECK_SUCCESS(status); + m_hef_reader = hef_reader; - hef__header_t header = {}; - hef_file.read(reinterpret_cast(&header), sizeof(header)); - CHECK(hef_file.good(), HAILO_FILE_OPERATION_FAILURE, "Failed reading HEF header"); + TRY(auto hef_header, parse_hef_header_before_distinct(hef_reader)); + init_hef_version(hef_header.version); - header.magic = BYTE_ORDER__htonl(header.magic); - header.version = BYTE_ORDER__htonl(header.version); - header.hef_proto_size = BYTE_ORDER__htonl(header.hef_proto_size); - header.ccws_size = BYTE_ORDER__htonl(header.ccws_size); + size_t ccws_offset = 0; // Relevant only for HEADER_VERSION_1 - auto hef_file_residue_size = get_istream_size(hef_file); - CHECK_EXPECTED_AS_STATUS(hef_file_residue_size); + if (HEADER_VERSION_0 == hef_header.version) { - MD5_SUM_t calculated_md5 = {}; - auto status = calc_istream_md5(hef_file, calculated_md5); - CHECK_SUCCESS(status); + status = hef_reader->read(reinterpret_cast(&hef_header.distinct), sizeof(hef__header_distinct_t::v0)); + CHECK_SUCCESS(status); - status = validate_hef_header(header, calculated_md5, hef_file_residue_size.value()); - CHECK_SUCCESS(status); + MD5_SUM_t calculated_md5 = {}; + status = calc_istream_md5(*hef_reader->get_fstream(), calculated_md5); + CHECK_SUCCESS(status); - if (HEADER_VERSION_1 == header.version) { - auto ptr = make_shared_nothrow(hef_path, - static_cast(sizeof(header) + header.hef_proto_size)); - CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY); - m_shef_file_handle = ptr; - } + TRY(const auto hef_file_residue_size, hef_reader->calculate_remaining_size()); + + status = validate_hef_header(hef_header, calculated_md5, hef_file_residue_size); + CHECK_SUCCESS(status); + + init_md5(calculated_md5); + + } else { // HEADER_VERSION_1 == common_hef_header.version + + status = fill_v1_hef_header(hef_header, hef_reader); + CHECK_SUCCESS(status); + + ccws_offset = HEF_HEADER_SIZE_V1 + hef_header.hef_proto_size; + + TRY(auto calculated_residue_size, calc_hef_residue_size(hef_reader, hef_header.version)); + TRY(auto calculated_crc, CRC32::calc_crc_on_stream(*hef_reader->get_fstream(), calculated_residue_size)); - init_md5(calculated_md5); + status = validate_hef_header(hef_header, calculated_crc, calculated_residue_size); + CHECK_SUCCESS(status); + + init_crc(calculated_crc); + + } ProtoHEFHef hef_message; - google::protobuf::io::IstreamInputStream zero_copy_input(&hef_file); - auto rb = hef_message.ParseFromBoundedZeroCopyStream(&zero_copy_input, header.hef_proto_size); + google::protobuf::io::IstreamInputStream zero_copy_input(hef_reader->get_fstream().get()); + auto rb = hef_message.ParseFromBoundedZeroCopyStream(&zero_copy_input, hef_header.hef_proto_size); // This line corrupts the file CHECK(rb, HAILO_INVALID_HEF, "Failed parsing HEF file"); + hef_reader->get_fstream()->clear(); // The call to ParseFromBoundedZeroCopyStream might corrupt the file, so we need to clear it's error flags + // TODO: Remove this reset after stopping support for V0 (in the new format (V1), the file is not corrupted after parsing the protobuf message). status = transfer_protobuf_field_ownership(hef_message); CHECK_SUCCESS(status); - fill_core_ops(); - - status = fill_networks_metadata(); + status = fill_core_ops_and_networks_metadata(hef_header.version, hef_reader, ccws_offset); CHECK_SUCCESS(status); - // Must be called after fill_networks_metadata - status = validate_hef_extensions(); + status = hef_reader->close(); CHECK_SUCCESS(status); TRACE(HefLoadedTrace, hef_path, m_header.sdk_version(), m_md5); return HAILO_SUCCESS; } +hailo_status Hef::Impl::parse_hef_memview_internal(const size_t proto_size, const uint8_t *proto_buffer, const uint32_t hef_version, + std::shared_ptr hef_reader, size_t ccws_offset) +{ + ProtoHEFHef hef_message; + auto rb = hef_message.ParseFromArray(proto_buffer, static_cast(proto_size)); + CHECK(rb, HAILO_INVALID_HEF, "Failed parsing HEF buffer"); + auto status = transfer_protobuf_field_ownership(hef_message); + CHECK_SUCCESS(status); + + status = fill_core_ops_and_networks_metadata(hef_version, hef_reader, ccws_offset); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +// TODO HRT-13920: remove duplications between parse_hef_file and parse_hef_memview hailo_status Hef::Impl::parse_hef_memview(const MemoryView &hef_memview) { #ifdef HAILO_SUPPORT_MULTI_PROCESS - auto hef_buffer = Buffer::create(hef_memview.data(), hef_memview.size()); - CHECK_EXPECTED_AS_STATUS(hef_buffer); - m_hef_buffer = hef_buffer.release(); + TRY(m_hef_buffer, Buffer::create(hef_memview.data(), hef_memview.size())); #endif // HAILO_SUPPORT_MULTI_PROCESS + TRY(auto hef_reader, SeekableBytesReader::create_reader(hef_memview)); + m_hef_reader = hef_reader; + + TRY(auto hef_header, parse_hef_header_before_distinct(hef_reader)); + init_hef_version(hef_header.version); + CHECK(hef_memview.size() >= sizeof(hef__header_t), HAILO_INVALID_HEF, "Invalid HEF header"); - const hef__header_t &raw_header = reinterpret_cast(*hef_memview.data()); - auto header = raw_header; - header.magic = BYTE_ORDER__htonl(header.magic); - header.version = BYTE_ORDER__htonl(header.version); - header.hef_proto_size = BYTE_ORDER__htonl(header.hef_proto_size); - header.ccws_size = BYTE_ORDER__htonl(header.ccws_size); - auto proto_buffer = (hef_memview.data() + sizeof(header)); - auto proto_size = (hef_memview.size() - sizeof(header)); + size_t ccws_offset = 0; // Relevant only for HEADER_VERSION_1 - MD5_CTX md5 = {}; - MD5_SUM_t calculated_md5 = {}; - MD5_Init(&md5); - MD5_Update(&md5, proto_buffer, proto_size); - MD5_Final(calculated_md5, &md5); + if (HEADER_VERSION_0 == hef_header.version) { + auto status = hef_reader->read(reinterpret_cast(&hef_header.distinct), sizeof(hef__header_distinct_t::v0)); + CHECK_SUCCESS(status); - auto status = validate_hef_header(header, calculated_md5, proto_size); - CHECK_SUCCESS(status); + auto proto_buffer = (hef_memview.data() + HEF_HEADER_SIZE_V0); + auto proto_size = (hef_memview.size() - HEF_HEADER_SIZE_V0); - init_md5(calculated_md5); + MD5_SUM_t calculated_md5 = {}; + status = calc_buffer_md5(proto_buffer, proto_size, calculated_md5); + CHECK_SUCCESS(status); - ProtoHEFHef hef_message; - auto rb = hef_message.ParseFromArray(proto_buffer, static_cast(proto_size)); - CHECK(rb, HAILO_INVALID_HEF, "Failed parsing HEF buffer"); - status = transfer_protobuf_field_ownership(hef_message); - CHECK_SUCCESS(status); + status = validate_hef_header(hef_header, calculated_md5, proto_size); + CHECK_SUCCESS(status); - fill_core_ops(); + init_md5(calculated_md5); - status = fill_networks_metadata(); - CHECK_SUCCESS(status); + return parse_hef_memview_internal(proto_size, proto_buffer, hef_header.version, hef_reader, ccws_offset); - // Must be called after fill_networks_metadata - status = validate_hef_extensions(); - CHECK_SUCCESS(status); + } else { // version is HEADER_VERSION_1 + auto status = fill_v1_hef_header(hef_header, hef_reader); + CHECK_SUCCESS(status); - return HAILO_SUCCESS; + auto proto_and_ccw_buffer = hef_memview.data() + HEF_HEADER_SIZE_V1; + auto proto_size = hef_memview.size() - HEF_HEADER_SIZE_V1 - hef_header.distinct.v1.ccws_size; + + ccws_offset = HEF_HEADER_SIZE_V1 + hef_header.hef_proto_size; + + TRY(auto proto_and_ccws_size, calc_hef_residue_size(hef_reader, hef_header.version)); + auto proto_and_ccws_buffer = MemoryView::create_const(hef_memview.data() + HEF_HEADER_SIZE_V1, proto_and_ccws_size); + TRY(auto calculated_crc, CRC32::calc_crc_on_buffer(proto_and_ccws_buffer)); + + status = validate_hef_header(hef_header, calculated_crc, proto_and_ccws_size); + CHECK_SUCCESS(status); + + init_crc(calculated_crc); + + return parse_hef_memview_internal(static_cast(proto_size), proto_and_ccw_buffer, hef_header.version, hef_reader, ccws_offset); + } } @@ -561,7 +622,7 @@ bool is_multi_layout(const ProtoHEFHwArch &hw_arch) { return (hw_arch == ProtoHEFHwArch::PROTO__HW_ARCH__HAILO8L) || (hw_arch == ProtoHEFHwArch::PROTO__HW_ARCH__HAILO15M); } -hailo_status Hef::Impl::fill_networks_metadata() +hailo_status Hef::Impl::fill_networks_metadata(uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset) { fill_extensions_bitset(); @@ -601,13 +662,11 @@ hailo_status Hef::Impl::fill_networks_metadata() if (m_supported_features.hailo_net_flow) { for (auto &partial_core_op : core_op.partial_core_ops) { partial_clusters_layout_bitmap = partial_core_op->layout.partial_clusters_layout_bitmap(); - auto metadata_per_arch_exp = create_metadata_per_arch(*(partial_core_op->core_op), sorted_network_names); - CHECK_EXPECTED_AS_STATUS(metadata_per_arch_exp); - auto metadata_per_arch = metadata_per_arch_exp.release(); + TRY(const auto metadata_per_arch, + create_metadata_per_arch(*(partial_core_op->core_op), sorted_network_names, hef_version, hef_reader, ccws_offset)); - auto expected_ops_metadata = create_ops_metadata(*network_group, *metadata_per_arch, get_device_arch()); - CHECK_EXPECTED_AS_STATUS(expected_ops_metadata); - m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), expected_ops_metadata.value()}); + TRY(const auto ops_metadata, create_ops_metadata(*network_group, *metadata_per_arch, get_device_arch())); + m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), ops_metadata}); core_op_metadata.add_metadata(metadata_per_arch, partial_clusters_layout_bitmap); } } else { @@ -623,9 +682,7 @@ hailo_status Hef::Impl::fill_networks_metadata() {} }; - auto metadata_per_arch_exp = create_metadata_per_arch(partial_core_op, sorted_network_names); - CHECK_EXPECTED_AS_STATUS(metadata_per_arch_exp); - auto metadata_per_arch = metadata_per_arch_exp.release(); + TRY(const auto metadata_per_arch, create_metadata_per_arch(partial_core_op, sorted_network_names, hef_version, hef_reader, ccws_offset)); std::vector empty_metadata_ops; m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), empty_metadata_ops}); @@ -634,20 +691,16 @@ hailo_status Hef::Impl::fill_networks_metadata() } } else { partial_clusters_layout_bitmap = PARTIAL_CLUSTERS_LAYOUT_IGNORE; - auto metadata_per_arch_exp = create_metadata_per_arch(core_op, sorted_network_names); - CHECK_EXPECTED_AS_STATUS(metadata_per_arch_exp); - auto metadata_per_arch = metadata_per_arch_exp.release(); + TRY(const auto metadata_per_arch, create_metadata_per_arch(core_op, sorted_network_names, hef_version, hef_reader, ccws_offset)); - auto expected_ops_metadata = create_ops_metadata(*network_group, *metadata_per_arch, get_device_arch()); - CHECK_EXPECTED_AS_STATUS(expected_ops_metadata); - m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), expected_ops_metadata.value()}); + TRY(auto ops_metadata, create_ops_metadata(*network_group, *metadata_per_arch, get_device_arch())); + m_post_process_ops_metadata_per_group.insert({metadata_per_arch->core_op_name(), ops_metadata}); core_op_metadata.add_metadata(metadata_per_arch, partial_clusters_layout_bitmap); } // Taking the full-layout's name (name is same across all layouts) - auto metadata_exp = core_op_metadata.get_metadata(PARTIAL_CLUSTERS_LAYOUT_IGNORE); - CHECK_EXPECTED_AS_STATUS(metadata_exp); - auto core_op_name = metadata_exp.value()->core_op_name(); + TRY(const auto metadata, core_op_metadata.get_metadata(PARTIAL_CLUSTERS_LAYOUT_IGNORE)); + auto core_op_name = metadata->core_op_name(); std::map core_op_metadata_map; core_op_metadata_map[core_op_name] = core_op_metadata; // Prepare network_group_metadata @@ -682,11 +735,9 @@ hailo_status Hef::Impl::fill_networks_metadata() sorted_output_names.push_back(name); } } - auto network_group_metadata = NetworkGroupMetadata::create(network_group_name, std::move(core_op_metadata_map), - sorted_output_names, m_supported_features, sorted_network_names, m_post_process_ops_metadata_per_group.at(network_group_name)); - - CHECK_EXPECTED_AS_STATUS(network_group_metadata); - m_network_group_metadata.emplace(network_group_name, network_group_metadata.release()); + TRY(auto network_group_metadata, NetworkGroupMetadata::create(network_group_name, std::move(core_op_metadata_map), + sorted_output_names, m_supported_features, sorted_network_names, m_post_process_ops_metadata_per_group.at(network_group_name))); + m_network_group_metadata.emplace(network_group_name, std::move(network_group_metadata)); } return HAILO_SUCCESS; } @@ -724,25 +775,23 @@ static Expected> parse_config_channels_info(const return config_channels_info; } -Expected Hef::Impl::create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op, const std::vector &sorted_network_names) +Expected Hef::Impl::create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op, const std::vector &sorted_network_names, + uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset) { - auto preliminary_context = HefUtils::parse_preliminary_context(core_op.preliminary_config, m_supported_features, m_shef_file_handle); - CHECK_EXPECTED(preliminary_context); - - auto dynamic_contexts = HefUtils::parse_dynamic_contexts(core_op, m_supported_features, get_device_arch(), m_shef_file_handle); - CHECK_EXPECTED(dynamic_contexts); - - auto config_channels_info = parse_config_channels_info(core_op); - CHECK_EXPECTED(config_channels_info); + // TODO: validate that there's a read+write layer for each cache + no cache_id is only read or written without the + // other. They can be across different contexts (HRT-13655) + TRY(auto preliminary_context, HefUtils::parse_preliminary_context(core_op.preliminary_config, m_supported_features, hef_version, hef_reader, ccws_offset)); + TRY_V(auto dynamic_contexts, HefUtils::parse_dynamic_contexts(core_op, m_supported_features, get_device_arch(), hef_version, hef_reader, ccws_offset)); + TRY(auto config_channels_info, parse_config_channels_info(core_op)); // If const input layer is found in the preliminary context, or first dynamic context we can't use fast batch switch const auto can_fast_batch_switch = - !(preliminary_context.value().const_input_layer_found() || dynamic_contexts.value()[0].const_input_layer_found()); + !(preliminary_context.const_input_layer_found() || dynamic_contexts[0].const_input_layer_found()); // Currently, CoreOp name is the same as network_group_name, thats why we init it with it. // TODO: HRT-9551 - Change it when supporting multi core ops. auto metadata_per_arch = make_shared_nothrow(core_op.network_group_metadata.network_group_name(), - preliminary_context.release(), dynamic_contexts.release(), config_channels_info.release(), + std::move(preliminary_context), std::move(dynamic_contexts), std::move(config_channels_info), m_supported_features, sorted_network_names, can_fast_batch_switch); CHECK_NOT_NULL_AS_EXPECTED(metadata_per_arch, HAILO_OUT_OF_HOST_MEMORY); return metadata_per_arch; @@ -859,7 +908,7 @@ const MemoryView Hef::Impl::get_hef_memview() } #endif // HAILO_SUPPORT_MULTI_PROCESS -Hef::Impl::Impl(const std::string &hef_path, hailo_status &status) : m_shef_file_handle(nullptr) +Hef::Impl::Impl(const std::string &hef_path, hailo_status &status) { status = HAILO_UNINITIALIZED; GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -873,7 +922,7 @@ Hef::Impl::Impl(const std::string &hef_path, hailo_status &status) : m_shef_file status = HAILO_SUCCESS; } -Hef::Impl::Impl(const MemoryView &hef_memview, hailo_status &status) : m_shef_file_handle(nullptr) +Hef::Impl::Impl(const MemoryView &hef_memview, hailo_status &status) { status = HAILO_UNINITIALIZED; GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -1004,12 +1053,9 @@ Expected create_yolov5_op_metadata(const Pro { auto nms_config = create_post_process_nms_config(op_proto); - auto yolo_config = create_yolov5_config(op_proto.nms_op().yolo_nms_op().bbox_decoders(), - op_proto.nms_op().yolo_nms_op().image_height(), op_proto.nms_op().yolo_nms_op().image_width(), pad_index_to_streams_info); - CHECK_EXPECTED(yolo_config); - - auto inputs_metadata = create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads); - CHECK_EXPECTED(inputs_metadata); + TRY(auto yolo_config, create_yolov5_config(op_proto.nms_op().yolo_nms_op().bbox_decoders(), + op_proto.nms_op().yolo_nms_op().image_height(), op_proto.nms_op().yolo_nms_op().image_width(), pad_index_to_streams_info)); + TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads)); std::unordered_map outputs_metadata; net_flow::BufferMetaData output_metadata{}; @@ -1019,7 +1065,7 @@ Expected create_yolov5_op_metadata(const Pro outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); - return net_flow::Yolov5OpMetadata::create(inputs_metadata.release(), outputs_metadata, nms_config, yolo_config.release(), + return net_flow::Yolov5OpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolo_config, network_name); } @@ -1029,23 +1075,20 @@ Expected create_yolov5_bbox_only_op_metadata { auto nms_config = create_post_process_nms_config(op_proto); - auto yolo_v5_config = create_yolov5_config(op_proto.nms_op().yolo_nms_op().bbox_decoders(), - op_proto.nms_op().yolo_nms_op().image_height(), op_proto.nms_op().yolo_nms_op().image_width(), pad_index_to_streams_info); - CHECK_EXPECTED(yolo_v5_config); - - auto inputs_metadata = create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads); - CHECK_EXPECTED(inputs_metadata); + TRY(auto yolo_v5_config, create_yolov5_config(op_proto.nms_op().yolo_nms_op().bbox_decoders(), + op_proto.nms_op().yolo_nms_op().image_height(), op_proto.nms_op().yolo_nms_op().image_width(), pad_index_to_streams_info)); + TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads)); std::unordered_map outputs_metadata; net_flow::BufferMetaData output_metadata{}; - uint32_t num_of_proposals = compute_num_of_proposals(inputs_metadata.value(), yolo_v5_config->anchors); + uint32_t num_of_proposals = compute_num_of_proposals(inputs_metadata, yolo_v5_config.anchors); output_metadata.shape = {1, num_of_proposals, YOLOV5_BBOX_NUM_OF_VALUES + op_proto.nms_op().classes()}; output_metadata.format = net_flow::NmsOpMetadata::expand_output_format_autos_by_op_type( { HAILO_FORMAT_TYPE_AUTO, HAILO_FORMAT_ORDER_AUTO, HAILO_FORMAT_FLAGS_NONE }, net_flow::OperationType::YOLOV5, nms_config.bbox_only); outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); - return net_flow::Yolov5BboxOnlyOpMetadata::create(inputs_metadata.release(), outputs_metadata, nms_config, yolo_v5_config.release(), + return net_flow::Yolov5BboxOnlyOpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolo_v5_config, network_name); } @@ -1054,21 +1097,18 @@ Expected create_yolov5_seg_op_metadata(const const std::string &network_name) { auto nms_config = create_post_process_nms_config(op_proto); - auto yolov5_config = create_yolov5_config(op_proto.nms_op().yolo_seg_op().bbox_decoders(), - op_proto.nms_op().yolo_seg_op().image_height(), op_proto.nms_op().yolo_seg_op().image_width(), pad_index_to_streams_info); - CHECK_EXPECTED(yolov5_config); - - auto inputs_metadata = create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads); - CHECK_EXPECTED(inputs_metadata); + TRY(auto yolov5_config, create_yolov5_config(op_proto.nms_op().yolo_seg_op().bbox_decoders(), + op_proto.nms_op().yolo_seg_op().image_height(), op_proto.nms_op().yolo_seg_op().image_width(), pad_index_to_streams_info)); + TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads)); auto proto_layer_name = op_proto.nms_op().yolo_seg_op().proto_info().proto_layer(); - CHECK_AS_EXPECTED(contains(inputs_metadata.value(), proto_layer_name), HAILO_INVALID_HEF); + CHECK_AS_EXPECTED(contains(inputs_metadata, proto_layer_name), HAILO_INVALID_HEF); const uint32_t SIZE_FACTOR = 2; net_flow::YoloV5SegPostProcessConfig yolov5_seg_config = {}; yolov5_seg_config.mask_threshold = static_cast(op_proto.nms_op().yolo_seg_op().mask_threshold()); yolov5_seg_config.max_accumulated_mask_size = static_cast( - yolov5_config->image_height * yolov5_config->image_width * SIZE_FACTOR); + yolov5_config.image_height * yolov5_config.image_width * SIZE_FACTOR); yolov5_seg_config.proto_layer_name = proto_layer_name; std::unordered_map outputs_metadata; @@ -1078,27 +1118,16 @@ Expected create_yolov5_seg_op_metadata(const net_flow::OperationType::YOLOV5SEG, nms_config.bbox_only); outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); - return net_flow::Yolov5SegOpMetadata::create(inputs_metadata.release(), outputs_metadata, nms_config, yolov5_config.release(), + return net_flow::Yolov5SegOpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolov5_config, yolov5_seg_config, network_name); } -Expected create_yolov8_op_metadata(const ProtoHEFOp &op_proto, - const std::map &pad_index_to_streams_info, const std::map &input_to_output_pads, - const std::string &network_name) +Expected create_yolov8_config(const ProtoHEFOp &op_proto, const std::map &pad_index_to_streams_info) { - auto nms_config = create_post_process_nms_config(op_proto); - net_flow::Yolov8PostProcessConfig yolov8_config{}; yolov8_config.image_height = (float32_t)op_proto.nms_op().yolov8_nms_op().image_height(); yolov8_config.image_width = (float32_t)op_proto.nms_op().yolov8_nms_op().image_width(); - std::unordered_map inputs_metadata; - std::unordered_map outputs_metadata; - net_flow::BufferMetaData output_metadata{}; - output_metadata.format = net_flow::NmsOpMetadata::expand_output_format_autos_by_op_type( - { HAILO_FORMAT_TYPE_AUTO, HAILO_FORMAT_ORDER_AUTO, HAILO_FORMAT_FLAGS_NONE }, net_flow::OperationType::YOLOV8, nms_config.bbox_only); - outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); - for (auto &bbox_proto : op_proto.nms_op().yolov8_nms_op().bbox_decoders()) { assert(contains(pad_index_to_streams_info, static_cast(bbox_proto.reg_pad_index()))); auto reg_name = pad_index_to_streams_info.at(bbox_proto.reg_pad_index()).name; @@ -1106,27 +1135,62 @@ Expected create_yolov8_op_metadata(const Pro auto cls_name = pad_index_to_streams_info.at(bbox_proto.cls_pad_index()).name; yolov8_config.reg_to_cls_inputs.emplace_back(net_flow::Yolov8MatchingLayersNames{reg_name, cls_name, bbox_proto.stride()}); } + return yolov8_config; +} - for (auto &input_pad : op_proto.input_pads()) { - CHECK_AS_EXPECTED(contains(input_to_output_pads, static_cast(input_pad.index())), HAILO_INVALID_HEF, - "NMS op is not connected to core op"); - auto output_pad_index = input_to_output_pads.at(input_pad.index()); - CHECK_AS_EXPECTED(contains(pad_index_to_streams_info, output_pad_index), HAILO_INVALID_HEF, - "Pad {} of post-process {} is not connected to any core output stream", - input_pad.index(), op_proto.name()); - const auto &op_input_stream = pad_index_to_streams_info.at(output_pad_index); - net_flow::BufferMetaData input_metadata{}; - input_metadata.format = op_input_stream.format; - input_metadata.quant_info = op_input_stream.quant_info; - input_metadata.shape = op_input_stream.shape; - input_metadata.padded_shape = op_input_stream.hw_shape; - inputs_metadata.insert({op_input_stream.name, input_metadata}); - } +Expected create_yolov8_op_metadata(const ProtoHEFOp &op_proto, + const std::map &pad_index_to_streams_info, const std::map &input_to_output_pads, + const std::string &network_name) +{ + auto nms_config = create_post_process_nms_config(op_proto); + TRY(auto yolov8_config, create_yolov8_config(op_proto, pad_index_to_streams_info)); + + std::unordered_map outputs_metadata; + net_flow::BufferMetaData output_metadata{}; + output_metadata.format = net_flow::NmsOpMetadata::expand_output_format_autos_by_op_type( + { HAILO_FORMAT_TYPE_AUTO, HAILO_FORMAT_ORDER_AUTO, HAILO_FORMAT_FLAGS_NONE }, net_flow::OperationType::YOLOV8, nms_config.bbox_only); + outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); + + TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads)); return net_flow::Yolov8OpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolov8_config, network_name); } +uint32_t compute_yolov8_bbox_only_num_of_proposals(const std::unordered_map &inputs_metadatas) +{ + static const uint32_t YOLOV8_NUM_OF_OUTPUTS_PER_SHAPE = 2; + uint32_t num_of_proposals = 0; + for (const auto &input_metadata_pair : inputs_metadatas) { + auto &input_metadata = input_metadata_pair.second; + num_of_proposals += input_metadata.shape.height * input_metadata.shape.width; + } + return num_of_proposals / YOLOV8_NUM_OF_OUTPUTS_PER_SHAPE; // we want to count the proposals from each unique shape only once +} + +Expected create_yolov8_bbox_only_op_metadata(const ProtoHEFOp &op_proto, + const std::map &pad_index_to_streams_info, const std::map &input_to_output_pads, + const std::string &network_name) +{ + auto nms_config = create_post_process_nms_config(op_proto); + TRY(auto yolov8_config, create_yolov8_config(op_proto, pad_index_to_streams_info)); + + TRY(auto inputs_metadata, create_inputs_metadata(op_proto, pad_index_to_streams_info, input_to_output_pads)); + + uint32_t num_of_proposals = compute_yolov8_bbox_only_num_of_proposals(inputs_metadata); + + std::unordered_map outputs_metadata; + net_flow::BufferMetaData output_metadata{}; + output_metadata.format = net_flow::NmsOpMetadata::expand_output_format_autos_by_op_type( + { HAILO_FORMAT_TYPE_AUTO, HAILO_FORMAT_ORDER_AUTO, HAILO_FORMAT_FLAGS_NONE }, net_flow::OperationType::YOLOV8, nms_config.bbox_only); + auto bbox_num_of_coordinates = static_cast(sizeof(hailo_rectangle_t) / sizeof(float32_t)); + output_metadata.shape = {1, num_of_proposals, bbox_num_of_coordinates + op_proto.nms_op().classes()}; + outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); + + return net_flow::Yolov8BboxOnlyOpMetadata::create(inputs_metadata, outputs_metadata, nms_config, yolov8_config, + network_name); +} + Expected create_yolox_op_metadata(const ProtoHEFOp &op_proto, const std::map &pad_index_to_streams_info, const std::map &input_to_output_pads, const std::string &network_name) @@ -1356,13 +1420,12 @@ Expected> create_logits_op_metadata(const // TODO: HRT-10603 const auto &op_input_layer_info = pad_index_to_streams_info.at(output_pad_index); - auto max_periph_bytes_from_hef = HefConfigurator::max_periph_bytes_value( - DeviceBase::hef_arch_to_device_arch(static_cast(hef_arch))); - CHECK_EXPECTED(max_periph_bytes_from_hef); + TRY(const auto max_periph_bytes_from_hef, HefConfigurator::max_periph_bytes_value( + DeviceBase::hef_arch_to_device_arch(static_cast(hef_arch)))); // TODO HRT-12099 - return invalid hef error when remove support for hefs with no max_shmifo size - const auto max_periph_bytes = (0 == op_input_layer_info.max_shmifo_size) ? max_periph_bytes_from_hef.value() : - std::min(max_periph_bytes_from_hef.value(), op_input_layer_info.max_shmifo_size); + const auto max_periph_bytes = (0 == op_input_layer_info.max_shmifo_size) ? max_periph_bytes_from_hef : + std::min(max_periph_bytes_from_hef, op_input_layer_info.max_shmifo_size); const auto is_core_hw_padding_supported = HefConfigurator::is_core_hw_padding_supported(op_input_layer_info, max_periph_bytes, is_core_hw_padding_config_in_dfc); @@ -1422,53 +1485,45 @@ Expected> Hef::Impl::create_ops_ switch (op_proto.nms_op().nms_op_case()) { case ProtoHEFNmsOp::kYoloNmsOp: { if (op_proto.nms_op().bbox_decoding_only()) { - auto expected_post_process_op_metadata = create_yolov5_bbox_only_op_metadata(op_proto, pad_index_to_streams_info, - input_to_output_pads, network_name); - CHECK_EXPECTED(expected_post_process_op_metadata); - post_process_op_metadata = expected_post_process_op_metadata.release(); + TRY(post_process_op_metadata, create_yolov5_bbox_only_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, network_name)); break; } else { - auto expected_post_process_op_metadata = create_yolov5_op_metadata(op_proto, pad_index_to_streams_info, - input_to_output_pads, network_name); - CHECK_EXPECTED(expected_post_process_op_metadata); - post_process_op_metadata = expected_post_process_op_metadata.release(); + TRY(post_process_op_metadata, create_yolov5_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, network_name)); break; } } case ProtoHEFNmsOp::kYoloxNmsOp: { - auto expected_post_process_op_metadata = create_yolox_op_metadata(op_proto, pad_index_to_streams_info, - input_to_output_pads, network_name); - CHECK_EXPECTED(expected_post_process_op_metadata); - post_process_op_metadata = expected_post_process_op_metadata.release(); + TRY(post_process_op_metadata, create_yolox_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, network_name)); break; } case ProtoHEFNmsOp::kSsdNmsOp: { - auto expected_post_process_op_metadata = create_ssd_op_metadata(op_proto, pad_index_to_streams_info, - input_to_output_pads, network_name); - CHECK_EXPECTED(expected_post_process_op_metadata); - post_process_op_metadata = expected_post_process_op_metadata.release(); + TRY(post_process_op_metadata, create_ssd_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, network_name)); break; } case ProtoHEFNmsOp::kIouOp: { - auto expected_post_process_op_metadata = create_iou_op_metadata(op_proto, pad_index_to_streams_info, - input_to_output_pads, network_name); - CHECK_EXPECTED(expected_post_process_op_metadata); - post_process_op_metadata = expected_post_process_op_metadata.release(); + TRY(post_process_op_metadata, create_iou_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, network_name)); break; } case ProtoHEFNmsOp::kYoloSegOp: { - auto expected_post_process_op_metadata = create_yolov5_seg_op_metadata(op_proto, pad_index_to_streams_info, - input_to_output_pads, network_name); - CHECK_EXPECTED(expected_post_process_op_metadata); - post_process_op_metadata = expected_post_process_op_metadata.release(); + TRY(post_process_op_metadata, create_yolov5_seg_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, network_name)); break; } case ProtoHEFNmsOp::kYolov8NmsOp: { - auto expected_post_process_op_metadata = create_yolov8_op_metadata(op_proto, pad_index_to_streams_info, - input_to_output_pads, network_name); - CHECK_EXPECTED(expected_post_process_op_metadata); - post_process_op_metadata = expected_post_process_op_metadata.release(); - break; + if (op_proto.nms_op().bbox_decoding_only()) { + TRY(post_process_op_metadata, create_yolov8_bbox_only_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, network_name)); + break; + } else { + TRY(post_process_op_metadata, create_yolov8_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, network_name)); + break; + } } default: { LOGGER__ERROR("Unsupported Net-Flow NMS-Op"); @@ -1480,11 +1535,8 @@ Expected> Hef::Impl::create_ops_ break; } case ProtoHEFOp::kLogitsOp: { - auto expected_logits_op_metadata = create_logits_op_metadata(op_proto, pad_index_to_streams_info, - input_to_output_pads, hef_arch, network_name, m_supported_features.core_hw_padding_config_in_dfc); - CHECK_EXPECTED(expected_logits_op_metadata); - auto post_process_op_metadata = expected_logits_op_metadata.release(); - + TRY(auto post_process_op_metadata, create_logits_op_metadata(op_proto, pad_index_to_streams_info, + input_to_output_pads, hef_arch, network_name, m_supported_features.core_hw_padding_config_in_dfc)); result.push_back(post_process_op_metadata); break; } @@ -1511,16 +1563,13 @@ Expected Hef::Impl::get_core_op_metadata(const std::string &n hailo_status Hef::Impl::validate_boundary_streams_were_created(const std::string &network_group_name, std::shared_ptr core_op) { - auto number_of_inputs = get_number_of_input_streams(network_group_name); - CHECK_EXPECTED_AS_STATUS(number_of_inputs); - - auto size = core_op->get_input_streams().size(); - CHECK((number_of_inputs.value() == size), + TRY(const auto number_of_inputs, get_number_of_input_streams(network_group_name)); + const auto size = core_op->get_input_streams().size(); + CHECK((number_of_inputs == size), HAILO_INVALID_ARGUMENT, "passed configure_params for network group {} did not contain all input streams", network_group_name); - auto number_of_outputs = get_number_of_output_streams(network_group_name); - CHECK_EXPECTED_AS_STATUS(number_of_inputs); - CHECK((number_of_outputs.value() == core_op->get_output_streams().size()), + TRY(const auto number_of_outputs, get_number_of_output_streams(network_group_name)); + CHECK((number_of_outputs == core_op->get_output_streams().size()), HAILO_INVALID_ARGUMENT, "passed configure_params for network group {} did not contain all output streams", network_group_name); return HAILO_SUCCESS; @@ -1581,6 +1630,7 @@ Expected HefConfigurator::max_periph_bytes_value(const hailo_device_ar case HAILO_ARCH_HAILO15H: case HAILO_ARCH_HAILO15M: case HAILO_ARCH_PLUTO: + case HAILO_ARCH_HAILO10H: return HAILO1X_PERIPH_BYTES_PER_BUFFER_MAX_SIZE; default: LOGGER__ERROR("Unknown device architecture!"); @@ -1598,6 +1648,7 @@ Expected HefConfigurator::max_periph_padding_payload_value(const hailo case HAILO_ARCH_HAILO15H: case HAILO_ARCH_HAILO15M: case HAILO_ARCH_PLUTO: + case HAILO_ARCH_HAILO10H: return HAILO1X_PERIPH_PAYLOAD_MAX_VALUE; default: LOGGER__ERROR("Unknown device architecture!"); @@ -1653,28 +1704,22 @@ bool HefConfigurator::is_core_hw_padding_supported(const LayerInfo &layer_info, Expected> Hef::Impl::get_input_stream_infos(const std::string &net_group_name, const std::string &network_name) { - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); - - return core_op_metadata.value()->get_input_stream_infos(network_name); + TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name)); + return core_op_metadata->get_input_stream_infos(network_name); } Expected> Hef::Impl::get_output_stream_infos(const std::string &net_group_name, const std::string &network_name) { - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); - - return core_op_metadata.value()->get_output_stream_infos(network_name); + TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name)); + return core_op_metadata->get_output_stream_infos(network_name); } Expected> Hef::Impl::get_all_stream_infos(const std::string &net_group_name, const std::string &network_name) { - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); - - return core_op_metadata.value()->get_all_stream_infos(network_name); + TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name)); + return core_op_metadata->get_all_stream_infos(network_name); } Expected> Hef::Impl::get_network_infos(const std::string &net_group_name) @@ -1686,21 +1731,18 @@ Expected> Hef::Impl::get_network_infos(const s Expected Hef::Impl::get_stream_info_by_name(const std::string &stream_name, hailo_stream_direction_t stream_direction, const std::string &net_group_name) { - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); + TRY(auto core_op_metadata, get_core_op_metadata(net_group_name)); if (HAILO_H2D_STREAM == stream_direction) { - auto stream_infos = core_op_metadata.value()->get_input_stream_infos(); - CHECK_EXPECTED(stream_infos); - for (auto &stream_info : stream_infos.value()) { + TRY(auto stream_infos, core_op_metadata->get_input_stream_infos()); + for (auto &stream_info : stream_infos) { if (stream_name == stream_info.name) { return std::move(stream_info); } } } else { - auto stream_infos = core_op_metadata.value()->get_output_stream_infos(); - CHECK_EXPECTED(stream_infos); - for (auto &stream_info : stream_infos.value()) { + TRY(auto stream_infos, core_op_metadata->get_output_stream_infos()); + for (auto &stream_info : stream_infos) { if (stream_name == stream_info.name) { return std::move(stream_info); } @@ -1862,22 +1904,16 @@ Expected> Hef::Impl::get_core_op_by_net_grou Expected Hef::Impl::get_number_of_input_streams(const std::string &net_group_name) { - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); - - auto input_stream_infos = core_op_metadata.value()->get_input_stream_infos(); - CHECK_EXPECTED(input_stream_infos); - return input_stream_infos->size(); + TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name)); + TRY(const auto input_stream_infos, core_op_metadata->get_input_stream_infos()); + return input_stream_infos.size(); } Expected Hef::Impl::get_number_of_output_streams(const std::string &net_group_name) { - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); - - auto output_stream_infos = core_op_metadata.value()->get_output_stream_infos(); - CHECK_EXPECTED(output_stream_infos); - return output_stream_infos->size(); + TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name)); + TRY(const auto output_stream_infos, core_op_metadata->get_output_stream_infos()); + return output_stream_infos.size(); } static Expected get_layer_type(const ProtoHEFEdgeConnectionType &edge_connection_type) @@ -1920,21 +1956,14 @@ hailo_status HefUtils::fill_layer_info_with_base_info(const ProtoHEFEdgeLayerBas bool transposed, const uint16_t context_index, const uint8_t network_index, LayerInfo &layer_info, const SupportedFeatures &supported_features, const ProtoHEFHwArch &hef_arch, const bool is_part_of_mux_layer) { - auto format_order_exp = HailoRTDefaults::get_device_format_order(base_info.format()); - CHECK_EXPECTED_AS_STATUS(format_order_exp); - - auto format_oder = format_order_exp.release(); - - auto layer_type = get_layer_type(edge_connection_type); - CHECK_EXPECTED_AS_STATUS(layer_type); - layer_info.type = layer_type.value(); + TRY(layer_info.format.order, HailoRTDefaults::get_device_format_order(base_info.format())); + TRY(layer_info.type, get_layer_type(edge_connection_type)); // Parse host shape - parse hw shape after determining if core hw padding is supported layer_info.shape = parse_layer_shape(base_info); layer_info.hw_data_bytes = base_info.data_bytes(); // TODO: remove duplications with stream info parse - layer_info.format.order = format_oder; layer_info.format.flags = HAILO_FORMAT_FLAGS_NONE; // The check network_group_proto.transposed_net() is for supporting backward compatability for old hefs @@ -1948,16 +1977,13 @@ hailo_status HefUtils::fill_layer_info_with_base_info(const ProtoHEFEdgeLayerBas layer_info.shape.features = 1; } - auto type = HailoRTCommon::get_format_type(layer_info.hw_data_bytes); - CHECK_EXPECTED_AS_STATUS(type); - layer_info.format.type = type.value(); + TRY(layer_info.format.type, HailoRTCommon::get_format_type(layer_info.hw_data_bytes)); - auto max_periph_bytes_from_hef = - HefConfigurator::max_periph_bytes_value(DeviceBase::hef_arch_to_device_arch(static_cast(hef_arch))); - CHECK_EXPECTED_AS_STATUS(max_periph_bytes_from_hef); + TRY(const auto max_periph_bytes_from_hef, + HefConfigurator::max_periph_bytes_value(DeviceBase::hef_arch_to_device_arch(static_cast(hef_arch)))); // TODO HRT-12099 - return invalid hef error when remove support for hefs with no max_shmifo size - const auto max_periph_bytes = (0 == base_info.max_shmifo_size()) ? max_periph_bytes_from_hef.value() : - std::min(max_periph_bytes_from_hef.value(), base_info.max_shmifo_size()); + const auto max_periph_bytes = (0 == base_info.max_shmifo_size()) ? max_periph_bytes_from_hef : + std::min(max_periph_bytes_from_hef, base_info.max_shmifo_size()); // TODO HRT-12051: remove when is_core_hw_padding_supported function is removed // Need to set layer_info.nn_stream_config.core_buffers_per_frame for condition in is_core_hw_padding_supported @@ -1967,10 +1993,8 @@ hailo_status HefUtils::fill_layer_info_with_base_info(const ProtoHEFEdgeLayerBas const bool core_hw_padding_supported = is_part_of_mux_layer ? false : HefConfigurator::is_core_hw_padding_supported(layer_info, max_periph_bytes, supported_features.core_hw_padding_config_in_dfc); - auto nn_stream_config = HefConfigurator::parse_nn_stream_config(base_info, core_hw_padding_supported, - edge_connection_type); - CHECK_EXPECTED_AS_STATUS(nn_stream_config, "Failed parse nn stream config"); - layer_info.nn_stream_config = nn_stream_config.release(); + TRY(layer_info.nn_stream_config, HefConfigurator::parse_nn_stream_config(base_info, core_hw_padding_supported, + edge_connection_type), "Failed parse nn stream config"); layer_info.network_index = network_index; layer_info.context_index = context_index; @@ -1985,10 +2009,8 @@ hailo_status HefUtils::fill_layer_info_with_base_info(const ProtoHEFEdgeLayerBas layer_info.dma_engine_index = static_cast(base_info.engine_id()); if (HAILO_FORMAT_ORDER_HAILO_NMS == layer_info.format.order) { - auto expected_nms_info = parse_proto_nms_info(base_info.additional_info().nms_info(), supported_features.nms_burst_mode, - hef_arch); - CHECK_EXPECTED_AS_STATUS(expected_nms_info); - layer_info.nms_info = expected_nms_info.release(); + TRY(layer_info.nms_info, parse_proto_nms_info(base_info.additional_info().nms_info(), supported_features.nms_burst_mode, + hef_arch)); } layer_info.max_shmifo_size = base_info.max_shmifo_size(); @@ -2091,9 +2113,7 @@ hailo_status HefUtils::fill_fused_nms_info(const ProtoHEFEdgeLayerFused &info, L const ProtoHEFHwArch &hef_arch) { auto base_info = info.layer_info().edge_layer_base(); - auto format_order_exp = HailoRTDefaults::get_device_format_order(base_info.format()); - CHECK_EXPECTED_AS_STATUS(format_order_exp); - layer_info.format.order = format_order_exp.release(); + TRY(layer_info.format.order, HailoRTDefaults::get_device_format_order(base_info.format())); layer_info.format.flags = HAILO_FORMAT_FLAGS_NONE; layer_info.shape.height = static_cast(info.nms_info().number_of_classes()); @@ -2103,13 +2123,9 @@ hailo_status HefUtils::fill_fused_nms_info(const ProtoHEFEdgeLayerFused &info, L layer_info.hw_data_bytes = base_info.data_bytes(); - auto type = HailoRTCommon::get_format_type(layer_info.hw_data_bytes); - CHECK_EXPECTED_AS_STATUS(type); - layer_info.format.type = type.value(); + TRY(layer_info.format.type, HailoRTCommon::get_format_type(layer_info.hw_data_bytes)); - auto expected_nms_info = parse_proto_nms_info(info.nms_info(), burst_mode_enabled, hef_arch); - CHECK_EXPECTED_AS_STATUS(expected_nms_info); - layer_info.nms_info = expected_nms_info.release(); + TRY(layer_info.nms_info, parse_proto_nms_info(info.nms_info(), burst_mode_enabled, hef_arch)); if (HAILO_MAX_STREAM_NAME_SIZE < (info.layer_info().name().length() + 1)) { LOGGER__ERROR("The edge layer '{}' has a too long name (max is HAILO_MAX_STREAM_NAME_SIZE)", info.layer_info().name()); @@ -2213,9 +2229,7 @@ hailo_status HefUtils::fill_planes_info(const ProtoHEFEdgeLayerPlanes &info, hailo_stream_direction_t direction, const uint16_t context_index, const std::string &partial_network_name, uint8_t network_index, LayerInfo &layer_info, const SupportedFeatures &supported_features, const ProtoHEFHwArch &hef_arch) { - auto layer_type = get_layer_type(edge_connection_type); - CHECK_EXPECTED_AS_STATUS(layer_type); - layer_info.type = layer_type.value(); + TRY(layer_info.type, get_layer_type(edge_connection_type)); layer_info.direction = direction; layer_info.shape.height = info.height(); @@ -2225,9 +2239,7 @@ hailo_status HefUtils::fill_planes_info(const ProtoHEFEdgeLayerPlanes &info, layer_info.shape.features = info.features(); layer_info.hw_shape.features = info.features(); - auto format_order = convert_planes_format_to_hailo_format_order(info.planes_format()); - CHECK_EXPECTED_AS_STATUS(format_order); - layer_info.format.order = format_order.release(); + TRY(layer_info.format.order, convert_planes_format_to_hailo_format_order(info.planes_format())); layer_info.format.flags = HAILO_FORMAT_FLAGS_NONE; layer_info.quant_info = {}; // quant_info doesnt make any sense as this is a logical layer layer_info.quant_infos = std::vector(1); // quant_info doesnt make any sense as this is a logical layer @@ -2264,9 +2276,7 @@ hailo_status HefUtils::fill_planes_info(const ProtoHEFEdgeLayerPlanes &info, return underlying_layer.hw_data_bytes == layer_info.hw_data_bytes; }), HAILO_INVALID_HEF, "Not all underlying layers of {} has the same format type", layer_info.name); - auto type = HailoRTCommon::get_format_type(layer_info.hw_data_bytes); - CHECK_EXPECTED_AS_STATUS(type); - layer_info.format.type = type.value(); + TRY(layer_info.format.type, HailoRTCommon::get_format_type(layer_info.hw_data_bytes)); return HAILO_SUCCESS; } @@ -2279,10 +2289,8 @@ hailo_status HefUtils::fill_boundary_layers_info( ContextMetadata &context_metadata, const ProtoHEFHwArch &hef_arch) { - auto layer_info = get_boundary_layer_info(core_op, context_index, layer, supported_features, hef_arch); - CHECK_EXPECTED_AS_STATUS(layer_info); - - context_metadata.add_boundary_layer(layer_info.release()); + TRY(auto layer_info, get_boundary_layer_info(core_op, context_index, layer, supported_features, hef_arch)); + context_metadata.add_boundary_layer(layer_info); return HAILO_SUCCESS; } @@ -2294,10 +2302,8 @@ hailo_status HefUtils::fill_inter_context_layers_info( const SupportedFeatures &supported_features, ContextMetadata &context_metadata) { - auto layer_info = get_inter_context_layer_info(core_op, context_index, layer, supported_features); - CHECK_EXPECTED_AS_STATUS(layer_info); - - context_metadata.add_inter_context_layer(layer_info.release()); + TRY(auto layer_info, get_inter_context_layer_info(core_op, context_index, layer, supported_features)); + context_metadata.add_inter_context_layer(layer_info); return HAILO_SUCCESS; } @@ -2308,10 +2314,20 @@ hailo_status HefUtils::fill_ddr_layers_info( const SupportedFeatures &supported_features, ContextMetadata &context_metadata) { - auto layer_info = get_ddr_layer_info(core_op, context_index, layer, supported_features); - CHECK_EXPECTED_AS_STATUS(layer_info); + TRY(auto layer_info, get_ddr_layer_info(core_op, context_index, layer, supported_features)); + context_metadata.add_ddr_layer(layer_info); + return HAILO_SUCCESS; +} - context_metadata.add_ddr_layer(layer_info.release()); +hailo_status HefUtils::fill_cache_layers_info( + const ProtoHEFCoreOpMock &core_op, + const uint16_t context_index, + const ProtoHEFEdgeLayer &layer, + const SupportedFeatures &supported_features, + ContextMetadata &context_metadata) +{ + TRY(auto layer_info, get_cache_layer_info(core_op, context_index, layer, supported_features)); + context_metadata.add_cache_layer(layer_info); return HAILO_SUCCESS; } @@ -2635,17 +2651,16 @@ static Expected build_config_buffer(const std::vector &ccw_b buffer_size += ccw_buffer.size(); } - auto config_buffer = Buffer::create(buffer_size); - CHECK_EXPECTED(config_buffer); + TRY(auto config_buffer, Buffer::create(buffer_size)); size_t current_offset = 0; for (const auto &ccw_buffer : ccw_buffers) { - assert(current_offset + ccw_buffer.size() <= config_buffer->size()); - memcpy(config_buffer->data() + current_offset, ccw_buffer.data(), ccw_buffer.size()); + assert(current_offset + ccw_buffer.size() <= config_buffer.size()); + memcpy(config_buffer.data() + current_offset, ccw_buffer.data(), ccw_buffer.size()); current_offset += ccw_buffer.size(); } - return config_buffer.release(); + return config_buffer; } static hailo_status merge_write_ccw_actions( @@ -2666,17 +2681,15 @@ static hailo_status merge_write_ccw_actions( for (const auto &ccw_buffers_per_config_stream : ccw_buffers_per_config_streams) { const auto config_stream_index = ccw_buffers_per_config_stream.first; const auto &ccw_buffers = ccw_buffers_per_config_stream.second; - auto config_buffer = build_config_buffer(ccw_buffers); - CHECK_EXPECTED_AS_STATUS(config_buffer); + TRY(auto config_buffer, build_config_buffer(ccw_buffers)); - assert(config_buffer->size() < std::numeric_limits::max()); - config_buffer_infos[config_stream_index].emplace_back(static_cast(config_buffer->size())); + assert(config_buffer.size() < std::numeric_limits::max()); + config_buffer_infos[config_stream_index].bursts_sizes.emplace_back(static_cast(config_buffer.size())); const size_t total_ccw_burst = ccw_buffers.size(); - auto action = WriteDataCcwActionByBuffer::create(config_buffer.release(), config_stream_index, total_ccw_burst); - CHECK_EXPECTED_AS_STATUS(action); - - actions.emplace_back(action.release()); + TRY(auto action, + WriteDataCcwActionByBuffer::create(std::move(config_buffer), config_stream_index, total_ccw_burst)); + actions.emplace_back(std::move(action)); } return HAILO_SUCCESS; @@ -2685,57 +2698,83 @@ static hailo_status merge_write_ccw_actions( static hailo_status build_write_ccw_actions( std::vector &actions, ConfigBufferInfoMap &config_buffer_infos, - const std::vector &write_ccw_actions, - std::shared_ptr shef_file_handle) + const std::vector &write_ccw_actions, + std::shared_ptr hef_reader, + size_t ccws_offset) { - std::unordered_map ccws_per_config_index; + std::unordered_map> ccw_write_ptrs_per_config_streams; for (const auto *write_ccw_action : write_ccw_actions) { CHECK(IS_FIT_IN_UINT8(write_ccw_action->cfg_channel_index()), HAILO_INVALID_HEF, "Invalid cfg channel index"); const auto config_stream_index = static_cast(write_ccw_action->cfg_channel_index()); - if (ccws_per_config_index.find(config_stream_index) == ccws_per_config_index.end()) { - ccws_per_config_index[config_stream_index] = 0; - } - ccws_per_config_index[config_stream_index]++; + ccw_write_ptr_t ccw_write_ptr = {ccws_offset + write_ccw_action->offset(), write_ccw_action->size()}; // offset is relative to the start of the ccws + ccw_write_ptrs_per_config_streams[config_stream_index].emplace_back(ccw_write_ptr); + config_buffer_infos[config_stream_index].bursts_sizes.emplace_back(ccw_write_ptr.size); } - for (const auto *write_ccw_action : write_ccw_actions) { - if (write_ccw_action->data().size() == 0) { - continue; - } - const shef__ccw_offset_t *ccw_offset = reinterpret_cast(write_ccw_action->data().data()); - const auto config_stream_index = static_cast(write_ccw_action->cfg_channel_index()); - assert(BYTE_ORDER__htonl(ccw_offset->size) < std::numeric_limits::max()); - config_buffer_infos[config_stream_index].emplace_back(static_cast(BYTE_ORDER__htonl(ccw_offset->size))); + for (auto &ccw_write_ptrs_per_config_stream : ccw_write_ptrs_per_config_streams) { + const auto config_stream_index = ccw_write_ptrs_per_config_stream.first; + CHECK(IS_FIT_IN_UINT16(ccw_write_ptrs_per_config_stream.second.size()), HAILO_INVALID_HEF, + "Too many ccw burst {} (must fit in uint16)", ccw_write_ptrs_per_config_stream.second.size()); - const size_t total_ccw_burst = ccws_per_config_index[config_stream_index]; - auto action = WriteDataCcwAction::create(BYTE_ORDER__htonl(ccw_offset->offset), BYTE_ORDER__htonl(ccw_offset->size), - config_stream_index, total_ccw_burst, shef_file_handle); - CHECK_EXPECTED_AS_STATUS(action); + config_buffer_infos[config_stream_index].offset_from_hef_base = ccws_offset + ccw_write_ptrs_per_config_stream.second.begin()->offset; - actions.emplace_back(action.release()); + TRY(auto action, + WriteDataCcwAction::create(std::move(ccw_write_ptrs_per_config_stream.second), config_stream_index, + static_cast(ccw_write_ptrs_per_config_stream.second.size()), hef_reader)); + actions.emplace_back(std::move(action)); } return HAILO_SUCCESS; } -static hailo_status parse_operation(std::vector &actions, - ConfigBufferInfoMap &config_buffer_infos, - const ProtoHEFOperation &operation_proto, - const SupportedFeatures &supported_features, - std::shared_ptr shef_file_handle, - bool &const_input_layer_found) +static hailo_status parse_hef_v1_actions(const ProtoHEFOperation &operation_proto, + std::vector &actions, ConfigBufferInfoMap &config_buffer_infos, + const SupportedFeatures &supported_features, bool &const_input_layer_found, + std::shared_ptr hef_reader, size_t ccws_offset) { - auto trigger_action = parse_trigger_action(operation_proto.trigger()); - CHECK_EXPECTED_AS_STATUS(trigger_action); - actions.emplace_back(trigger_action.release()); + std::vector current_write_ccw_ptr_actions; + + for (int action_index = 0; action_index < operation_proto.actions_size(); action_index++) { + const auto &proto_action = operation_proto.actions(action_index); + CHECK(proto_action.action_case() != ProtoHEFAction::kWriteDataCcw, HAILO_INVALID_HEF, "WriteDataCcw action is not supported for hef version 1"); + + if (proto_action.action_case() == ProtoHEFAction::kWriteDataCcwPtr) { + // Keep in vector, parse later + current_write_ccw_ptr_actions.push_back(&proto_action.write_data_ccw_ptr()); - // If current_write_ccw_actions isn't empty, means we currently parsing a group of consecutive write ccw actions. - // we will merge those actions into one write ccw per config channel. + const auto next_action_index = action_index + 1; + const bool is_last_ccw = + (next_action_index == operation_proto.actions_size()) || + (operation_proto.actions(next_action_index).action_case() != ProtoHEFAction::kWriteDataCcwPtr); + + if (is_last_ccw) { + assert(nullptr != hef_reader); + auto status = build_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_ptr_actions, + hef_reader, ccws_offset); + CHECK_SUCCESS(status); + current_write_ccw_ptr_actions.clear(); + } + } else { + TRY(auto action, parse_action(proto_action, supported_features, const_input_layer_found)); + actions.emplace_back(std::move(action)); + } + } + assert(current_write_ccw_ptr_actions.empty()); + + return HAILO_SUCCESS; +} + +static hailo_status parse_hef_v0_actions(const ProtoHEFOperation &operation_proto, + std::vector &actions, ConfigBufferInfoMap &config_buffer_infos, + const SupportedFeatures &supported_features, bool &const_input_layer_found) +{ std::vector current_write_ccw_actions; for (int action_index = 0; action_index < operation_proto.actions_size(); action_index++) { const auto &proto_action = operation_proto.actions(action_index); + CHECK(proto_action.action_case() != ProtoHEFAction::kWriteDataCcwPtr, HAILO_INVALID_HEF, "kWriteDataCcwPtr action is not supported for hef version 0"); + if (proto_action.action_case() == ProtoHEFAction::kWriteDataCcw) { // Keep in vector, parse later current_write_ccw_actions.push_back(&proto_action.write_data_ccw()); @@ -2745,19 +2784,13 @@ static hailo_status parse_operation(std::vector &a (next_action_index == operation_proto.actions_size()) || (operation_proto.actions(next_action_index).action_case() != ProtoHEFAction::kWriteDataCcw); if (is_last_ccw) { - if (nullptr != shef_file_handle) { - auto status = build_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_actions, shef_file_handle); - CHECK_SUCCESS(status); - } else { - auto status = merge_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_actions); - CHECK_SUCCESS(status); - } + auto status = merge_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_actions); + CHECK_SUCCESS(status); current_write_ccw_actions.clear(); } } else { - auto action = parse_action(proto_action, supported_features, const_input_layer_found); - CHECK_EXPECTED_AS_STATUS(action); - actions.emplace_back(action.release()); + TRY(auto action, parse_action(proto_action, supported_features, const_input_layer_found)); + actions.emplace_back(std::move(action)); } } assert(current_write_ccw_actions.empty()); @@ -2765,17 +2798,47 @@ static hailo_status parse_operation(std::vector &a return HAILO_SUCCESS; } +static hailo_status parse_operation(std::vector &actions, + ConfigBufferInfoMap &config_buffer_infos, + const ProtoHEFOperation &operation_proto, + const SupportedFeatures &supported_features, + bool &const_input_layer_found, uint32_t hef_version, + std::shared_ptr hef_reader, size_t ccws_offset) +{ + TRY(auto trigger_action, parse_trigger_action(operation_proto.trigger())); + actions.emplace_back(std::move(trigger_action)); + hailo_status status = HAILO_UNINITIALIZED; + + switch (hef_version) + { + case HEADER_VERSION_0: + status = parse_hef_v0_actions(operation_proto, actions, config_buffer_infos, supported_features, const_input_layer_found); + CHECK_SUCCESS(status); + break; + case HEADER_VERSION_1: + status = parse_hef_v1_actions(operation_proto, actions, config_buffer_infos, supported_features, const_input_layer_found, hef_reader, ccws_offset); + CHECK_SUCCESS(status); + break; + default: + LOGGER__ERROR("Unsupported hef version {}", hef_version); + return HAILO_INVALID_HEF; + } + + return HAILO_SUCCESS; +} + static Expected parse_operations( const google::protobuf::RepeatedPtrField &operations_proto, - const SupportedFeatures &supported_features, std::shared_ptr shef_file_handle) + const SupportedFeatures &supported_features, + uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset) { std::vector actions; ConfigBufferInfoMap config_buffer_infos; bool const_input_layer_found = false; for (const auto &operation_proto : operations_proto) { - auto status = parse_operation(actions, config_buffer_infos, operation_proto, supported_features, shef_file_handle, - const_input_layer_found); + auto status = parse_operation(actions, config_buffer_infos, operation_proto, supported_features, + const_input_layer_found, hef_version, hef_reader, ccws_offset); CHECK_SUCCESS_AS_EXPECTED(status); } @@ -2783,18 +2846,17 @@ static Expected parse_operations( } Expected HefUtils::parse_preliminary_context(const ProtoHEFPreliminaryConfig &preliminary_proto, - const SupportedFeatures &supported_features, std::shared_ptr shef_file_handle) + const SupportedFeatures &supported_features, uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset) { - return parse_operations(preliminary_proto.operation(), supported_features, shef_file_handle); + return parse_operations(preliminary_proto.operation(), supported_features, hef_version, hef_reader, ccws_offset); } Expected HefUtils::parse_single_dynamic_context(const ProtoHEFCoreOpMock &core_op, const ProtoHEFContext &context_proto, uint16_t context_index, const SupportedFeatures &supported_features, - const ProtoHEFHwArch &hef_arch, std::shared_ptr shef_file_handle) + const ProtoHEFHwArch &hef_arch, uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset) { - auto context_metadata_exp = parse_operations(context_proto.operations(), supported_features, shef_file_handle); - CHECK_EXPECTED(context_metadata_exp); - ContextMetadata context_metadata = context_metadata_exp.release(); + TRY(auto context_metadata, + parse_operations(context_proto.operations(), supported_features, hef_version, hef_reader, ccws_offset)); for (const auto &edge_layer : context_proto.metadata().edge_layers()) { if (ProtoHEFEdgeConnectionType::PROTO__EDGE_CONNECTION_TYPE__BOUNDARY == @@ -2812,6 +2874,14 @@ Expected HefUtils::parse_single_dynamic_context(const ProtoHEFC auto status = fill_ddr_layers_info(core_op, context_index, edge_layer, supported_features, context_metadata); CHECK_SUCCESS_AS_EXPECTED(status); + } else if (ProtoHEFEdgeConnectionType::PROTO__EDGE_CONNECTION_TYPE__CACHE == + edge_layer.context_switch_info().edge_connection_type()) { + auto status = fill_cache_layers_info(core_op, context_index, edge_layer, + supported_features, context_metadata); + CHECK_SUCCESS_AS_EXPECTED(status); + } else { + LOGGER__ERROR("Unsupported edge connection type given {}", edge_layer.context_switch_info().edge_connection_type()); + return make_unexpected(HAILO_INVALID_HEF); } } @@ -2842,15 +2912,14 @@ static hailo_status validate_unique_boundary_names(const std::vector> HefUtils::parse_dynamic_contexts(const ProtoHEFCoreOpMock &core_op, const SupportedFeatures &supported_features, - const ProtoHEFHwArch &hef_arch, std::shared_ptr shef_file_handle) + const ProtoHEFHwArch &hef_arch, uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset) { std::vector contexts_metadata; for (uint16_t context_index = 0; context_index < core_op.contexts.size(); context_index++) { auto &context_proto = core_op.contexts[context_index]; - auto context_metadata = parse_single_dynamic_context(core_op, context_proto, context_index, supported_features, - hef_arch, shef_file_handle); - CHECK_EXPECTED(context_metadata); - contexts_metadata.emplace_back(context_metadata.release()); + TRY(auto context_metadata, parse_single_dynamic_context(core_op, context_proto, context_index, supported_features, + hef_arch, hef_version, hef_reader, ccws_offset)); + contexts_metadata.emplace_back(std::move(context_metadata)); } const auto status = validate_unique_boundary_names(contexts_metadata); @@ -2878,10 +2947,11 @@ static Expected get_nms_burst_mode(const ProtoHEFNmsInfo } case PROTO__HW_ARCH__HAILO15H: case PROTO__HW_ARCH__HAILO15M: + case PROTO__HW_ARCH__HAILO10H: case PROTO__HW_ARCH__GINGER: case PROTO__HW_ARCH__LAVENDER: case PROTO__HW_ARCH__PLUTO: - // Second generation of hw NMS - included in hailo15 and pluto. + // Second generation of hw NMS - included in hailo15, hailo10 and pluto. switch (nms_info.burst_type()) { case PROTO__NMS_BURST_TYPE__H15_PER_CLASS: return HAILO_BURST_TYPE_H15_PER_CLASS; @@ -2913,6 +2983,7 @@ static Expected get_nms_bbox_mode(const ProtoHEFNmsInfo return HAILO_BURST_TYPE_H8_BBOX; case PROTO__HW_ARCH__HAILO15H: case PROTO__HW_ARCH__HAILO15M: + case PROTO__HW_ARCH__HAILO10H: case PROTO__HW_ARCH__GINGER: case PROTO__HW_ARCH__LAVENDER: case PROTO__HW_ARCH__PLUTO: @@ -2935,20 +3006,14 @@ Expected HefUtils::parse_proto_nms_info(const ProtoHEFNmsInfo if (burst_mode_enabled) { nms_info.burst_size = static_cast(proto_nms_info.burst_size()); - - auto burst_type = get_nms_burst_mode(proto_nms_info, hef_arch); - CHECK_EXPECTED(burst_type); - nms_info.burst_type = *burst_type; - + TRY(nms_info.burst_type, get_nms_burst_mode(proto_nms_info, hef_arch)); CHECK_AS_EXPECTED((nms_info.burst_size * nms_info.bbox_size) <= HailoRTCommon::MAX_NMS_BURST_SIZE, HAILO_INVALID_HEF, "Invalid HEF, nms burst size {} larger than maximum burst size {}", (nms_info.burst_size * nms_info.bbox_size), HailoRTCommon::MAX_NMS_BURST_SIZE); } else { // In case of bbox mode make burst size DEFAULT_NMS_NO_BURST_SIZE nms_info.burst_size = DEFAULT_NMS_NO_BURST_SIZE; - auto burst_type = get_nms_bbox_mode(proto_nms_info, hef_arch); - CHECK_EXPECTED(burst_type); - nms_info.burst_type = *burst_type; + TRY(nms_info.burst_type, get_nms_bbox_mode(proto_nms_info, hef_arch)); } if (nms_info.chunks_per_frame == 0) { @@ -2985,23 +3050,22 @@ Expected HefUtils::get_boundary_layer_info(const ProtoHEFCoreOpMock & HAILO_D2H_STREAM : HAILO_H2D_STREAM; auto support_multi_networks = supported_features.multi_network_support; auto network_index = static_cast((support_multi_networks) ? layer.network_index() : 0); - auto partial_network_name = HefUtils::get_partial_network_name_by_index(core_op, network_index, supported_features); - CHECK_EXPECTED(partial_network_name); + TRY(const auto partial_network_name, HefUtils::get_partial_network_name_by_index(core_op, network_index, supported_features)); if (ProtoHEFEdgeLayerType::PROTO__EDGE_LAYER_TYPE__INFO == layer.edge_layer_type()) { // TODO: return LayerInfo auto status = fill_layer_info(layer.layer_info(), layer.context_switch_info().edge_connection_type(), core_op, - direction, context_index, partial_network_name.value(), network_index, result, supported_features, hef_arch, false); + direction, context_index, partial_network_name, network_index, result, supported_features, hef_arch, false); CHECK_SUCCESS_AS_EXPECTED(status); } else if (ProtoHEFEdgeLayerType::PROTO__EDGE_LAYER_TYPE__MUX == layer.edge_layer_type()) { // TODO: return LayerInfo auto status = fill_mux_info(layer.layer_mux(), layer.context_switch_info().edge_connection_type(), core_op, - direction, context_index, partial_network_name.value(), network_index, result, supported_features, hef_arch); + direction, context_index, partial_network_name, network_index, result, supported_features, hef_arch); CHECK_SUCCESS_AS_EXPECTED(status); } else if (ProtoHEFEdgeLayerType::PROTO__EDGE_LAYER_TYPE__PLANES == layer.edge_layer_type()) { // TODO: return LayerInfo auto status = fill_planes_info(layer.layer_planes(), layer.context_switch_info().edge_connection_type(), core_op, - direction, context_index, partial_network_name.value(), network_index, result, supported_features, hef_arch); + direction, context_index, partial_network_name, network_index, result, supported_features, hef_arch); CHECK_SUCCESS_AS_EXPECTED(status); } else { LOGGER__ERROR("Invalid layer type"); @@ -3044,18 +3108,15 @@ Expected HefUtils::get_inter_context_layer_info(const ProtoHEFCoreOpM result.type = LayerType::INTER_CONTEXT; auto support_multi_networks = supported_features.multi_network_support; result.network_index = static_cast((support_multi_networks) ? layer.network_index() : 0); - auto partial_network_name = HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features); - CHECK_EXPECTED(partial_network_name); - result.network_name = HefUtils::get_network_name(core_op, partial_network_name.release()); + TRY(const auto partial_network_name, HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features)); + result.network_name = HefUtils::get_network_name(core_op, partial_network_name); result.context_index = context_index; result.name = layer.layer_info().name(); // Core hw padding is only supported on boundary layers const bool CORE_HW_PADDING_NOT_SUPPORTED = false; - auto nn_stream_config_exp = HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(), - CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type()); - CHECK_EXPECTED(nn_stream_config_exp); - result.nn_stream_config = nn_stream_config_exp.release(); + TRY(result.nn_stream_config, HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(), + CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type())); CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(layer.layer_info().edge_layer_base().sys_index()), HAILO_INVALID_HEF, "Failed to parse HEF. Invalid sys_index: {}.", layer.layer_info().edge_layer_base().sys_index()); result.stream_index = static_cast(layer.layer_info().edge_layer_base().sys_index()); @@ -3069,14 +3130,10 @@ Expected HefUtils::get_inter_context_layer_info(const ProtoHEFCoreOpM result.hw_shape = parse_layer_hw_shape(layer.layer_info().edge_layer_base(), CORE_HW_PADDING_NOT_SUPPORTED); result.hw_data_bytes = layer.layer_info().edge_layer_base().data_bytes(); - auto format_order_exp = HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format()); - CHECK_EXPECTED(format_order_exp); - auto format_oder = format_order_exp.release(); - result.format.order = format_oder; + TRY(result.format.order, + HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format())); result.format.flags = HAILO_FORMAT_FLAGS_NONE; - auto type = HailoRTCommon::get_format_type(result.hw_data_bytes); - CHECK_EXPECTED(type); - result.format.type = type.value(); + TRY(result.format.type, HailoRTCommon::get_format_type(result.hw_data_bytes)); result.direction = (ProtoHEFEdgeLayerDirection::PROTO__EDGE_LAYER_DIRECTION__DEVICE_TO_HOST == layer.direction()) ? HAILO_D2H_STREAM : HAILO_H2D_STREAM; @@ -3084,9 +3141,8 @@ Expected HefUtils::get_inter_context_layer_info(const ProtoHEFCoreOpM // HRT-7201 - The system supports one src and multiple dstinations. Right now we're saving only one dstination CHECK_AS_EXPECTED(layer.context_switch_info().connected_contexts_size() >= 1, HAILO_INVALID_HEF, "Inter context layer info must contain connected_context"); - auto connected_context = parse_connected_context_info(layer.context_switch_info().connected_contexts(0)); - CHECK_EXPECTED(connected_context); - result.connected_context_info = connected_context.release(); + TRY(result.connected_context_info, + parse_connected_context_info(layer.context_switch_info().connected_contexts(0))); return result; } @@ -3101,18 +3157,17 @@ Expected HefUtils::get_ddr_layer_info(const ProtoHEFCoreOpMock &core_ auto support_multi_networks = supported_features.multi_network_support; result.network_index = static_cast((support_multi_networks) ? layer.network_index() : 0); - auto partial_network_name = HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features); - CHECK_EXPECTED(partial_network_name); - result.network_name = HefUtils::get_network_name(core_op, partial_network_name.release()); + TRY(const auto partial_network_name, + HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features)); + result.network_name = HefUtils::get_network_name(core_op, partial_network_name); result.context_index = context_index; result.name = layer.layer_info().name(); // Core hw padding is only supported on boundary layers const bool CORE_HW_PADDING_NOT_SUPPORTED = false; - auto nn_stream_config_exp = HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(), - CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type()); - CHECK_EXPECTED(nn_stream_config_exp); - result.nn_stream_config = nn_stream_config_exp.release(); + TRY(result.nn_stream_config, + HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(), + CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type())); CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(layer.layer_info().edge_layer_base().sys_index()), HAILO_INVALID_HEF, "Failed to parse HEF. Invalid sys_index: {}.", layer.layer_info().edge_layer_base().sys_index()); result.stream_index = static_cast(layer.layer_info().edge_layer_base().sys_index()); @@ -3123,11 +3178,10 @@ Expected HefUtils::get_ddr_layer_info(const ProtoHEFCoreOpMock &core_ CHECK_AS_EXPECTED(layer.context_switch_info().connected_contexts_size() == 1, HAILO_INVALID_HEF, "Only single connected context is supported on DDR channels"); - auto connected_context = parse_connected_context_info(layer.context_switch_info().connected_contexts(0)); - CHECK_EXPECTED(connected_context); - CHECK_AS_EXPECTED(context_index == connected_context->context_index, + TRY(result.connected_context_info, + parse_connected_context_info(layer.context_switch_info().connected_contexts(0))); + CHECK_AS_EXPECTED(context_index == result.connected_context_info.context_index, HAILO_INVALID_HEF, "for ddr layer, connected_context_index must be same to the edge layer's context"); - result.connected_context_info = connected_context.release(); result.direction = (ProtoHEFEdgeLayerDirection::PROTO__EDGE_LAYER_DIRECTION__DEVICE_TO_HOST == layer.direction()) ? HAILO_D2H_STREAM : HAILO_H2D_STREAM; @@ -3136,14 +3190,10 @@ Expected HefUtils::get_ddr_layer_info(const ProtoHEFCoreOpMock &core_ result.hw_shape = parse_layer_hw_shape(layer.layer_info().edge_layer_base(), CORE_HW_PADDING_NOT_SUPPORTED); result.hw_data_bytes = layer.layer_info().edge_layer_base().data_bytes(); - auto format_order_exp = HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format()); - CHECK_EXPECTED(format_order_exp); - auto format_oder = format_order_exp.release(); - result.format.order = format_oder; + TRY(result.format.order, + HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format())); result.format.flags = HAILO_FORMAT_FLAGS_NONE; - auto type = HailoRTCommon::get_format_type(result.hw_data_bytes); - CHECK_EXPECTED(type); - result.format.type = type.value(); + TRY(result.format.type, HailoRTCommon::get_format_type(result.hw_data_bytes)); CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(layer.layer_info().edge_layer_base().core_buffers_per_frame()), HAILO_INVALID_HEF, "Failed to parse HEF. Invalid core_buffers_per_frame: {}.", layer.layer_info().edge_layer_base().core_buffers_per_frame()); @@ -3156,6 +3206,54 @@ Expected HefUtils::get_ddr_layer_info(const ProtoHEFCoreOpMock &core_ return result; } +Expected HefUtils::get_cache_layer_info( + const ProtoHEFCoreOpMock &core_op, const uint16_t context_index, + const ProtoHEFEdgeLayer &layer, const SupportedFeatures &supported_features) +{ + LayerInfo result{}; + CHECK_AS_EXPECTED(PROTO__EDGE_LAYER_TYPE__INFO == layer.edge_layer_type(), HAILO_INVALID_HEF, "Inter-context layer can't be mux."); + + result.type = LayerType::CACHE; + const auto support_multi_networks = supported_features.multi_network_support; + result.network_index = static_cast((support_multi_networks) ? layer.network_index() : 0); + TRY(const auto partial_network_name, HefUtils::get_partial_network_name_by_index(core_op, result.network_index, supported_features)); + result.network_name = HefUtils::get_network_name(core_op, partial_network_name); + result.context_index = context_index; + result.name = layer.layer_info().name(); + + // Core hw padding is only supported on boundary layers + const bool CORE_HW_PADDING_NOT_SUPPORTED = false; + TRY(result.nn_stream_config, HefConfigurator::parse_nn_stream_config(layer.layer_info().edge_layer_base(), + CORE_HW_PADDING_NOT_SUPPORTED, layer.context_switch_info().edge_connection_type())); + CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(layer.layer_info().edge_layer_base().sys_index()), HAILO_INVALID_HEF, + "Failed to parse HEF. Invalid sys_index: {}.", layer.layer_info().edge_layer_base().sys_index()); + result.stream_index = static_cast(layer.layer_info().edge_layer_base().sys_index()); + CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(layer.layer_info().edge_layer_base().engine_id()), HAILO_INVALID_HEF, + "Failed to parse HEF. Invalid engine_id: {}.", layer.layer_info().edge_layer_base().engine_id()); + result.dma_engine_index = static_cast(layer.layer_info().edge_layer_base().engine_id()); + + result.max_shmifo_size = layer.layer_info().edge_layer_base().max_shmifo_size(); + + result.shape = parse_layer_shape(layer.layer_info().edge_layer_base()); + result.hw_shape = parse_layer_hw_shape(layer.layer_info().edge_layer_base(), CORE_HW_PADDING_NOT_SUPPORTED); + result.hw_data_bytes = layer.layer_info().edge_layer_base().data_bytes(); + + TRY(result.format.order, + HailoRTDefaults::get_device_format_order(layer.layer_info().edge_layer_base().format())); + result.format.flags = HAILO_FORMAT_FLAGS_NONE; + TRY(result.format.type, HailoRTCommon::get_format_type(result.hw_data_bytes)); + + result.direction = (ProtoHEFEdgeLayerDirection::PROTO__EDGE_LAYER_DIRECTION__DEVICE_TO_HOST == + layer.direction()) ? HAILO_D2H_STREAM : HAILO_H2D_STREAM; + + // Negative cache_id means that the cache is not used + const int32_t cache_id = layer.context_switch_info().cache_id(); + CHECK_AS_EXPECTED(cache_id >= 0, HAILO_INVALID_HEF, "Invalid cache_id: {}", cache_id); + result.cache_info.id = static_cast(cache_id); + + return result; +} + Expected HefUtils::get_partial_network_name_by_index(const ProtoHEFCoreOpMock &core_op, uint8_t network_index, const SupportedFeatures &supported_features) { @@ -3233,9 +3331,8 @@ static Expected parse_ccw_buffer(const std::string &ccw_buffer) CHECK_AS_EXPECTED(data_length == expected_ccw_data_length, HAILO_INVALID_HEF, "Invalid ccw buffer was parsed from HEF"); - auto data_buff = Buffer::create(reinterpret_cast(ccw_buffer.data() + CCW_DATA_OFFSET), data_length); - CHECK_EXPECTED(data_buff); - write_memory_info.data = data_buff.release(); + TRY(write_memory_info.data, + Buffer::create(reinterpret_cast(ccw_buffer.data() + CCW_DATA_OFFSET), data_length)); return write_memory_info; } @@ -3265,18 +3362,15 @@ Expected> Hef::Impl::create_single_context_core_op_ case ProtoHEFAction::kWriteData: { WriteMemoryInfo write_memory_info = {}; write_memory_info.address = static_cast(action.write_data().address()); - auto data_buff = Buffer::create( - reinterpret_cast(action.write_data().data().data()), - action.write_data().data().length()); - CHECK_EXPECTED(data_buff); - write_memory_info.data = data_buff.release(); + TRY(write_memory_info.data, + Buffer::create(reinterpret_cast(action.write_data().data().data()), + action.write_data().data().length())); config_buffers.emplace_back(std::move(write_memory_info)); break; } case ProtoHEFAction::kWriteDataCcw: { - auto config_buffer = parse_ccw_buffer(action.write_data_ccw().data()); // TODO: make this not supported in sHEF - CHECK_EXPECTED(config_buffer); - config_buffers.emplace_back(config_buffer.release()); + TRY(auto config_buffer, parse_ccw_buffer(action.write_data_ccw().data())); // TODO: make this not supported in sHEF + config_buffers.emplace_back(std::move(config_buffer)); break; } case ProtoHEFAction::kDisableLcu: { @@ -3286,9 +3380,8 @@ Expected> Hef::Impl::create_single_context_core_op_ case ProtoHEFAction::kEnableLcu: { WriteMemoryInfo write_memory_info = {}; write_memory_info.address = action.enable_lcu().lcu_enable_address(); - auto data_buff = Buffer::create(ENABLE_LCU_CONTROL_WORD, sizeof(ENABLE_LCU_CONTROL_WORD)); - CHECK_EXPECTED(data_buff); - write_memory_info.data = data_buff.release(); + TRY(write_memory_info.data, + Buffer::create(ENABLE_LCU_CONTROL_WORD, sizeof(ENABLE_LCU_CONTROL_WORD))); config_buffers.emplace_back(std::move(write_memory_info)); break; } @@ -3327,16 +3420,15 @@ ProtoHEFHwArch Hef::Impl::get_device_arch() return m_header.hw_arch(); } -std::shared_ptr Hef::Impl::get_shef_file_handle() +std::shared_ptr Hef::Impl::get_hef_reader() { - return m_shef_file_handle; + return m_hef_reader; } Expected Hef::Impl::get_bottleneck_fps(const std::string &net_group_name) { - auto core_op = get_core_op_by_net_group_name(net_group_name); - CHECK_EXPECTED(core_op); - return core_op.value()->network_group_metadata.bottleneck_fps(); + TRY(const auto core_op, get_core_op_by_net_group_name(net_group_name)); + return core_op->network_group_metadata.bottleneck_fps(); } bool Hef::Impl::contains_ddr_layers(const ProtoHEFCoreOpMock& core_op) @@ -3393,12 +3485,11 @@ Expected Hef::Impl::get_vstream_name_from_original_name_mux(const s Expected Hef::Impl::get_vstream_name_from_original_name(const std::string &original_name, const std::string &net_group_name) { - auto core_op = get_core_op_by_net_group_name(net_group_name); - CHECK_EXPECTED(core_op); + TRY(const auto core_op, get_core_op_by_net_group_name(net_group_name)); std::string results; - for (const auto &context : core_op.value()->contexts) { + for (const auto &context : core_op->contexts) { for (const auto &layer_info : context.metadata().edge_layers()) { if ((is_h2d_boundary_info_layer(layer_info)) || (is_d2h_boundary_info_layer(layer_info))) { for (auto &name : layer_info.layer_info().original_names()) { @@ -3461,12 +3552,11 @@ Expected> Hef::Impl::get_original_names_from_vstream_na Expected> Hef::Impl::get_original_names_from_vstream_name(const std::string &vstream_name, const std::string &net_group_name) { - auto copre_op = get_core_op_by_net_group_name(net_group_name); - CHECK_EXPECTED(copre_op); + TRY(const auto copre_op, get_core_op_by_net_group_name(net_group_name)); std::vector results; - for (const auto &context : copre_op.value()->contexts) { + for (const auto &context : copre_op->contexts) { for (const auto &layer_info : context.metadata().edge_layers()) { if ((is_h2d_boundary_info_layer(layer_info)) || (is_d2h_boundary_info_layer(layer_info))) { if (vstream_name == layer_info.layer_info().name()) { @@ -3535,9 +3625,8 @@ Expected Hef::create_configure_params(hailo_stream_inter { NetworkGroupsParamsMap results; for (const auto &name : pimpl->get_network_groups_names()) { - auto params = create_configure_params(stream_interface, name); - CHECK_EXPECTED(params); - results.emplace(std::make_pair(name, params.release())); + TRY(auto params, create_configure_params(stream_interface, name)); + results.emplace(std::make_pair(name, params)); } return results; } @@ -3552,9 +3641,8 @@ Expected Hef::create_configure_params_mipi_input(hailo_s { NetworkGroupsParamsMap results; for (const auto &name : pimpl->get_network_groups_names()) { - auto params = create_configure_params_mipi_input(output_interface, mipi_params, name); - CHECK_EXPECTED(params); - results.emplace(std::make_pair(name, params.release())); + TRY(auto params, create_configure_params_mipi_input(output_interface, mipi_params, name)); + results.emplace(std::make_pair(name, params)); } return results; } @@ -3568,9 +3656,9 @@ Expected Hef::create_configure_params_mipi_input(hailo_s std::string Hef::hash() const { - const auto &md5 = pimpl->md5(); + const auto &hash = pimpl->get_hash_as_memview(); const bool LOWERCASE = false; - return StringUtils::to_hex_string(md5, MD5_DIGEST_LENGTH, LOWERCASE); + return StringUtils::to_hex_string(hash.data(), hash.size(), LOWERCASE); } std::vector Hef::Impl::get_network_groups_names() @@ -3595,20 +3683,22 @@ Expected> Hef::get_network_groups_infos( Expected> Hef::Impl::get_stream_infos_description(const std::string &network_group_name, const std::string &network_name) { std::vector infos_strings; - auto input_stream_infos = get_input_stream_infos(network_group_name, network_name); - CHECK_EXPECTED(input_stream_infos, "Failed to parse input stream infos"); - auto output_stream_infos = get_output_stream_infos(network_group_name, network_name); - CHECK_EXPECTED(output_stream_infos, "Failed to parse output stream infos"); - infos_strings.reserve(input_stream_infos.value().size() + output_stream_infos.value().size()); + TRY(const auto input_stream_infos, + get_input_stream_infos(network_group_name, network_name), + "Failed to parse input stream infos"); + TRY(const auto output_stream_infos, + get_output_stream_infos(network_group_name, network_name), + "Failed to parse output stream infos"); + infos_strings.reserve(input_stream_infos.size() + output_stream_infos.size()); std::string infos_string; - for (const auto &stream_info : input_stream_infos.value()) { + for (const auto &stream_info : input_stream_infos) { auto shape_str = get_shape_str(stream_info); infos_string = "Input " + std::string(stream_info.name) + " " + shape_str + "\n"; infos_strings.emplace_back(infos_string); } - for (const auto &stream_info : output_stream_infos.value()) { + for (const auto &stream_info : output_stream_infos) { auto shape_str = get_shape_str(stream_info); infos_string = "Output " + std::string(stream_info.name) + " " + shape_str + "\n"; infos_strings.emplace_back(infos_string); @@ -3620,20 +3710,22 @@ Expected> Hef::Impl::get_stream_infos_description(const Expected> Hef::Impl::get_vstream_infos_description(const std::string &network_group_name, const std::string &network_name) { std::vector infos_strings; - auto input_vstream_infos = get_input_vstream_infos(network_group_name, network_name); - CHECK_EXPECTED(input_vstream_infos, "Failed to parse input vstream infos"); - auto output_vstream_infos = get_output_vstream_infos(network_group_name, network_name); - CHECK_EXPECTED(output_vstream_infos, "Failed to parse output stream infos"); - infos_strings.reserve(input_vstream_infos.value().size() + output_vstream_infos.value().size()); + TRY(const auto input_vstream_infos, + get_input_vstream_infos(network_group_name, network_name), + "Failed to parse input vstream infos"); + TRY(const auto output_vstream_infos, + get_output_vstream_infos(network_group_name, network_name), + "Failed to parse output vstream infos"); + infos_strings.reserve(input_vstream_infos.size() + output_vstream_infos.size()); std::string infos_string; - for (const auto &vstream_info : input_vstream_infos.value()) { + for (const auto &vstream_info : input_vstream_infos) { auto shape_str = get_shape_str(vstream_info); infos_string = "Input " + std::string(vstream_info.name) + " " + shape_str + "\n"; infos_strings.emplace_back(infos_string); } - for (const auto &vstream_info : output_vstream_infos.value()) { + for (const auto &vstream_info : output_vstream_infos) { auto shape_str = get_shape_str(vstream_info); infos_string = "Output " + std::string(vstream_info.name) + " " + shape_str + "\n"; infos_strings.emplace_back(infos_string); @@ -3675,9 +3767,8 @@ Expected> Hef::Impl::get_post_processes_infos_descripti Expected Hef::get_description(bool stream_infos, bool vstream_infos) const { - auto arch = get_hef_device_arch(); - CHECK_EXPECTED(arch); - return pimpl->get_description(stream_infos, vstream_infos, arch.value()); + TRY(const auto arch, get_hef_device_arch()); + return pimpl->get_description(stream_infos, vstream_infos, arch); } Expected Hef::Impl::get_description(bool stream_infos, bool vstream_infos, hailo_device_architecture_t device_arch) @@ -3686,46 +3777,44 @@ Expected Hef::Impl::get_description(bool stream_infos, bool vstream auto hef_arch_str = HailoRTCommon::get_device_arch_str(device_arch); hef_infos += "Architecture HEF was compiled for: " + hef_arch_str + "\n"; - auto network_group_infos = get_network_groups_infos(); - CHECK_EXPECTED(network_group_infos); - for (const auto &network_group_info : network_group_infos.release()) { - auto core_op_metadata = get_core_op_metadata(network_group_info.name); - CHECK_EXPECTED(core_op_metadata); - const auto number_of_dynamic_contexts = core_op_metadata.value()->get_dynamic_contexts_count(); + TRY(const auto network_group_infos, get_network_groups_infos()); + for (const auto &network_group_info : network_group_infos) { + TRY(const auto core_op_metadata, get_core_op_metadata(network_group_info.name)); + const auto number_of_dynamic_contexts = core_op_metadata->get_dynamic_contexts_count(); auto contexts_str = network_group_info.is_multi_context ? "Multi Context - Number of contexts: " + std::to_string(number_of_dynamic_contexts) : "Single Context"; hef_infos += "Network group name: " + std::string(network_group_info.name) + ", " + contexts_str + "\n"; - auto network_infos = get_network_infos(network_group_info.name); - CHECK_EXPECTED(network_infos, "Failed to parse networks infos"); + TRY(const auto network_infos, get_network_infos(network_group_info.name), + "Failed to parse networks infos"); - for (const auto &network_info : network_infos.value()) { + for (const auto &network_info : network_infos) { hef_infos += add_tabs(1) + "Network name: " + network_info.name + "\n"; if (stream_infos) { - auto stream_infos_strings = get_stream_infos_description(network_group_info.name, network_info.name); - CHECK_EXPECTED(stream_infos_strings); + TRY(const auto stream_infos_strings, + get_stream_infos_description(network_group_info.name, network_info.name)); hef_infos += add_tabs(2) + "Stream infos:" + "\n"; - for (auto stream_info_string : stream_infos_strings.value()) { + for (auto stream_info_string : stream_infos_strings) { hef_infos += add_tabs(3) + stream_info_string; } } if (vstream_infos) { - auto vstream_infos_strings = get_vstream_infos_description(network_group_info.name, network_info.name); - CHECK_EXPECTED(vstream_infos_strings); + TRY(const auto vstream_infos_strings, + get_vstream_infos_description(network_group_info.name, network_info.name)); hef_infos += add_tabs(2) + "VStream infos:" + "\n"; - for (auto vstream_info_string : vstream_infos_strings.value()) { + for (auto vstream_info_string : vstream_infos_strings) { hef_infos += add_tabs(3) + vstream_info_string; } - auto post_processes_infos_strings = get_post_processes_infos_description(network_group_info.name); - CHECK_EXPECTED(post_processes_infos_strings); + TRY(const auto post_processes_infos_strings, + get_post_processes_infos_description(network_group_info.name)); /* Validating that there is a postprocess info. */ - if (post_processes_infos_strings->size() <= 0) { + if (post_processes_infos_strings.size() <= 0) { continue; } hef_infos += add_tabs(3) + "Operation:" + "\n"; - for (auto post_process_info_string : post_processes_infos_strings.value()) { + for (auto post_process_info_string : post_processes_infos_strings) { hef_infos += add_tabs(4) + post_process_info_string; } } @@ -3761,10 +3850,8 @@ Expected> Hef::make_input_vstream_ const std::string &name, bool /*unused*/, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) { - auto network_pair = pimpl->get_network_group_and_network_name(name); - CHECK_EXPECTED(network_pair); - - return pimpl->make_input_vstream_params(network_pair.value().first, network_pair.value().second, format_type, + TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name)); + return pimpl->make_input_vstream_params(network_pair.first, network_pair.second, format_type, timeout_ms, queue_size); } @@ -3784,10 +3871,8 @@ Expected> Hef::make_output_vstream const std::string &name, bool /*unused*/, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) { - auto network_pair = pimpl->get_network_group_and_network_name(name); - CHECK_EXPECTED(network_pair); - - return pimpl->make_output_vstream_params(network_pair.value().first, network_pair.value().second, format_type, + TRY(const auto network_pair, pimpl->get_network_group_and_network_name(name)); + return pimpl->make_output_vstream_params(network_pair.first, network_pair.second, format_type, timeout_ms, queue_size); } @@ -3808,10 +3893,10 @@ hailo_status Hef::Impl::fill_missing_input_vstream_params_with_default(const std hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) { CHECK(contains(m_network_group_metadata, net_group_name), HAILO_NOT_FOUND); - auto input_vstream_infos = m_network_group_metadata.at(net_group_name).get_input_vstream_infos(network_name); - CHECK_EXPECTED_AS_STATUS(input_vstream_infos); + TRY(const auto input_vstream_infos, + m_network_group_metadata.at(net_group_name).get_input_vstream_infos(network_name)); - return fill_missing_vstream_params_with_default(input_vstreams_params, input_vstream_infos.value(), + return fill_missing_vstream_params_with_default(input_vstreams_params, input_vstream_infos, format_type, timeout_ms, queue_size); } @@ -3820,15 +3905,15 @@ hailo_status Hef::Impl::fill_missing_output_vstream_params_with_default(const st hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) { CHECK(contains(m_network_group_metadata, net_group_name), HAILO_NOT_FOUND); - auto output_vstream_infos = m_network_group_metadata.at(net_group_name).get_output_vstream_infos(network_name); - CHECK_EXPECTED_AS_STATUS(output_vstream_infos); + TRY(const auto output_vstream_infos, + m_network_group_metadata.at(net_group_name).get_output_vstream_infos(network_name)); - return fill_missing_vstream_params_with_default(output_vstream_params, output_vstream_infos.value(), + return fill_missing_vstream_params_with_default(output_vstream_params, output_vstream_infos, format_type, timeout_ms, queue_size); } hailo_status Hef::Impl::fill_missing_vstream_params_with_default(std::map &vstream_params, - std::vector &vstream_infos, hailo_format_type_t format_type, uint32_t timeout_ms, + const std::vector &vstream_infos, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) { hailo_format_flags_t flags = HAILO_FORMAT_FLAGS_NONE; @@ -3852,12 +3937,10 @@ hailo_status Hef::Impl::fill_missing_vstream_params_with_default(std::map Hef::Impl::create_configure_params(hailo_stream_interface_t stream_interface, const std::string &network_group_name) { auto params = HailoRTDefaults::get_configure_params(); - auto stream_params_by_name = create_stream_parameters_by_name(network_group_name, stream_interface); - CHECK_EXPECTED(stream_params_by_name); - params.stream_params_by_name = stream_params_by_name.release(); - auto network_params_by_name = create_network_parameters_by_name(network_group_name); - CHECK_EXPECTED(network_params_by_name); - params.network_params_by_name = network_params_by_name.release(); + TRY(params.stream_params_by_name, + create_stream_parameters_by_name(network_group_name, stream_interface)); + TRY(params.network_params_by_name, + create_network_parameters_by_name(network_group_name)); return params; } @@ -3866,12 +3949,10 @@ Expected Hef::Impl::create_configure_params_mipi_input(h const hailo_mipi_input_stream_params_t &mipi_params, const std::string &network_group_name) { auto params = HailoRTDefaults::get_configure_params(); - auto stream_params_by_name = create_stream_parameters_by_name_mipi_input(network_group_name, output_interface, mipi_params); - CHECK_EXPECTED(stream_params_by_name); - params.stream_params_by_name = stream_params_by_name.release(); - auto network_params_by_name = create_network_parameters_by_name(network_group_name); - CHECK_EXPECTED(network_params_by_name); - params.network_params_by_name = network_params_by_name.release(); + TRY(params.stream_params_by_name, + create_stream_parameters_by_name_mipi_input(network_group_name, output_interface, mipi_params)); + TRY(params.network_params_by_name, + create_network_parameters_by_name(network_group_name)); return params; } @@ -3879,9 +3960,9 @@ Expected Hef::Impl::create_configure_params_mipi_input(h Expected> Hef::create_stream_parameters_by_name( const std::string &net_group_name, hailo_stream_interface_t stream_interface) { - auto network_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(network_group_name_pair); - auto net_group_name_str = network_group_name_pair->first; + TRY(const auto network_group_name_pair, + pimpl->get_network_group_and_network_name(net_group_name)); + const auto &net_group_name_str = network_group_name_pair.first; return pimpl->create_stream_parameters_by_name(net_group_name_str, stream_interface); } @@ -3889,23 +3970,22 @@ Expected> Hef::create_stream_pa Expected> Hef::Impl::create_stream_parameters_by_name( const std::string &net_group_name, hailo_stream_interface_t stream_interface) { - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); + TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name)); std::map results; - auto input_stream_infos = core_op_metadata.value()->get_input_stream_infos(); - CHECK_EXPECTED(input_stream_infos); - for (auto &input_layer : input_stream_infos.value()) { - auto params = HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_H2D_STREAM); - CHECK_EXPECTED(params); - results.emplace(std::make_pair(input_layer.name, params.release())); + TRY(const auto input_stream_infos, + core_op_metadata->get_input_stream_infos()); + for (const auto &input_layer : input_stream_infos) { + TRY(auto params, + HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_H2D_STREAM)); + results.emplace(std::make_pair(input_layer.name, params)); } - auto output_stream_infos = core_op_metadata.value()->get_output_stream_infos(); - CHECK_EXPECTED(output_stream_infos); - for (auto &output_layer : output_stream_infos.value()) { - auto params = HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_D2H_STREAM); - CHECK_EXPECTED(params); - results.emplace(std::make_pair(output_layer.name, params.release())); + TRY(const auto output_stream_infos, + core_op_metadata->get_output_stream_infos()); + for (const auto &output_layer : output_stream_infos) { + TRY(auto params, + HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_D2H_STREAM)); + results.emplace(std::make_pair(output_layer.name, params)); } return results; @@ -3920,18 +4000,15 @@ Expected> Hef::create_network_ Expected> Hef::Impl::create_network_parameters_by_name( const std::string &net_group_name) { - auto core_op = get_core_op_by_net_group_name(net_group_name); - CHECK_EXPECTED(core_op); - - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); + TRY(const auto core_op, get_core_op_by_net_group_name(net_group_name)); + TRY(const auto core_op_metadata, get_core_op_metadata(net_group_name)); std::map results; - if (core_op_metadata.value()->supported_features().multi_network_support) { - CHECK_AS_EXPECTED((core_op.value()->networks_names.size() != 0), HAILO_INTERNAL_FAILURE, + if (core_op_metadata->supported_features().multi_network_support) { + CHECK_AS_EXPECTED((core_op->networks_names.size() != 0), HAILO_INTERNAL_FAILURE, "Hef support multiple networks, but no networks found in the proto"); - for (const auto &partial_network_name : core_op.value()->networks_names) { + for (const auto &partial_network_name : core_op->networks_names) { auto network_name = HefUtils::get_network_name(net_group_name, partial_network_name); auto params = HailoRTDefaults::get_network_parameters(); results.emplace(std::make_pair(network_name, params)); @@ -3950,9 +4027,9 @@ Expected> Hef::create_stream_pa const std::string &net_group_name, hailo_stream_interface_t output_interface, const hailo_mipi_input_stream_params_t &mipi_params) { - auto network_group_name_pair = pimpl->get_network_group_and_network_name(net_group_name); - CHECK_EXPECTED(network_group_name_pair); - auto net_group_name_str = network_group_name_pair->first; + TRY(const auto network_group_name_pair, + pimpl->get_network_group_and_network_name(net_group_name)); + const auto &net_group_name_str = network_group_name_pair.first; return pimpl->create_stream_parameters_by_name_mipi_input(net_group_name_str, output_interface, mipi_params); } @@ -3961,25 +4038,24 @@ Expected> Hef::Impl::create_str const std::string &net_group_name, hailo_stream_interface_t output_interface, const hailo_mipi_input_stream_params_t &mipi_params) { - auto core_op_metadata = get_core_op_metadata(net_group_name); - CHECK_EXPECTED(core_op_metadata); + TRY(const auto core_op_metadata, + get_core_op_metadata(net_group_name)); std::map results; - auto input_stream_infos = core_op_metadata.value()->get_input_stream_infos(); - CHECK_EXPECTED(input_stream_infos); - for (auto &input_layer : input_stream_infos.value()) { + TRY(const auto input_stream_infos, + core_op_metadata->get_input_stream_infos()); + for (auto &input_layer : input_stream_infos) { hailo_stream_parameters_t params = {}; params.direction = HAILO_H2D_STREAM; params.stream_interface = HAILO_STREAM_INTERFACE_MIPI; params.mipi_input_params = mipi_params; results.emplace(std::make_pair(input_layer.name, params)); } - auto output_stream_infos = core_op_metadata.value()->get_output_stream_infos(); - CHECK_EXPECTED(output_stream_infos); - for (auto &output_layer : output_stream_infos.value()) { - auto params = HailoRTDefaults::get_stream_parameters(output_interface, HAILO_D2H_STREAM); - CHECK_EXPECTED(params); - results.emplace(std::make_pair(output_layer.name, params.release())); + TRY(const auto output_stream_infos, + core_op_metadata->get_output_stream_infos()); + for (auto &output_layer : output_stream_infos) { + TRY(auto params, HailoRTDefaults::get_stream_parameters(output_interface, HAILO_D2H_STREAM)); + results.emplace(std::make_pair(output_layer.name, params)); } return results; diff --git a/hailort/libhailort/src/hef/hef_internal.hpp b/hailort/libhailort/src/hef/hef_internal.hpp index 7e6d91a..ad11dde 100644 --- a/hailort/libhailort/src/hef/hef_internal.hpp +++ b/hailort/libhailort/src/hef/hef_internal.hpp @@ -39,6 +39,8 @@ #include "net_flow/ops/op.hpp" #include "device_common/control_protocol.hpp" +#include "common/file_utils.hpp" + #include "control_protocol.h" #include #include @@ -111,21 +113,31 @@ struct ProtoHEFCoreOpMock { }; #pragma pack(push, 1) +// TODO HRT-13921: change structure of hef header types + +typedef union { + struct { + uint32_t reserved; + MD5_SUM_t expected_md5; + } v0; + struct { + uint32_t crc; + uint64_t ccws_size; + uint32_t reserved; + } v1; +} hef__header_distinct_t; + typedef struct { uint32_t magic; uint32_t version; uint32_t hef_proto_size; - uint32_t ccws_size; - MD5_SUM_t expected_md5; + hef__header_distinct_t distinct; } hef__header_t; #pragma pack(pop) -#pragma pack(push, 1) -typedef struct { - uint32_t offset; - uint32_t size; -} shef__ccw_offset_t; -#pragma pack(pop) +static const size_t HEF_COMMON_SIZE = sizeof(hef__header_t) - sizeof(hef__header_distinct_t); +static const size_t HEF_HEADER_SIZE_V0 = HEF_COMMON_SIZE + sizeof(hef__header_distinct_t::v0); +static const size_t HEF_HEADER_SIZE_V1 = HEF_COMMON_SIZE + sizeof(hef__header_distinct_t::v1); typedef enum { HEF__FORMAT__TF_RGB = 0, @@ -147,6 +159,10 @@ typedef enum { HAILO_NET_FLOW_OP_TYPE_MAX_ENUM = HAILO_MAX_ENUM } hailo_net_flow_op_type_t; +#define HEADER_MAGIC (0x01484546) +#define HEADER_VERSION_0 (0) +#define HEADER_VERSION_1 (1) + const static uint32_t SUPPORTED_EXTENSIONS_BITSET_SIZE = 1000; static const std::vector SUPPORTED_EXTENSIONS = { ABBALE, @@ -230,26 +246,9 @@ class VdmaConfigCoreOp; class VdmaDevice; class HailoRTDriver; -class ShefFileHandle final -{ -public: - ShefFileHandle(const std::string &hef_path, uint32_t ccws_buffer_offset); - hailo_status open(); - Expected read(uint32_t offset, size_t size); - hailo_status close(); - -private: - std::string m_hef_path; - std::ifstream m_hef_file; - uint32_t m_ccws_buffer_offset; -}; - class Hef::Impl final { public: - static const uint32_t HEADER_MAGIC = 0x01484546; - static const uint32_t HEADER_VERSION_0 = 0; // Old HEF - static const uint32_t HEADER_VERSION_1 = 1; // New HEF (SHEF) static Expected create(const std::string &hef_path); static Expected create(const MemoryView &hef_buffer); @@ -284,7 +283,7 @@ public: Expected get_number_of_input_streams(const std::string &net_group_name=""); Expected get_number_of_output_streams(const std::string &net_group_name=""); ProtoHEFHwArch get_device_arch(); - std::shared_ptr get_shef_file_handle(); + std::shared_ptr get_hef_reader(); Expected get_bottleneck_fps(const std::string &net_group_name=""); static bool contains_ddr_layers(const ProtoHEFCoreOpMock &core_op); static hailo_status validate_core_op_unique_layer_names(const ProtoHEFCoreOpMock &core_op); @@ -337,7 +336,7 @@ public: const std::string &network_name, std::map &output_vstream_params, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size); static hailo_status fill_missing_vstream_params_with_default(std::map &vstream_params, - std::vector &name_to_format_info, hailo_format_type_t format_type, uint32_t timeout_ms, + const std::vector &vstream_infos, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size); // Also adds information to CoreOpMetadata // TODO: When supporting multiple core ops in same netflow - Change metadata param to a map of core_ops_metadata. @@ -349,9 +348,13 @@ public: Expected get_description(bool stream_infos, bool vstream_infos, hailo_device_architecture_t device_arch); - const MD5_SUM_t &md5() const + const MemoryView get_hash_as_memview() const { - return m_md5; + if (HEADER_VERSION_0 == m_hef_version){ + return MemoryView::create_const(m_md5, sizeof(m_md5)); + } else { // HEADER_VERSION_1 + return MemoryView::create_const(&m_crc, sizeof(m_crc)); + } } static hailo_status update_network_batch_size(ConfigureNetworkParams &network_group_config_params) @@ -398,11 +401,18 @@ private: hailo_status parse_hef_file(const std::string &hef_path); hailo_status parse_hef_memview(const MemoryView &hef_memview); + hailo_status parse_hef_memview_internal(const size_t proto_size, const uint8_t *proto_buffer, const uint32_t hef_version, + std::shared_ptr hef_reader, size_t ccws_offset); + Expected parse_hef_header_before_distinct(std::shared_ptr hef_reader); + hailo_status fill_v1_hef_header(hef__header_t &hef_header, std::shared_ptr hef_reader); + hailo_status fill_core_ops_and_networks_metadata(uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset); hailo_status transfer_protobuf_field_ownership(ProtoHEFHef &hef_message); void fill_core_ops(); - hailo_status fill_networks_metadata(); + hailo_status fill_networks_metadata(uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset); void fill_extensions_bitset(); void init_md5(MD5_SUM_t &calculated_md5); + void init_crc(uint32_t crc_32); + void init_hef_version(uint32_t version); static bool check_hef_extension(const ProtoHEFExtensionType &extension, const ProtoHEFHeader &header, const std::vector &hef_extensions, const ProtoHEFIncludedFeatures &included_features); @@ -415,6 +425,8 @@ private: hailo_status validate_hef_extensions(); static hailo_status validate_hef_header(const hef__header_t &header, MD5_SUM_t &calculated_md5, size_t proto_size); + static hailo_status validate_hef_header(const hef__header_t &header, const uint32_t &crc_32, size_t hef_file_residue_size); + Expected> get_inputs_vstream_names_and_format_info( const std::string &net_group_name, const std::string &network_name); @@ -424,7 +436,8 @@ private: static Expected get_vstream_name_from_original_name_mux(const std::string &original_name, const ProtoHefEdge &layer); static Expected> get_original_names_from_vstream_name_mux(const std::string &vstream_name, const ProtoHefEdge &layer); - Expected create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op, const std::vector &sorted_network_names); // TODO: Remove sorted_network_names + Expected create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op, const std::vector &sorted_network_names, + uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset); // TODO: Remove sorted_network_names Expected> get_stream_infos_description(const std::string &network_group_name, const std::string &network_name); Expected> get_vstream_infos_description(const std::string &network_group_name, const std::string &network_name); Expected> get_post_processes_infos_description(const std::string &network_group_name); @@ -439,8 +452,10 @@ private: std::vector m_hef_extensions; std::vector m_hef_optional_extensions; std::bitset m_supported_extensions_bitset; + uint32_t m_hef_version; MD5_SUM_t m_md5; - std::shared_ptr m_shef_file_handle; + uint32_t m_crc; + std::shared_ptr m_hef_reader; #ifdef HAILO_SUPPORT_MULTI_PROCESS Buffer m_hef_buffer; @@ -502,13 +517,23 @@ public: const std::vector &context_ddr_input_layers, const std::vector &context_ddr_output_layers, const uint16_t context_index); + static hailo_status fill_cache_layers_info( + const ProtoHEFCoreOpMock &core_op, + const uint16_t context_index, + const ProtoHEFEdgeLayer &layer, + const SupportedFeatures &supported_features, + ContextMetadata &context_metadata); + static Expected get_cache_layer_info( + const ProtoHEFCoreOpMock &core_op, const uint16_t context_index, + const ProtoHEFEdgeLayer &layer, const SupportedFeatures &supported_features); static Expected parse_preliminary_context(const ProtoHEFPreliminaryConfig &preliminary_proto, - const SupportedFeatures &supported_features, std::shared_ptr shef_file_handle); + const SupportedFeatures &supported_features, const uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset); static Expected parse_single_dynamic_context(const ProtoHEFCoreOpMock &core_op, const ProtoHEFContext &context_proto, uint16_t context_index, const SupportedFeatures &supported_features, - const ProtoHEFHwArch &hef_arch, std::shared_ptr shef_file_handle); + const ProtoHEFHwArch &hef_arch, const uint32_t hef_version, std::shared_ptr hef_reader, size_t ccws_offset); static Expected> parse_dynamic_contexts(const ProtoHEFCoreOpMock &core_op, - const SupportedFeatures &supported_features, const ProtoHEFHwArch &hef_arch, std::shared_ptr shef_file_handle); + const SupportedFeatures &supported_features, const ProtoHEFHwArch &hef_arch, const uint32_t hef_version, + std::shared_ptr hef_reader, size_t ccws_offset); static Expected parse_proto_nms_info(const ProtoHEFNmsInfo &proto_nms_info, const bool burst_mode_enabled, const ProtoHEFHwArch &hef_arch); static Expected get_boundary_layer_info(const ProtoHEFCoreOpMock &core_op, diff --git a/hailort/libhailort/src/hef/layer_info.hpp b/hailort/libhailort/src/hef/layer_info.hpp index 5e996be..e2d16f0 100644 --- a/hailort/libhailort/src/hef/layer_info.hpp +++ b/hailort/libhailort/src/hef/layer_info.hpp @@ -36,7 +36,8 @@ enum class LayerType BOUNDARY = 1, INTER_CONTEXT = 2, DDR = 3, - CFG = 4 + CFG = 4, + CACHE = 5 }; struct BufferIndices { @@ -57,6 +58,11 @@ struct DdrInfo { uint16_t min_buffered_rows; }; +struct CacheInfo { + uint32_t id; + uint32_t size; +}; + struct LayerInfo { LayerType type = LayerType::NOT_SET; hailo_stream_direction_t direction; @@ -101,6 +107,7 @@ struct LayerInfo { // Context switch info TODO: we should use std::optional for this structures (or implement our self). ConnectedContextInfo connected_context_info; DdrInfo ddr_info; + CacheInfo cache_info; }; // LayerIdentifier = diff --git a/hailort/libhailort/src/net_flow/CMakeLists.txt b/hailort/libhailort/src/net_flow/CMakeLists.txt index 8c9f7a6..abf393a 100644 --- a/hailort/libhailort/src/net_flow/CMakeLists.txt +++ b/hailort/libhailort/src/net_flow/CMakeLists.txt @@ -10,6 +10,7 @@ set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/ops/softmax_post_process.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ops/yolov5_seg_post_process.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ops/yolov8_post_process.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ops/yolov8_bbox_only_post_process.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/pipeline.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/pipeline_internal.cpp @@ -21,6 +22,8 @@ set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/async_pipeline_builder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/async_infer_runner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/infer_model.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/infer_model_hrpc_client.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/configured_infer_model_hrpc_client.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/vstream_builder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/vstream.cpp diff --git a/hailort/libhailort/src/net_flow/ops/softmax_post_process.cpp b/hailort/libhailort/src/net_flow/ops/softmax_post_process.cpp index 9e40fef..8e07a68 100644 --- a/hailort/libhailort/src/net_flow/ops/softmax_post_process.cpp +++ b/hailort/libhailort/src/net_flow/ops/softmax_post_process.cpp @@ -12,6 +12,22 @@ #include "hailo/hailort_common.hpp" #include "hailo/hailort_defaults.hpp" +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4244 4267 4127) +#else +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif +#include +#if defined(_MSC_VER) +#pragma warning(pop) +#else +#pragma GCC diagnostic pop +#endif + #include "common/utils.hpp" #include @@ -36,21 +52,19 @@ hailo_status SoftmaxPostProcessOp::execute_not_supported(const BufferMetaData &i hailo_status SoftmaxPostProcessOp::softmax(float32_t *src, float32_t *dst, size_t num_of_elements) { - // In order to avoid overflows, we will perform the following: - // We find the maximal value and then we substract it from all of the values. - // This will preserve the original softmax values + prevent overflows - float32_t max_val = *std::max_element(src, src + num_of_elements); - float32_t sum_exp = 0; // denominator - for (uint32_t c = 0; c < num_of_elements; c++) { - auto ¤t_value = *(src + c); - current_value -= max_val; // This step preserves the original softmax values + prevent overflows - current_value = std::exp(static_cast(current_value)); // Set src[c] to e^(src[c]) so that we only calculate it once - sum_exp += current_value; - } - for (uint32_t c = 0; c < num_of_elements; c++) { - const auto ¤t_value = *(src + c); - dst[c] = static_cast(current_value / sum_exp); - } + // Create an Eigen Matrix view for src and dst + Eigen::Map> src_eigen(src, num_of_elements); + Eigen::Map> dst_eigen(dst, num_of_elements); + + src_eigen = src_eigen.array().exp(); // Compute exponentials in place + + assert(!src_eigen.hasNaN()); // Checks if any element in the array is NaN (Not a Number) + + float sum_exp = src_eigen.sum(); // Compute the sum of exponentials + + assert(0.0f != sum_exp); // Checks for division by zero + dst_eigen = src_eigen / sum_exp; // Perform softmax operation + return HAILO_SUCCESS; } diff --git a/hailort/libhailort/src/net_flow/ops/yolov5_bbox_only_post_process.cpp b/hailort/libhailort/src/net_flow/ops/yolov5_bbox_only_post_process.cpp index d03dc18..c44eece 100644 --- a/hailort/libhailort/src/net_flow/ops/yolov5_bbox_only_post_process.cpp +++ b/hailort/libhailort/src/net_flow/ops/yolov5_bbox_only_post_process.cpp @@ -28,11 +28,9 @@ Expected> YOLOv5BboxOnlyPostProcessOp::create(std::shared_pt Expected Yolov5BboxOnlyOpMetadata::get_output_vstream_info() { - auto vstream_info = NmsOpMetadata::get_output_vstream_info(); - CHECK_EXPECTED(vstream_info); - - vstream_info->shape = m_outputs_metadata.begin()->second.shape; - return vstream_info.release(); + TRY(auto vstream_info, NmsOpMetadata::get_output_vstream_info()); + vstream_info.shape = m_outputs_metadata.begin()->second.shape; + return vstream_info; } hailo_status Yolov5BboxOnlyOpMetadata::validate_format_info() diff --git a/hailort/libhailort/src/net_flow/ops/yolov5_post_process.hpp b/hailort/libhailort/src/net_flow/ops/yolov5_post_process.hpp index 15c9b6b..c4c9fee 100644 --- a/hailort/libhailort/src/net_flow/ops/yolov5_post_process.hpp +++ b/hailort/libhailort/src/net_flow/ops/yolov5_post_process.hpp @@ -54,12 +54,12 @@ protected: const uint32_t W_OFFSET, const uint32_t H_OFFSET, hailo_quant_info_t quant_info, uint32_t anchor, const std::vector &layer_anchors, uint32_t col, uint32_t row, hailo_3d_image_shape_t shape) { - auto tx = dequantize_and_sigmoid(data[entry_idx + X_OFFSET], quant_info); - auto ty = dequantize_and_sigmoid(data[entry_idx + Y_OFFSET], quant_info); - auto tw = dequantize_and_sigmoid(data[entry_idx + W_OFFSET], quant_info); - auto th = dequantize_and_sigmoid(data[entry_idx + H_OFFSET], quant_info); - return decode(tx, ty, tw, th, layer_anchors[anchor * 2], layer_anchors[anchor * 2 + 1], col, row, - shape.width, shape.height); + auto tx = dequantize_and_sigmoid(data[entry_idx + X_OFFSET], quant_info); + auto ty = dequantize_and_sigmoid(data[entry_idx + Y_OFFSET], quant_info); + auto tw = dequantize_and_sigmoid(data[entry_idx + W_OFFSET], quant_info); + auto th = dequantize_and_sigmoid(data[entry_idx + H_OFFSET], quant_info); + return decode(tx, ty, tw, th, layer_anchors[anchor * 2], layer_anchors[anchor * 2 + 1], col, row, + shape.width, shape.height); } template diff --git a/hailort/libhailort/src/net_flow/ops/yolov5_seg_post_process.cpp b/hailort/libhailort/src/net_flow/ops/yolov5_seg_post_process.cpp index e89082b..a7284df 100644 --- a/hailort/libhailort/src/net_flow/ops/yolov5_seg_post_process.cpp +++ b/hailort/libhailort/src/net_flow/ops/yolov5_seg_post_process.cpp @@ -17,6 +17,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wconversion" #pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif #define STB_IMAGE_RESIZE_IMPLEMENTATION #include "stb_image_resize.h" @@ -99,11 +100,9 @@ std::string Yolov5SegOpMetadata::get_op_description() Expected Yolov5SegOpMetadata::get_output_vstream_info() { - auto vstream_info = NmsOpMetadata::get_output_vstream_info(); - CHECK_EXPECTED(vstream_info); - - vstream_info->nms_shape.max_accumulated_mask_size = m_yolo_seg_config.max_accumulated_mask_size; - return vstream_info.release(); + TRY(auto vstream_info, NmsOpMetadata::get_output_vstream_info()); + vstream_info.nms_shape.max_accumulated_mask_size = m_yolo_seg_config.max_accumulated_mask_size; + return vstream_info; } Expected> Yolov5SegPostProcess::create(std::shared_ptr metadata) @@ -115,17 +114,16 @@ Expected> Yolov5SegPostProcess::create(std::shared_ptrinputs_metadata(), metadata->yolov5seg_config().proto_layer_name)); auto proto_layer_metadata = metadata->inputs_metadata().at(metadata->yolov5seg_config().proto_layer_name); auto transformed_proto_layer_frame_size = HailoRTCommon::get_shape_size(proto_layer_metadata.shape) * sizeof(float32_t); - auto transformed_proto_buffer = Buffer::create(transformed_proto_layer_frame_size); - CHECK_EXPECTED(transformed_proto_buffer); - auto mask_mult_result_buffer = Buffer::create(proto_layer_metadata.shape.height * proto_layer_metadata.shape.width * sizeof(float32_t)); - CHECK_EXPECTED(mask_mult_result_buffer); + TRY(auto transformed_proto_buffer, + Buffer::create(transformed_proto_layer_frame_size)); + TRY(auto mask_mult_result_buffer, + Buffer::create(proto_layer_metadata.shape.height * proto_layer_metadata.shape.width * sizeof(float32_t))); - auto image_size = static_cast(metadata->yolov5_config().image_width) * static_cast(metadata->yolov5_config().image_height); - auto resized_buffer = Buffer::create(image_size * sizeof(float32_t)); - CHECK_EXPECTED(resized_buffer); + const auto image_size = static_cast(metadata->yolov5_config().image_width) * static_cast(metadata->yolov5_config().image_height); + TRY(auto resized_buffer, Buffer::create(image_size * sizeof(float32_t))); auto op = std::shared_ptr(new (std::nothrow) Yolov5SegPostProcess(std::move(metadata), - mask_mult_result_buffer.release(), resized_buffer.release(), transformed_proto_buffer.release())); + std::move(mask_mult_result_buffer), std::move(resized_buffer), std::move(transformed_proto_buffer))); CHECK_NOT_NULL_AS_EXPECTED(op, HAILO_OUT_OF_HOST_MEMORY); return std::shared_ptr(std::move(op)); @@ -310,7 +308,7 @@ hailo_status Yolov5SegPostProcess::fill_nms_with_byte_mask_format(MemoryView &bu status = copied_bytes_amount.status(); break; } - CHECK_EXPECTED_AS_STATUS(copied_bytes_amount); + CHECK_EXPECTED_AS_STATUS(copied_bytes_amount); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here buffer_offset += copied_bytes_amount.release(); detections_count++; } diff --git a/hailort/libhailort/src/net_flow/ops/yolov8_bbox_only_post_process.cpp b/hailort/libhailort/src/net_flow/ops/yolov8_bbox_only_post_process.cpp new file mode 100644 index 0000000..3407e2c --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/yolov8_bbox_only_post_process.cpp @@ -0,0 +1,127 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file yolov8_bbox_only_post_process.cpp + * @brief YOLOv8 bbox only post process + * + **/ + +#include "net_flow/ops/yolov8_bbox_only_post_process.hpp" + +namespace hailort +{ +namespace net_flow +{ + +Expected> YOLOv8BboxOnlyPostProcessOp::create(std::shared_ptr metadata) +{ + auto status = metadata->validate_format_info(); + CHECK_SUCCESS_AS_EXPECTED(status); + + auto op = std::shared_ptr(new (std::nothrow) YOLOv8BboxOnlyPostProcessOp(metadata)); + CHECK_AS_EXPECTED(op != nullptr, HAILO_OUT_OF_HOST_MEMORY); + + return std::shared_ptr(std::move(op)); +} + +Expected Yolov8BboxOnlyOpMetadata::get_output_vstream_info() +{ + TRY(auto vstream_info, NmsOpMetadata::get_output_vstream_info()); + + vstream_info.shape = m_outputs_metadata.begin()->second.shape; + return vstream_info; +} + +hailo_status Yolov8BboxOnlyOpMetadata::validate_format_info() +{ + for (const auto& output_metadata : m_outputs_metadata) { + + CHECK(HAILO_FORMAT_TYPE_FLOAT32 == output_metadata.second.format.type, HAILO_INVALID_ARGUMENT, "The given output format type {} is not supported, " + "should be HAILO_FORMAT_TYPE_FLOAT32", HailoRTCommon::get_format_type_str(output_metadata.second.format.type)); + + CHECK(HAILO_FORMAT_ORDER_NHWC == output_metadata.second.format.order, HAILO_INVALID_ARGUMENT, "The given output format order {} is not supported, " + "should be HAILO_FORMAT_ORDER_NHWC", HailoRTCommon::get_format_order_str(output_metadata.second.format.order)); + + CHECK(!(HAILO_FORMAT_FLAGS_TRANSPOSED & output_metadata.second.format.flags), HAILO_INVALID_ARGUMENT, "Output {} is marked as transposed, which is not supported for this model.", + output_metadata.first); + CHECK(!(HAILO_FORMAT_FLAGS_HOST_ARGMAX & output_metadata.second.format.flags), HAILO_INVALID_ARGUMENT, "Output {} is marked as argmax, which is not supported for this model.", + output_metadata.first); + } + + assert(1 <= m_inputs_metadata.size()); + const hailo_format_type_t& first_input_type = m_inputs_metadata.begin()->second.format.type; + for (const auto& input_metadata : m_inputs_metadata) { + CHECK(HAILO_FORMAT_ORDER_NHCW == input_metadata.second.format.order, HAILO_INVALID_ARGUMENT, "The given input format order {} is not supported, " + "should be HAILO_FORMAT_ORDER_NHCW", HailoRTCommon::get_format_order_str(input_metadata.second.format.order)); + + CHECK((HAILO_FORMAT_TYPE_UINT8 == input_metadata.second.format.type) || + (HAILO_FORMAT_TYPE_UINT16 == input_metadata.second.format.type), + HAILO_INVALID_ARGUMENT, "The given input format type {} is not supported, should be HAILO_FORMAT_TYPE_UINT8 or HAILO_FORMAT_TYPE_UINT16", + HailoRTCommon::get_format_type_str(input_metadata.second.format.type)); + + CHECK(input_metadata.second.format.type == first_input_type, HAILO_INVALID_ARGUMENT,"All inputs format type should be the same"); + } + + return HAILO_SUCCESS; +} + +std::string Yolov8BboxOnlyOpMetadata::get_op_description() +{ + auto nms_config_info = fmt::format("Classes: {}", + nms_config().number_of_classes); + auto config_info = fmt::format("Op {}, Name: {}, {}, Image height: {:d}, Image width: {:d}", + OpMetadata::get_operation_type_str(m_type), m_name, nms_config_info, + static_cast(m_yolov8_config.image_height), static_cast(m_yolov8_config.image_width)); + return config_info; +} + +Expected> Yolov8BboxOnlyOpMetadata::create(const std::unordered_map &inputs_metadata, + const std::unordered_map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const Yolov8PostProcessConfig &yolov8_post_process_config, + const std::string &network_name) +{ + auto op_metadata = std::shared_ptr(new (std::nothrow) Yolov8BboxOnlyOpMetadata(inputs_metadata, outputs_metadata, + nms_post_process_config, yolov8_post_process_config, network_name)); + CHECK_AS_EXPECTED(op_metadata != nullptr, HAILO_OUT_OF_HOST_MEMORY); + + auto status = op_metadata->validate_params(); + CHECK_SUCCESS_AS_EXPECTED(status); + + return std::shared_ptr(std::move(op_metadata)); +} + +hailo_status YOLOv8BboxOnlyPostProcessOp::execute(const std::map &inputs, std::map &outputs) +{ + const auto &yolov8_config = m_metadata->yolov8_config(); + const auto &inputs_metadata = m_metadata->inputs_metadata(); + + auto dst_ptr = (float32_t*)outputs.begin()->second.data(); + + size_t next_bbox_output_offset = 0; + for (const auto ®_to_cls_name : yolov8_config.reg_to_cls_inputs) { + hailo_status status = HAILO_UNINITIALIZED; + assert(contains(inputs, reg_to_cls_name.cls)); + assert(contains(inputs, reg_to_cls_name.reg)); + + auto &input_metadata = inputs_metadata.at(reg_to_cls_name.reg); + + if (HAILO_FORMAT_TYPE_UINT8 == input_metadata.format.type) { + status = add_bboxes(dst_ptr, next_bbox_output_offset, reg_to_cls_name, + inputs.at(reg_to_cls_name.reg), inputs.at(reg_to_cls_name.cls), reg_to_cls_name.stride); + } else if (HAILO_FORMAT_TYPE_UINT16 == input_metadata.format.type) { + status = add_bboxes(dst_ptr, next_bbox_output_offset, reg_to_cls_name, + inputs.at(reg_to_cls_name.reg), inputs.at(reg_to_cls_name.cls), reg_to_cls_name.stride); + } else { + CHECK_SUCCESS(HAILO_INVALID_ARGUMENT, "YOLOV8 bbox only post-process received invalid input type {}", input_metadata.format.type); + } + + CHECK_SUCCESS(status); + } + return HAILO_SUCCESS; +} + +} // namespace net_flow +} // namespace hailort \ No newline at end of file diff --git a/hailort/libhailort/src/net_flow/ops/yolov8_bbox_only_post_process.hpp b/hailort/libhailort/src/net_flow/ops/yolov8_bbox_only_post_process.hpp new file mode 100644 index 0000000..7916505 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/yolov8_bbox_only_post_process.hpp @@ -0,0 +1,90 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file yolov8_bbox_only_post_process.hpp + * @brief YOLOV8 bbox only post process + * Output format of yolov8_bbox_only is NHWC - [1, total_proposals, 4 + number_of_classes] + * 1 is the batch size and 4 is the number of coordinates for each proposal + * The bboxes entry in the output of yolov8_bbox_only is a list of bboxes, such that each of them looks like this: + * (y_min, x_min, y_max, x_max, score_per_class) + **/ + +#ifndef _HAILO_YOLOV8_BBOX_ONLY_POST_PROCESS_HPP_ +#define _HAILO_YOLOV8_BBOX_ONLY_POST_PROCESS_HPP_ + +#include "net_flow/ops/yolov8_post_process.hpp" +#include "net_flow/ops_metadata/yolov8_bbox_only_op_metadata.hpp" + +namespace hailort +{ +namespace net_flow +{ + +class YOLOv8BboxOnlyPostProcessOp : public YOLOV8PostProcessOp +{ +public: + static Expected> create(std::shared_ptr metadata); + + hailo_status execute(const std::map &inputs, std::map &outputs) override; + +private: + + YOLOv8BboxOnlyPostProcessOp(std::shared_ptr metadata) : + YOLOV8PostProcessOp(static_cast>(metadata)) + {} + + template + hailo_status add_bboxes(DstType *dst_ptr, size_t &next_bbox_output_offset, const Yolov8MatchingLayersNames &layers_names, + const MemoryView ®_buffer, const MemoryView &cls_buffer, uint32_t stride) + { + const auto &inputs_metadata = m_metadata->inputs_metadata(); + const auto &nms_config = m_metadata->nms_config(); + + assert(contains(inputs_metadata, layers_names.reg)); + assert(contains(inputs_metadata, layers_names.cls)); + const auto ®_shape = inputs_metadata.at(layers_names.reg).shape; + const auto &cls_shape = inputs_metadata.at(layers_names.cls).shape; + const auto ®_padded_shape = inputs_metadata.at(layers_names.reg).padded_shape; + const auto &cls_padded_shape = inputs_metadata.at(layers_names.cls).padded_shape; + const auto ®_quant_info = inputs_metadata.at(layers_names.reg).quant_info; + const auto &cls_quant_info = inputs_metadata.at(layers_names.cls).quant_info; + + CHECK_SUCCESS(validate_regression_buffer_size(reg_padded_shape, reg_buffer, layers_names)); + CHECK_SUCCESS(validate_classes_buffer_size(cls_padded_shape, cls_buffer, layers_names, nms_config)); + + // Format is NHCW -> each row size is (padded C size) * (padded W size) + auto cls_row_size = cls_padded_shape.features * cls_padded_shape.width; + + SrcType *reg_data = (SrcType*)reg_buffer.data(); + SrcType *cls_data = (SrcType*)cls_buffer.data(); + + for (uint32_t row = 0; row < cls_shape.height; row++) { + for (uint32_t col = 0; col < cls_shape.width; col++) { + auto cls_idx = (cls_row_size * row) + col; + + assert(contains(m_d_matrix, layers_names.reg)); + auto &d_matrix = m_d_matrix.at(layers_names.reg); + auto bbox = get_bbox(row, col, stride, reg_padded_shape, reg_shape, reg_quant_info, + (SrcType*)reg_data, d_matrix); // we don't pass confidence here + memcpy(&dst_ptr[next_bbox_output_offset], &bbox, sizeof(hailo_rectangle_t)); // copy y_min, x_min, y_max, x_max + next_bbox_output_offset += sizeof(hailo_rectangle_t) / sizeof(float32_t); + + // copy class confidence for each of the classes + for (uint32_t curr_class_idx = 0; curr_class_idx < nms_config.number_of_classes; curr_class_idx++) { + auto class_entry_idx = cls_idx + (curr_class_idx * cls_padded_shape.width); + auto class_confidence = Quantization::dequantize_output( + cls_data[class_entry_idx], cls_quant_info); + dst_ptr[next_bbox_output_offset++] = class_confidence; + } + } + } + return HAILO_SUCCESS; + } +}; + +} /* namespace net_flow */ +} /* namespace hailort */ + +#endif /* _HAILO_YOLOV5_BBOX_ONLY_POST_PROCESS_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/net_flow/ops/yolov8_post_process.cpp b/hailort/libhailort/src/net_flow/ops/yolov8_post_process.cpp index b700ed5..c5c8be0 100644 --- a/hailort/libhailort/src/net_flow/ops/yolov8_post_process.cpp +++ b/hailort/libhailort/src/net_flow/ops/yolov8_post_process.cpp @@ -43,8 +43,6 @@ hailo_status Yolov8OpMetadata::validate_params() { CHECK_SUCCESS(NmsOpMetadata::validate_params()); - CHECK(!nms_config().bbox_only, HAILO_INVALID_ARGUMENT, "YOLOV8PostProcessOp: bbox_only is not supported for YOLOV8 model"); - // We go over the inputs metadata and check that it includes all of the regs and clss for (const auto &layer_names : m_yolov8_config.reg_to_cls_inputs) { CHECK(contains(m_inputs_metadata, layer_names.reg), HAILO_INVALID_ARGUMENT, diff --git a/hailort/libhailort/src/net_flow/ops/yolov8_post_process.hpp b/hailort/libhailort/src/net_flow/ops/yolov8_post_process.hpp index d0433f9..62f74f8 100644 --- a/hailort/libhailort/src/net_flow/ops/yolov8_post_process.hpp +++ b/hailort/libhailort/src/net_flow/ops/yolov8_post_process.hpp @@ -14,6 +14,8 @@ #include "net_flow/ops/nms_post_process.hpp" #include "net_flow/ops/softmax_post_process.hpp" #include "net_flow/ops_metadata/yolov8_op_metadata.hpp" +#include "net_flow/ops/softmax_post_process.hpp" + namespace hailort { namespace net_flow @@ -26,9 +28,8 @@ public: hailo_status execute(const std::map &inputs, std::map &outputs) override; -private: +protected: std::shared_ptr m_metadata; - std::vector m_d_values_matrix; // Holds the values of the bbox boundaries distances from the stride's center std::unordered_map>> m_d_matrix; // Holds the values from which we compute those distances YOLOV8PostProcessOp(std::shared_ptr metadata) : NmsPostProcessOp(static_cast>(metadata)) @@ -36,7 +37,7 @@ private: { for (const auto &input_metadata : m_metadata->inputs_metadata()) { m_d_matrix[input_metadata.first] = std::vector>(NUM_OF_D_VALUES, - std::vector(input_metadata.second.shape.features / NUM_OF_D_VALUES)); + std::vector(input_metadata.second.padded_shape.features / NUM_OF_D_VALUES)); } } @@ -61,11 +62,13 @@ private: auto &tmp_vector = d_matrix.at(vector_index); SoftmaxPostProcessOp::softmax(tmp_vector.data(), tmp_vector.data(), tmp_vector.size()); } + // Performing dot product on each vector // (A, B, C, ..., F, G) -> 0*A + 1*B + 2*C + ... + 14*F + 15*G for (uint32_t vector_index = 0; vector_index < NUM_OF_D_VALUES; vector_index++) { m_d_values_matrix[vector_index] = dot_product(d_matrix.at(vector_index)); } + // The decode function extract x_min, y_min, x_max, y_max from d1, d2, d3, d4 const auto &d1 = m_d_values_matrix.at(0); const auto &d2 = m_d_values_matrix.at(1); @@ -76,6 +79,32 @@ private: return bbox; } + template + hailo_status validate_regression_buffer_size(const hailo_3d_image_shape_t ®_padded_shape, const MemoryView ®_buffer, + const Yolov8MatchingLayersNames &layers_names) + { + auto number_of_entries = reg_padded_shape.height * reg_padded_shape.width; + auto buffer_size = number_of_entries * reg_padded_shape.features * sizeof(SrcType); + CHECK(buffer_size == reg_buffer.size(), HAILO_INVALID_ARGUMENT, + "Failed to extract_detections, reg {} buffer_size should be {}, but is {}", layers_names.reg, buffer_size, reg_buffer.size()); + return HAILO_SUCCESS; + } + + template + hailo_status validate_classes_buffer_size(const hailo_3d_image_shape_t &cls_padded_shape, const MemoryView &cls_buffer, + const Yolov8MatchingLayersNames &layers_names, const NmsPostProcessConfig &nms_config) + { + const uint32_t cls_entry_size = nms_config.number_of_classes; + auto number_of_entries = cls_padded_shape.height * cls_padded_shape.width; + auto buffer_size = number_of_entries * cls_entry_size * sizeof(SrcType); + CHECK(buffer_size == cls_buffer.size(), HAILO_INVALID_ARGUMENT, + "Failed to extract_detections, cls {} buffer_size should be {}, but is {}", layers_names.cls, buffer_size, cls_buffer.size()); + return HAILO_SUCCESS; + } + +private: + std::vector m_d_values_matrix; // Holds the values of the bbox boundaries distances from the stride's center + static const uint32_t CLASSES_START_INDEX = 0; static const uint32_t NO_OBJECTNESS = 1; static const uint32_t NUM_OF_D_VALUES = 4; @@ -96,18 +125,8 @@ private: const auto ®_quant_info = inputs_metadata.at(layers_names.reg).quant_info; const auto &cls_quant_info = inputs_metadata.at(layers_names.cls).quant_info; - // Validate regression buffer size - auto number_of_entries = reg_padded_shape.height * reg_padded_shape.width; - auto buffer_size = number_of_entries * reg_padded_shape.features * sizeof(SrcType); - CHECK(buffer_size == reg_buffer.size(), HAILO_INVALID_ARGUMENT, - "Failed to extract_detections, reg {} buffer_size should be {}, but is {}", layers_names.reg, buffer_size, reg_buffer.size()); - - // Validate classes buffer size - const uint32_t cls_entry_size = nms_config.number_of_classes; - number_of_entries = cls_padded_shape.height * cls_padded_shape.width; - buffer_size = number_of_entries * cls_entry_size * sizeof(SrcType); - CHECK(buffer_size == cls_buffer.size(), HAILO_INVALID_ARGUMENT, - "Failed to extract_detections, cls {} buffer_size should be {}, but is {}", layers_names.cls, buffer_size, cls_buffer.size()); + CHECK_SUCCESS(validate_regression_buffer_size(reg_padded_shape, reg_buffer, layers_names)); + CHECK_SUCCESS(validate_classes_buffer_size(cls_padded_shape, cls_buffer, layers_names, nms_config)); // Format is NHCW -> each row size is (padded C size) * (padded W size) auto cls_row_size = cls_padded_shape.features * cls_padded_shape.width; @@ -155,10 +174,6 @@ private: return HAILO_SUCCESS; } - template - hailo_bbox_float32_t get_bbox(uint32_t row, uint32_t col, uint32_t stride, const hailo_3d_image_shape_t ®_padded_shape, - const hailo_quant_info_t ®_quant_info, SrcType *reg_data, std::vector> &d_matrix, DstType class_confidence); - virtual hailo_bbox_float32_t decode(float32_t tx, float32_t ty, float32_t tw, float32_t th, uint32_t col, uint32_t row, uint32_t stride) const; diff --git a/hailort/libhailort/src/net_flow/ops_metadata/yolov8_bbox_only_op_metadata.hpp b/hailort/libhailort/src/net_flow/ops_metadata/yolov8_bbox_only_op_metadata.hpp new file mode 100644 index 0000000..b0d21f8 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops_metadata/yolov8_bbox_only_op_metadata.hpp @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file yolov8_bbox_only_op_metadata.hpp + * @brief YOLOv8 Bbox Only Post-Process op metadata + **/ + +#ifndef _HAILO_YOLOV8_BBOX_ONLY_OP_METADATA_HPP_ +#define _HAILO_YOLOV8_BBOX_ONLY_OP_METADATA_HPP_ + +#include "hailo/hailort.h" +#include "net_flow/ops_metadata/yolov8_op_metadata.hpp" + +namespace hailort +{ +namespace net_flow +{ + +class Yolov8BboxOnlyOpMetadata : public Yolov8OpMetadata +{ +public: + static Expected> create(const std::unordered_map &inputs_metadata, + const std::unordered_map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const Yolov8PostProcessConfig &yolov8_post_process_config, + const std::string &network_name); + hailo_status validate_format_info() override; + std::string get_op_description() override; + virtual Expected get_output_vstream_info() override; + +private: + Yolov8BboxOnlyOpMetadata(const std::unordered_map &inputs_metadata, + const std::unordered_map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const Yolov8PostProcessConfig &yolov8_post_process_config, + const std::string &network_name) + : Yolov8OpMetadata(inputs_metadata, outputs_metadata, nms_post_process_config, yolov8_post_process_config, network_name, + "YOLOv8Bbox-Only-Post-Process") + {} + +}; + +} /* namespace hailort */ +} /* namespace net_flow */ + +#endif /* _HAILO_YOLOV8_BBOX_ONLY_OP_METADATA_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/net_flow/ops_metadata/yolov8_op_metadata.hpp b/hailort/libhailort/src/net_flow/ops_metadata/yolov8_op_metadata.hpp index 078cbfe..1c6666b 100644 --- a/hailort/libhailort/src/net_flow/ops_metadata/yolov8_op_metadata.hpp +++ b/hailort/libhailort/src/net_flow/ops_metadata/yolov8_op_metadata.hpp @@ -16,7 +16,7 @@ namespace hailort namespace net_flow { - struct Yolov8MatchingLayersNames +struct Yolov8MatchingLayersNames { // Regression layer std::string reg; @@ -51,17 +51,17 @@ public: std::string get_op_description() override; Yolov8PostProcessConfig &yolov8_config() { return m_yolov8_config;}; -private: - Yolov8PostProcessConfig m_yolov8_config; +protected: Yolov8OpMetadata(const std::unordered_map &inputs_metadata, const std::unordered_map &outputs_metadata, const NmsPostProcessConfig &nms_post_process_config, const Yolov8PostProcessConfig &yolov8_post_process_config, - const std::string &network_name) - : NmsOpMetadata(inputs_metadata, outputs_metadata, nms_post_process_config, "YOLOV8-Post-Process", network_name, OperationType::YOLOV8) + const std::string &network_name, + const std::string &name = "YOLOV8-Post-Process") + : NmsOpMetadata(inputs_metadata, outputs_metadata, nms_post_process_config, name, network_name, OperationType::YOLOV8) , m_yolov8_config(yolov8_post_process_config) {} - + Yolov8PostProcessConfig m_yolov8_config; hailo_status validate_params() override; }; diff --git a/hailort/libhailort/src/net_flow/pipeline/async_infer_runner.cpp b/hailort/libhailort/src/net_flow/pipeline/async_infer_runner.cpp index 0b0d701..3707d1e 100644 --- a/hailort/libhailort/src/net_flow/pipeline/async_infer_runner.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/async_infer_runner.cpp @@ -155,10 +155,10 @@ Expected> AsyncInferRunnerImpl::create(std auto pipeline_status = make_shared_nothrow>(HAILO_SUCCESS); CHECK_AS_EXPECTED(nullptr != pipeline_status, HAILO_OUT_OF_HOST_MEMORY); - auto async_pipeline_expected = AsyncPipelineBuilder::create_pipeline(net_group, inputs_formats, outputs_formats, timeout, pipeline_status); - CHECK_EXPECTED(async_pipeline_expected); + TRY(auto async_pipeline, + AsyncPipelineBuilder::create_pipeline(net_group, inputs_formats, outputs_formats, timeout, pipeline_status)); - auto async_infer_runner_ptr = make_shared_nothrow(async_pipeline_expected.release(), pipeline_status); + auto async_infer_runner_ptr = make_shared_nothrow(std::move(async_pipeline), pipeline_status); CHECK_NOT_NULL_AS_EXPECTED(async_infer_runner_ptr, HAILO_OUT_OF_HOST_MEMORY); auto status = async_infer_runner_ptr->start_pipeline(); @@ -221,20 +221,18 @@ void AsyncInferRunnerImpl::abort() return; } -Expected AsyncInferRunnerImpl::can_push_buffers() +Expected AsyncInferRunnerImpl::can_push_buffers(uint32_t frames_count) { for (auto &last_element : m_async_pipeline->get_last_elements()) { - auto can_push_buffer = last_element.second->can_push_buffer_upstream(); - CHECK_EXPECTED(can_push_buffer); - if (!can_push_buffer.release()) { + TRY(const auto can_push_buffer, last_element.second->can_push_buffer_upstream(frames_count)); + if (!can_push_buffer) { return false; } } for (auto &entry_element : m_async_pipeline->get_entry_elements()) { - auto can_push_buffer = entry_element.second->can_push_buffer_downstream(); - CHECK_EXPECTED(can_push_buffer); - if (!can_push_buffer.release()) { + TRY(const auto can_push_buffer, entry_element.second->can_push_buffer_downstream(frames_count)); + if (!can_push_buffer) { return false; } } @@ -243,12 +241,11 @@ Expected AsyncInferRunnerImpl::can_push_buffers() } hailo_status AsyncInferRunnerImpl::set_buffers(std::unordered_map &inputs, - std::unordered_map> &outputs) + std::unordered_map &outputs) { for (auto &last_element : m_async_pipeline->get_last_elements()) { // TODO: handle the non-recoverable case where one buffer is enqueued successfully and the second isn't (HRT-11783) - auto status = last_element.second->enqueue_execution_buffer(outputs.at(last_element.first).first, - outputs.at(last_element.first).second); + auto status = last_element.second->enqueue_execution_buffer(std::move(outputs.at(last_element.first))); CHECK_SUCCESS(status); } @@ -259,46 +256,27 @@ hailo_status AsyncInferRunnerImpl::set_buffers(std::unordered_map &inputs, hailo_pix_buffer_t userptr_pix_buffer, +void AsyncInferRunnerImpl::set_pix_buffer_inputs(std::unordered_map &inputs, hailo_pix_buffer_t pix_buffer, TransferDoneCallbackAsyncInfer input_done, const std::string &input_name) { - if (1 == userptr_pix_buffer.number_of_planes) { - inputs[input_name] = PipelineBuffer(MemoryView(userptr_pix_buffer.planes[0].user_ptr, userptr_pix_buffer.planes[0].bytes_used), input_done); + if (1 == pix_buffer.number_of_planes) { + if (HAILO_PIX_BUFFER_MEMORY_TYPE_DMABUF == pix_buffer.memory_type) { + hailo_dma_buffer_t dma_buffer = {pix_buffer.planes[0].fd, pix_buffer.planes[0].plane_size}; + inputs[input_name] = PipelineBuffer(dma_buffer, input_done); + } else if (HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR == pix_buffer.memory_type) { + inputs[input_name] = PipelineBuffer(MemoryView(pix_buffer.planes[0].user_ptr, pix_buffer.planes[0].bytes_used), input_done); + } else { + LOGGER__ERROR("Buffer type Pix buffer supports only memory of types USERPTR or DMABUF."); + inputs[input_name] = PipelineBuffer(HAILO_INVALID_OPERATION, input_done); + } } else if (m_async_pipeline->is_multi_planar()) { // If model is multi-planar - inputs[input_name] = PipelineBuffer(userptr_pix_buffer, input_done); + inputs[input_name] = PipelineBuffer(pix_buffer, input_done); } else { // Other cases - return error, as on async flow we do not support copy to new buffer LOGGER__ERROR("HEF was compiled for single input layer, while trying to pass non-contiguous planes buffers."); inputs[input_name] = PipelineBuffer(HAILO_INVALID_OPERATION, input_done); } - -} - -Expected AsyncInferRunnerImpl::convert_dma_pix_buffer_to_userptr_pix_buffer(const hailo_pix_buffer_t &dma_pix_buffer) -{ - hailo_pix_buffer_t userptr_pix_buffer; - userptr_pix_buffer.index = dma_pix_buffer.index; - userptr_pix_buffer.number_of_planes = dma_pix_buffer.number_of_planes; - userptr_pix_buffer.memory_type = HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR; - - for (uint32_t i = 0; i < dma_pix_buffer.number_of_planes; i++ ) { - auto current_plane = dma_pix_buffer.planes[i]; - hailo_dma_buffer_t dma_buffer = {current_plane.fd, current_plane.bytes_used}; - - auto dma_buffer_memview_expected = DmaBufferUtils::mmap_dma_buffer_read(dma_buffer); - CHECK_EXPECTED_AS_STATUS(dma_buffer_memview_expected); - auto dma_buffer_memview = dma_buffer_memview_expected.release(); - - hailo_pix_buffer_plane_t new_plane; - new_plane.bytes_used = current_plane.bytes_used; - new_plane.plane_size = current_plane.plane_size; - new_plane.user_ptr = dma_buffer_memview.data(); - - userptr_pix_buffer.planes[i] = new_plane; - } - - return userptr_pix_buffer; } hailo_status AsyncInferRunnerImpl::run(ConfiguredInferModel::Bindings &bindings, TransferDoneCallbackAsyncInfer transfer_done) @@ -307,33 +285,20 @@ hailo_status AsyncInferRunnerImpl::run(ConfiguredInferModel::Bindings &bindings, hailo_status status = m_async_pipeline->get_pipeline_status()->load(); CHECK_SUCCESS(status, "Can't handle infer request since Pipeline status is {}.", status); - TRY(auto are_pools_ready, can_push_buffers()); + TRY(auto are_pools_ready, can_push_buffers(1)); CHECK(are_pools_ready, HAILO_QUEUE_IS_FULL, "Can't handle infer request since a queue in the pipeline is full."); - std::unordered_map> outputs; + std::unordered_map outputs; + for (auto &last_element : m_async_pipeline->get_last_elements()) { auto buff_type = bindings.output(last_element.first)->m_pimpl->get_type(); + bool is_user_buffer = true; if (BufferType::DMA_BUFFER == buff_type) { TRY(auto dma_buffer, bindings.output(last_element.first)->get_dma_buffer(), "Couldnt find output buffer for '{}'", last_element.first); - - TRY(auto dma_buffer_memview, DmaBufferUtils::mmap_dma_buffer_write(dma_buffer)); - - auto output_done = [dma_buffer_memview, dma_buffer=dma_buffer, transfer_done](hailo_status status) { - auto mumap_status = DmaBufferUtils::munmap_dma_buffer_write(dma_buffer, dma_buffer_memview); - if (HAILO_SUCCESS != mumap_status) { - LOGGER__ERROR("Failed to unmap dma buffer"); - status = HAILO_FILE_OPERATION_FAILURE; - } - transfer_done(status); - }; - std::pair buffer_cb_pair (dma_buffer_memview, output_done); - outputs[last_element.first] = buffer_cb_pair; - + outputs[last_element.first] = PipelineBuffer(dma_buffer, transfer_done, HAILO_SUCCESS, is_user_buffer); } else { TRY(auto buffer, bindings.output(last_element.first)->get_buffer(), "Couldnt find output buffer for '{}'", last_element.first); - - std::pair buffer_cb_pair (buffer, transfer_done); - outputs[last_element.first] = buffer_cb_pair; + outputs[last_element.first] = PipelineBuffer(buffer, transfer_done, HAILO_SUCCESS, is_user_buffer); } } @@ -351,50 +316,14 @@ hailo_status AsyncInferRunnerImpl::run(ConfiguredInferModel::Bindings &bindings, case BufferType::DMA_BUFFER: { TRY(auto dma_buffer, bindings.input(entry_element.first)->get_dma_buffer(), "Couldnt find input buffer for '{}'", entry_element.first); - - TRY(auto dma_buffer_memview, DmaBufferUtils::mmap_dma_buffer_read(dma_buffer)); - - auto input_done = [dma_buffer_memview, dma_buffer, transfer_done](hailo_status status) { - auto mumap_status = DmaBufferUtils::munmap_dma_buffer_read(dma_buffer, dma_buffer_memview); - if (HAILO_SUCCESS != mumap_status) { - // Note: we overide the status even if it was not success before (but either way it's set to non-success) - LOGGER__ERROR("Failed to unmap dma buffer"); - status = mumap_status; - } - transfer_done(status); - }; - inputs[entry_element.first] = PipelineBuffer(dma_buffer_memview, input_done); + inputs[entry_element.first] = PipelineBuffer(dma_buffer, transfer_done); break; } case BufferType::PIX_BUFFER: { - // TODO: handle a case in which the pix_buffer is DMA buffers (HRT-12771) TRY(auto pix_buffer, bindings.input(entry_element.first)->get_pix_buffer(), "Couldnt find input buffer for '{}'", entry_element.first); - if (HAILO_PIX_BUFFER_MEMORY_TYPE_DMABUF == pix_buffer.memory_type) { - TRY(auto userptr_pix_buffer, convert_dma_pix_buffer_to_userptr_pix_buffer(pix_buffer)); - - auto input_done = [userptr_pix_buffer, transfer_done, dma_pix_buffer=pix_buffer](hailo_status status) { - for (uint32_t i = 0; i < dma_pix_buffer.number_of_planes; i++ ) { - auto plane_in_dma_buffer = dma_pix_buffer.planes[i]; - hailo_dma_buffer_t dma_buffer = {plane_in_dma_buffer.fd, plane_in_dma_buffer.bytes_used}; - - auto dma_buffer_memview = MemoryView(userptr_pix_buffer.planes[i].user_ptr, userptr_pix_buffer.planes[i].bytes_used); - - auto mumap_status = DmaBufferUtils::munmap_dma_buffer_read(dma_buffer, dma_buffer_memview); - if (HAILO_SUCCESS != mumap_status) { - // Note: we overide the status even if it was not success before (but either way it's set to non-success) - LOGGER__ERROR("Failed to unmap dma buffer"); - status = mumap_status; - } - } - transfer_done(status); - }; - - set_pix_buffer_inputs(inputs, userptr_pix_buffer, input_done, entry_element.first); - } else { - set_pix_buffer_inputs(inputs, pix_buffer, transfer_done, entry_element.first); - } + set_pix_buffer_inputs(inputs, pix_buffer, transfer_done, entry_element.first); break; } @@ -404,8 +333,8 @@ hailo_status AsyncInferRunnerImpl::run(ConfiguredInferModel::Bindings &bindings, } status = set_buffers(inputs, outputs); + // TODO: (HRT-14283) If set_buffers fails after a buffer is enqueued, the buffer's CB will be called - and might call user's CB CHECK_SUCCESS(status); - return HAILO_SUCCESS; } diff --git a/hailort/libhailort/src/net_flow/pipeline/async_infer_runner.hpp b/hailort/libhailort/src/net_flow/pipeline/async_infer_runner.hpp index 2d0db04..4e9a4e8 100644 --- a/hailort/libhailort/src/net_flow/pipeline/async_infer_runner.hpp +++ b/hailort/libhailort/src/net_flow/pipeline/async_infer_runner.hpp @@ -69,11 +69,11 @@ public: hailo_status run(ConfiguredInferModel::Bindings &bindings, TransferDoneCallbackAsyncInfer transfer_done); hailo_status set_buffers(std::unordered_map &inputs, - std::unordered_map> &outputs); + std::unordered_map &outputs); void abort(); - Expected can_push_buffers(); + Expected can_push_buffers(uint32_t frames_count); void add_element_to_pipeline(std::shared_ptr pipeline_element); void add_entry_element(std::shared_ptr pipeline_element, const std::string &input_name); @@ -91,7 +91,6 @@ protected: hailo_status start_pipeline(); hailo_status stop_pipeline(); - static Expected convert_dma_pix_buffer_to_userptr_pix_buffer(const hailo_pix_buffer_t &dma_pix_buffer); void set_pix_buffer_inputs(std::unordered_map &inputs, hailo_pix_buffer_t userptr_pix_buffer, TransferDoneCallbackAsyncInfer input_done, const std::string &input_name); diff --git a/hailort/libhailort/src/net_flow/pipeline/async_pipeline_builder.cpp b/hailort/libhailort/src/net_flow/pipeline/async_pipeline_builder.cpp index 72f88a3..6f5933e 100644 --- a/hailort/libhailort/src/net_flow/pipeline/async_pipeline_builder.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/async_pipeline_builder.cpp @@ -12,6 +12,7 @@ #include "net_flow/ops/yolov5_seg_post_process.hpp" #include "net_flow/ops/yolov5_bbox_only_post_process.hpp" #include "net_flow/ops/yolov8_post_process.hpp" +#include "net_flow/ops/yolov8_bbox_only_post_process.hpp" #include "net_flow/ops/argmax_post_process.hpp" #include "net_flow/ops/softmax_post_process.hpp" #include "net_flow/ops/yolox_post_process.hpp" @@ -27,14 +28,11 @@ Expected> AsyncPipelineBuilder:: { std::unordered_map expanded_input_format; for (auto &input_format : inputs_formats) { - auto input_streams_names = net_group->get_stream_names_from_vstream_name(input_format.first); - CHECK_EXPECTED(input_streams_names); + TRY(const auto input_streams_names, net_group->get_stream_names_from_vstream_name(input_format.first)); - auto is_multi_planar = (input_streams_names.value().size() > 1); + auto is_multi_planar = (input_streams_names.size() > 1); if(is_multi_planar) { - auto vstream_info_exp = net_group->get_input_vstream_infos(); - CHECK_EXPECTED(vstream_info_exp); - auto vstream_infos = vstream_info_exp.release(); + TRY(const auto vstream_infos, net_group->get_input_vstream_infos()); auto matching_vstream_info = std::find_if(vstream_infos.begin(), vstream_infos.end(), [&](const auto &item) { return item.name == input_format.first; } ); CHECK_AS_EXPECTED(vstream_infos.end() != matching_vstream_info, HAILO_NOT_FOUND, @@ -42,7 +40,7 @@ Expected> AsyncPipelineBuilder:: expanded_input_format[input_format.first] = VStreamsBuilderUtils::expand_user_buffer_format_autos_multi_planar(*matching_vstream_info, input_format.second); } else { - const auto &stream_name = input_streams_names.value()[0]; + const auto &stream_name = input_streams_names[0]; CHECK_AS_EXPECTED(contains(named_stream_infos, stream_name), HAILO_INTERNAL_FAILURE); const auto &stream_info = named_stream_infos.at(stream_name); @@ -58,11 +56,10 @@ Expected> AsyncPipelineBuilder:: { std::unordered_map expanded_output_format; for (auto &output_format : outputs_formats) { - auto output_streams_names = net_group->get_stream_names_from_vstream_name(output_format.first); - CHECK_EXPECTED(output_streams_names); + TRY(const auto output_streams_names, net_group->get_stream_names_from_vstream_name(output_format.first)); // TODO: Taking data from the first ll stream will not work in multi-planar work - const auto &stream_name = output_streams_names.value()[0]; + const auto &stream_name = output_streams_names[0]; CHECK_AS_EXPECTED(contains(named_stream_infos, stream_name), HAILO_INTERNAL_FAILURE); const auto &stream_info = named_stream_infos.at(stream_name); @@ -76,20 +73,18 @@ hailo_status AsyncPipelineBuilder::create_pre_async_hw_elements_per_input(std::s const std::vector &stream_names, const std::unordered_map &inputs_formats, const std::unordered_map &named_stream_infos, std::shared_ptr async_pipeline) { - auto vstream_names = net_group->get_vstream_names_from_stream_name(*stream_names.begin()); - CHECK_EXPECTED_AS_STATUS(vstream_names); - CHECK(vstream_names.value().size() == 1, HAILO_NOT_SUPPORTED, "low level stream must have exactly 1 user input"); - const auto &vstream_name = vstream_names.value()[0]; + TRY(const auto vstream_names, net_group->get_vstream_names_from_stream_name(*stream_names.begin())); + CHECK(vstream_names.size() == 1, HAILO_NOT_SUPPORTED, "low level stream must have exactly 1 user input"); + const auto &vstream_name = vstream_names[0]; std::shared_ptr multi_plane_splitter = nullptr; std::shared_ptr last_element_connected_to_pipeline = nullptr; auto is_empty = true; auto interacts_with_hw = true; // We want the entry queue size to be the size of queues interacts with HW auto is_entry = true; - auto entry_queue_elem_expected = add_push_queue_element(PipelineObject::create_element_name("EntryPushQEl", vstream_name, 0), - async_pipeline, 0, is_empty, interacts_with_hw, nullptr, 0, is_entry); - CHECK_EXPECTED_AS_STATUS(entry_queue_elem_expected); - auto entry_queue_elem = entry_queue_elem_expected.release(); + TRY(auto entry_queue_elem, + add_push_queue_element(PipelineObject::create_element_name("EntryPushQEl", vstream_name, 0), + async_pipeline, 0, is_empty, interacts_with_hw, nullptr, 0, is_entry)); async_pipeline->add_entry_element(entry_queue_elem, vstream_name); last_element_connected_to_pipeline = entry_queue_elem; @@ -98,10 +93,8 @@ hailo_status AsyncPipelineBuilder::create_pre_async_hw_elements_per_input(std::s async_pipeline->set_as_multi_planar(); const auto &vstream_order = inputs_formats.at(vstream_name).order; - auto multi_plane_splitter_expected = create_multi_plane_splitter_element(vstream_name, vstream_order, - async_pipeline->get_build_params().pipeline_status, async_pipeline); - CHECK_EXPECTED_AS_STATUS(multi_plane_splitter_expected); - multi_plane_splitter = multi_plane_splitter_expected.release(); + TRY(multi_plane_splitter, create_multi_plane_splitter_element(vstream_name, vstream_order, + async_pipeline->get_build_params().pipeline_status, async_pipeline)); async_pipeline->add_element_to_pipeline(multi_plane_splitter); CHECK_SUCCESS(PipelinePad::link_pads(entry_queue_elem, multi_plane_splitter)); @@ -113,47 +106,41 @@ hailo_status AsyncPipelineBuilder::create_pre_async_hw_elements_per_input(std::s const auto &input_stream_info = named_stream_infos.at(stream_name); auto src_format = inputs_formats.at(vstream_name); - auto sink_index_expected = async_pipeline->get_async_hw_element()->get_sink_index_from_input_stream_name(stream_name); - CHECK_EXPECTED_AS_STATUS(sink_index_expected); - auto sink_index = static_cast(sink_index_expected.release()); + TRY(const auto sink_index, async_pipeline->get_async_hw_element()->get_sink_index_from_input_stream_name(stream_name)); if(is_multi_planar) { is_empty = true; interacts_with_hw = false; - auto post_split_push_queue = add_push_queue_element( + TRY(auto post_split_push_queue, add_push_queue_element( PipelineObject::create_element_name("PostSplitPushQEl", stream_name, sink_index), - async_pipeline, 0, is_empty, interacts_with_hw, nullptr); - CHECK_EXPECTED_AS_STATUS(post_split_push_queue); - CHECK_SUCCESS(PipelinePad::link_pads(multi_plane_splitter, post_split_push_queue.value(), plane_index++)); + async_pipeline, 0, is_empty, interacts_with_hw, nullptr)); + CHECK_SUCCESS(PipelinePad::link_pads(multi_plane_splitter, post_split_push_queue, plane_index++)); - last_element_connected_to_pipeline = post_split_push_queue.value(); + last_element_connected_to_pipeline = post_split_push_queue; /* In multi-planar case, the format order of each plane (stream) is determined by the ll-stream's order. Type and flags are determined by the vstream params */ src_format.order = input_stream_info.format.order; } - auto should_transform = InputTransformContext::is_transformation_required(input_stream_info.shape, + TRY(const auto should_transform, InputTransformContext::is_transformation_required(input_stream_info.shape, src_format, input_stream_info.hw_shape, input_stream_info.format, - std::vector(1, input_stream_info.quant_info)); // Inputs always have single quant_info - CHECK_EXPECTED_AS_STATUS(should_transform); + std::vector(1, input_stream_info.quant_info))); // Inputs always have single quant_info - if (should_transform.value()) { - auto pre_infer_elem = PreInferElement::create(input_stream_info.shape, src_format, + if (should_transform) { + TRY(auto pre_infer_elem, PreInferElement::create(input_stream_info.shape, src_format, input_stream_info.hw_shape, input_stream_info.format, { input_stream_info.quant_info }, PipelineObject::create_element_name("PreInferEl", stream_name, input_stream_info.index), - async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED_AS_STATUS(pre_infer_elem); - async_pipeline->add_element_to_pipeline(pre_infer_elem.value()); - CHECK_SUCCESS(PipelinePad::link_pads(last_element_connected_to_pipeline, pre_infer_elem.value())); + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); + async_pipeline->add_element_to_pipeline(pre_infer_elem); + CHECK_SUCCESS(PipelinePad::link_pads(last_element_connected_to_pipeline, pre_infer_elem)); is_empty = false; interacts_with_hw = true; - auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl", stream_name, input_stream_info.index), - async_pipeline, input_stream_info.hw_frame_size, is_empty, interacts_with_hw, pre_infer_elem.value()); - CHECK_EXPECTED_AS_STATUS(queue_elem); - CHECK_SUCCESS(PipelinePad::link_pads(pre_infer_elem.value(), queue_elem.value())); - CHECK_SUCCESS(PipelinePad::link_pads(queue_elem.value(), async_pipeline->get_async_hw_element(), 0, sink_index)); + TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl", stream_name, input_stream_info.index), + async_pipeline, input_stream_info.hw_frame_size, is_empty, interacts_with_hw, pre_infer_elem)); + CHECK_SUCCESS(PipelinePad::link_pads(pre_infer_elem, queue_elem)); + CHECK_SUCCESS(PipelinePad::link_pads(queue_elem, async_pipeline->get_async_hw_element(), 0, sink_index)); } else { CHECK_SUCCESS(PipelinePad::link_pads(last_element_connected_to_pipeline, async_pipeline->get_async_hw_element(), 0, sink_index)); } @@ -167,10 +154,10 @@ hailo_status AsyncPipelineBuilder::create_pre_async_hw_elements(std::shared_ptr< std::shared_ptr async_pipeline) { for(const auto &input : inputs_formats) { - auto stream_names_under_vstream = net_group->get_stream_names_from_vstream_name(input.first); - CHECK_EXPECTED_AS_STATUS(stream_names_under_vstream); + TRY(const auto stream_names_under_vstream, + net_group->get_stream_names_from_vstream_name(input.first)); - auto status = create_pre_async_hw_elements_per_input(net_group, stream_names_under_vstream.release(), inputs_formats, + auto status = create_pre_async_hw_elements_per_input(net_group, stream_names_under_vstream, inputs_formats, named_stream_infos, async_pipeline); CHECK_SUCCESS(status); } @@ -186,38 +173,35 @@ Expected> AsyncPipelineBuilder::add_post_infer HailoRTCommon::get_nms_hw_frame_size(nms_info) : HailoRTCommon::get_periph_frame_size(src_image_shape, src_format); auto is_empty = false; auto interacts_with_hw = true; - auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl", final_elem->name(), + TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl", final_elem->name(), static_cast(final_elem_source_index)), async_pipeline, pre_transform_frame_size, is_empty, interacts_with_hw, - final_elem, final_elem_source_index); - CHECK_EXPECTED(queue_elem); + final_elem, final_elem_source_index)); - auto post_infer_elem = PostInferElement::create(src_image_shape, src_format, dst_image_shape, output_format, + TRY(auto post_infer_elem, PostInferElement::create(src_image_shape, src_format, dst_image_shape, output_format, dst_quant_infos, nms_info, PipelineObject::create_element_name("PostInferEl", final_elem->name(), static_cast(final_elem_source_index)), async_pipeline->get_build_params(), - PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED(post_infer_elem); + PipelineDirection::PUSH, async_pipeline)); - async_pipeline->add_element_to_pipeline(post_infer_elem.value()); + async_pipeline->add_element_to_pipeline(post_infer_elem); - CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(queue_elem.value(), post_infer_elem.value())); - return post_infer_elem.release(); + CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(queue_elem, post_infer_elem)); + return post_infer_elem; } Expected> AsyncPipelineBuilder::add_push_queue_element(const std::string &queue_name, std::shared_ptr async_pipeline, size_t frame_size, bool is_empty, bool interacts_with_hw, std::shared_ptr final_elem, const uint32_t final_elem_source_index, bool is_entry) { - auto push_queue_elem = AsyncPushQueueElement::create(queue_name, async_pipeline->get_build_params(), frame_size, - is_empty, interacts_with_hw, async_pipeline, is_entry); - CHECK_EXPECTED(push_queue_elem); + TRY(auto push_queue_elem, AsyncPushQueueElement::create(queue_name, async_pipeline->get_build_params(), frame_size, + is_empty, interacts_with_hw, async_pipeline, is_entry)); - async_pipeline->add_element_to_pipeline(push_queue_elem.value()); + async_pipeline->add_element_to_pipeline(push_queue_elem); // final elem will be nullptr in case it's the first element in pipeline if (final_elem) { - CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, push_queue_elem.value(), final_elem_source_index, 0)); + CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, push_queue_elem, final_elem_source_index, 0)); } - return push_queue_elem.release(); + return push_queue_elem; } Expected> AsyncPipelineBuilder::add_nms_to_detections_convert_element(std::shared_ptr async_pipeline, @@ -227,15 +211,14 @@ Expected> AsyncPipelineBuilder::a auto metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != metadata); - auto nms_to_detections_element = ConvertNmsToDetectionsElement::create(metadata->nms_info(), + TRY(auto nms_to_detections_element, ConvertNmsToDetectionsElement::create(metadata->nms_info(), PipelineObject::create_element_name(element_name, output_stream_name, stream_index), - async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED(nms_to_detections_element); + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); - async_pipeline->add_element_to_pipeline(nms_to_detections_element.value()); + async_pipeline->add_element_to_pipeline(nms_to_detections_element); - CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, nms_to_detections_element.value(), final_elem_index, 0)); - return nms_to_detections_element.release(); + CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, nms_to_detections_element, final_elem_index, 0)); + return nms_to_detections_element; } Expected> AsyncPipelineBuilder::add_remove_overlapping_bboxes_element(std::shared_ptr async_pipeline, @@ -245,14 +228,13 @@ Expected> AsyncPipelineBuilder:: auto metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != metadata); - auto remove_overlapping_bboxes_element = RemoveOverlappingBboxesElement::create(metadata->nms_config(), + TRY(auto remove_overlapping_bboxes_element, RemoveOverlappingBboxesElement::create(metadata->nms_config(), PipelineObject::create_element_name(element_name, output_stream_name, stream_index), - async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED(remove_overlapping_bboxes_element); + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); - async_pipeline->add_element_to_pipeline(remove_overlapping_bboxes_element.value()); + async_pipeline->add_element_to_pipeline(remove_overlapping_bboxes_element); - CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, remove_overlapping_bboxes_element.value(), final_elem_index, 0)); + CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, remove_overlapping_bboxes_element, final_elem_index, 0)); return remove_overlapping_bboxes_element; } @@ -263,30 +245,28 @@ Expected> AsyncPipelineBuilder::add_fill_n auto metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != metadata); - auto fill_nms_format_element = FillNmsFormatElement::create(metadata->nms_config(), + TRY(auto fill_nms_format_element, FillNmsFormatElement::create(metadata->nms_config(), PipelineObject::create_element_name(element_name, output_stream_name, stream_index), - async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED(fill_nms_format_element); + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); - async_pipeline->add_element_to_pipeline(fill_nms_format_element.value()); + async_pipeline->add_element_to_pipeline(fill_nms_format_element); - CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, fill_nms_format_element.value(), final_elem_index, 0)); + CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, fill_nms_format_element, final_elem_index, 0)); return fill_nms_format_element; } Expected> AsyncPipelineBuilder::add_last_async_element(std::shared_ptr async_pipeline, const std::string &output_format_name, size_t frame_size, std::shared_ptr final_elem, const uint32_t final_elem_source_index) { - auto last_async_element = LastAsyncElement::create(PipelineObject::create_element_name("LastAsyncEl", - final_elem->name(), static_cast(final_elem_source_index)), async_pipeline->get_build_params(), frame_size, async_pipeline); - CHECK_EXPECTED(last_async_element); + TRY(auto last_async_element, LastAsyncElement::create(PipelineObject::create_element_name("LastAsyncEl", + final_elem->name(), static_cast(final_elem_source_index)), async_pipeline->get_build_params(), frame_size, async_pipeline)); - async_pipeline->add_element_to_pipeline(last_async_element.value()); - CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, last_async_element.value(), final_elem_source_index, 0)); + async_pipeline->add_element_to_pipeline(last_async_element); + CHECK_SUCCESS_AS_EXPECTED(PipelinePad::link_pads(final_elem, last_async_element, final_elem_source_index, 0)); - async_pipeline->add_last_element(last_async_element.value(), output_format_name); + async_pipeline->add_last_element(last_async_element, output_format_name); - return last_async_element.release(); + return last_async_element; } Expected> AsyncPipelineBuilder::get_output_format_from_edge_info_name(const std::string &edge_info_name, @@ -307,63 +287,54 @@ hailo_status AsyncPipelineBuilder::add_output_demux_flow(const std::string &outp CHECK(contains(named_stream_infos, output_stream_name), HAILO_INTERNAL_FAILURE); const auto &stream_info = named_stream_infos.at(output_stream_name); - auto source_index = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(output_stream_name); - CHECK_EXPECTED_AS_STATUS(source_index); + TRY(const auto source_index, + async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(output_stream_name)); auto is_empty = false; auto interacts_with_hw = true; - auto hw_queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQueueElement_post_hw", stream_info.name, stream_info.index), - async_pipeline, stream_info.hw_frame_size, is_empty, interacts_with_hw, async_pipeline->get_async_hw_element(), *source_index); - CHECK_EXPECTED_AS_STATUS(hw_queue_elem); + TRY_V(auto hw_queue_elem, + add_push_queue_element(PipelineObject::create_element_name("PushQueueElement_post_hw", stream_info.name, stream_info.index), + async_pipeline, stream_info.hw_frame_size, is_empty, interacts_with_hw, async_pipeline->get_async_hw_element(), source_index)); - auto layer_info = net_group->get_layer_info(output_stream_name); - CHECK_EXPECTED_AS_STATUS(layer_info); + TRY(const auto layer_info, net_group->get_layer_info(output_stream_name)); - auto expected_demuxer = OutputDemuxerBase::create(stream_info.hw_frame_size, *layer_info.value()); - CHECK_EXPECTED_AS_STATUS(expected_demuxer); + TRY(auto demuxer, OutputDemuxerBase::create(stream_info.hw_frame_size, *layer_info)); - auto demuxer_ptr = make_shared_nothrow(expected_demuxer.release()); + auto demuxer_ptr = make_shared_nothrow(std::move(demuxer)); CHECK_ARG_NOT_NULL(demuxer_ptr); - auto demux_elem = TransformDemuxElement::create(demuxer_ptr, + TRY(auto demux_elem, TransformDemuxElement::create(demuxer_ptr, PipelineObject::create_element_name("TransformDemuxEl", output_stream_name, stream_info.index), - async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED_AS_STATUS(demux_elem); - async_pipeline->add_element_to_pipeline(demux_elem.value()); + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); + async_pipeline->add_element_to_pipeline(demux_elem); - CHECK_SUCCESS(PipelinePad::link_pads(hw_queue_elem.value(), demux_elem.value())); + CHECK_SUCCESS(PipelinePad::link_pads(hw_queue_elem, demux_elem)); uint8_t i = 0; for (auto &edge_info : demuxer_ptr->get_edges_stream_info()) { - auto output_format_expected = get_output_format_from_edge_info_name(edge_info.name, outputs_formats); - CHECK_EXPECTED_AS_STATUS(output_format_expected); + TRY(const auto output_format, get_output_format_from_edge_info_name(edge_info.name, outputs_formats)); - is_empty = false; - interacts_with_hw = false; - auto demux_queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_demux", edge_info.name, i), async_pipeline, - edge_info.hw_frame_size, is_empty, interacts_with_hw, demux_elem.value(), i); - CHECK_EXPECTED_AS_STATUS(demux_queue_elem); + TRY(const auto should_transform, OutputTransformContext::is_transformation_required(edge_info.hw_shape, + edge_info.format, edge_info.shape, output_format.second, std::vector{edge_info.quant_info})); // TODO: Get quant vector (HRT-11077) - auto should_transform = OutputTransformContext::is_transformation_required(edge_info.hw_shape, - edge_info.format, edge_info.shape, output_format_expected.value().second, std::vector{edge_info.quant_info}); // TODO: Get quant vector (HRT-11077) - CHECK_EXPECTED_AS_STATUS(should_transform); + if (should_transform) { + is_empty = false; + interacts_with_hw = false; + TRY(auto demux_queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_demux", edge_info.name, i), async_pipeline, + edge_info.hw_frame_size, is_empty, interacts_with_hw, demux_elem, i)); - if (should_transform.value()) { - auto post_infer_elem = add_post_infer_element(output_format_expected.value().second, edge_info.nms_info, - async_pipeline, edge_info.hw_shape, edge_info.format, edge_info.shape, {edge_info.quant_info}, demux_queue_elem.value()); - CHECK_EXPECTED_AS_STATUS(post_infer_elem); + TRY(auto post_infer_elem, add_post_infer_element(output_format.second, edge_info.nms_info, + async_pipeline, edge_info.hw_shape, edge_info.format, edge_info.shape, {edge_info.quant_info}, demux_queue_elem)); auto post_transform_frame_size = (HailoRTCommon::is_nms(edge_info.format.order)) ? - HailoRTCommon::get_nms_host_frame_size(edge_info.nms_info, output_format_expected.value().second) : - HailoRTCommon::get_frame_size(edge_info.shape, output_format_expected.value().second); + HailoRTCommon::get_nms_host_frame_size(edge_info.nms_info, output_format.second) : + HailoRTCommon::get_frame_size(edge_info.shape, output_format.second); - auto last_async_element = add_last_async_element(async_pipeline, output_format_expected.value().first, post_transform_frame_size, - post_infer_elem.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, + post_infer_elem)); } else { - auto last_async_element = add_last_async_element(async_pipeline, output_format_expected.value().first, edge_info.hw_frame_size, - demux_queue_elem.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, edge_info.hw_frame_size, + demux_elem)); } i++; } @@ -373,10 +344,8 @@ hailo_status AsyncPipelineBuilder::add_output_demux_flow(const std::string &outp Expected AsyncPipelineBuilder::should_transform(const hailo_stream_info_t &stream_info, const std::vector &stream_quant_infos, const hailo_format_t &output_format) { - auto should_transform = OutputTransformContext::is_transformation_required(stream_info.hw_shape, + return OutputTransformContext::is_transformation_required(stream_info.hw_shape, stream_info.format, stream_info.shape, output_format, stream_quant_infos); - CHECK_EXPECTED(should_transform); - return should_transform.release(); } hailo_status AsyncPipelineBuilder::add_nms_fuse_flow(const std::vector &output_streams_names, @@ -399,28 +368,24 @@ hailo_status AsyncPipelineBuilder::add_nms_fuse_flow(const std::vectorget_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED_AS_STATUS(nms_elem); - - async_pipeline->add_element_to_pipeline(nms_elem.value()); + TRY(auto nms_elem, NmsMuxElement::create(nms_infos, PipelineObject::create_element_name("NmsMuxEl", fused_layer_name, 0), + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); + async_pipeline->add_element_to_pipeline(nms_elem); uint32_t i = 0; for (const auto &stream_name : output_streams_names) { CHECK(contains(named_stream_infos, stream_name), HAILO_INTERNAL_FAILURE); const auto &curr_stream_info = named_stream_infos.at(stream_name); - auto output_index = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name); - CHECK_EXPECTED_AS_STATUS(output_index); + TRY(const auto output_index, async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name)); auto is_empty = false; auto interacts_with_hw = true; - auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_nms", curr_stream_info.name, curr_stream_info.index), + TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_nms", curr_stream_info.name, curr_stream_info.index), async_pipeline, curr_stream_info.hw_frame_size, is_empty, interacts_with_hw, - async_pipeline->get_async_hw_element(), output_index.value()); - CHECK_EXPECTED_AS_STATUS(queue_elem); + async_pipeline->get_async_hw_element(), output_index)); - CHECK_SUCCESS(PipelinePad::link_pads(queue_elem.value(), nms_elem.value(), 0, i)); + CHECK_SUCCESS(PipelinePad::link_pads(queue_elem, nms_elem, 0, i)); i++; } @@ -428,17 +393,15 @@ hailo_status AsyncPipelineBuilder::add_nms_fuse_flow(const std::vector(1, first_defused_stream_info.quant_info); // On NMS models we always need tp post-infer - auto fused_layer_nms_info = nms_elem.value()->get_fused_nms_info(); + const auto &fused_layer_nms_info = nms_elem->get_fused_nms_info(); - auto post_infer_elem = add_post_infer_element(output_format.second, fused_layer_nms_info, async_pipeline, - first_defused_stream_info.hw_shape, first_defused_stream_info.format, first_defused_stream_info.shape, stream_quant_infos, nms_elem.value()); - CHECK_EXPECTED_AS_STATUS(post_infer_elem); + TRY(auto post_infer_elem, add_post_infer_element(output_format.second, fused_layer_nms_info, async_pipeline, + first_defused_stream_info.hw_shape, first_defused_stream_info.format, first_defused_stream_info.shape, stream_quant_infos, nms_elem)); - auto post_transform_frame_size = HailoRTCommon::get_nms_host_frame_size(fused_layer_nms_info, output_format.second); + const auto post_transform_frame_size = HailoRTCommon::get_nms_host_frame_size(fused_layer_nms_info, output_format.second); - auto last_async_element = add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, - post_infer_elem.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, + post_infer_elem)); return HAILO_SUCCESS; } @@ -455,25 +418,22 @@ hailo_status AsyncPipelineBuilder::add_softmax_flow(std::shared_ptrget_async_hw_element()->get_source_index_from_output_stream_name(stream_name); - CHECK_EXPECTED_AS_STATUS(hw_async_elem_index); + TRY(const auto hw_async_elem_index, async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name)); - auto op_input_format = softmax_op_metadata->inputs_metadata().begin()->second.format; - auto output_format_expanded = net_flow::SoftmaxOpMetadata::expand_output_format_autos(updated_output_format.second, op_input_format); + const auto op_input_format = softmax_op_metadata->inputs_metadata().begin()->second.format; + const auto output_format_expanded = net_flow::SoftmaxOpMetadata::expand_output_format_autos(updated_output_format.second, op_input_format); // TODO (HRT-11078): Fix multi qp for PP auto stream_quant_infos = std::vector(1, stream_info.quant_info); - auto post_infer_elem = add_post_infer_element(output_format_expanded, {}, async_pipeline, stream_info.hw_shape, stream_info.format, - stream_info.shape, stream_quant_infos, async_pipeline->get_async_hw_element(), hw_async_elem_index.value()); - CHECK_EXPECTED_AS_STATUS(post_infer_elem); + TRY(auto post_infer_elem, add_post_infer_element(output_format_expanded, {}, async_pipeline, stream_info.hw_shape, stream_info.format, + stream_info.shape, stream_quant_infos, async_pipeline->get_async_hw_element(), hw_async_elem_index)); auto is_empty = false; auto interacts_with_hw = false; const auto post_transform_frame_size = HailoRTCommon::get_frame_size(stream_info.shape, output_format_expanded); - auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_softmax", async_pipeline->get_async_hw_element()->name(), - static_cast(hw_async_elem_index.value())), async_pipeline, post_transform_frame_size, is_empty, interacts_with_hw, post_infer_elem.value()); - CHECK_EXPECTED_AS_STATUS(queue_elem); + TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_softmax", async_pipeline->get_async_hw_element()->name(), + static_cast(hw_async_elem_index)), async_pipeline, post_transform_frame_size, is_empty, interacts_with_hw, post_infer_elem)); // Updating metadata according to user request // Currently softmax only supports inputs to be float32 and order NHWC or NC @@ -487,21 +447,16 @@ hailo_status AsyncPipelineBuilder::add_softmax_flow(std::shared_ptrset_inputs_metadata(updated_inputs_metadata); CHECK_SUCCESS(metadata->validate_format_info()); - auto op_expected = net_flow::SoftmaxPostProcessOp::create(metadata); - CHECK_EXPECTED_AS_STATUS(op_expected); - - auto softmax_op = op_expected.release(); - auto softmax_element = SoftmaxPostProcessElement::create(softmax_op, + TRY(auto softmax_op, net_flow::SoftmaxPostProcessOp::create(metadata)); + TRY(auto softmax_element, SoftmaxPostProcessElement::create(softmax_op, PipelineObject::create_element_name("SoftmaxPPEl", stream_name, stream_info.index), - async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED_AS_STATUS(softmax_element); + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); - async_pipeline->add_element_to_pipeline(softmax_element.value()); - CHECK_SUCCESS(PipelinePad::link_pads(queue_elem.value(), softmax_element.value())); + async_pipeline->add_element_to_pipeline(softmax_element); + CHECK_SUCCESS(PipelinePad::link_pads(queue_elem, softmax_element)); - auto last_async_element = add_last_async_element(async_pipeline, updated_output_format.first, post_transform_frame_size, - softmax_element.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, add_last_async_element(async_pipeline, updated_output_format.first, post_transform_frame_size, + softmax_element)); return HAILO_SUCCESS; } @@ -516,18 +471,16 @@ hailo_status AsyncPipelineBuilder::add_argmax_flow(std::shared_ptrget_async_hw_element()->get_source_index_from_output_stream_name(stream_name); - CHECK_EXPECTED_AS_STATUS(hw_async_elem_index); + TRY(const auto hw_async_elem_index, async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(stream_name)); auto is_empty = false; auto interacts_with_hw = true; - auto queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_argmax", async_pipeline->get_async_hw_element()->name(), - static_cast(hw_async_elem_index.value())), async_pipeline, stream_info.hw_frame_size, is_empty, interacts_with_hw, - async_pipeline->get_async_hw_element(), hw_async_elem_index.value()); - CHECK_EXPECTED_AS_STATUS(queue_elem); + TRY(auto queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_argmax", async_pipeline->get_async_hw_element()->name(), + static_cast(hw_async_elem_index)), async_pipeline, stream_info.hw_frame_size, is_empty, interacts_with_hw, + async_pipeline->get_async_hw_element(), hw_async_elem_index)); // Updating metadata according to user request - auto op_input_format = argmax_op_metadata->inputs_metadata().begin()->second.format; + const auto op_input_format = argmax_op_metadata->inputs_metadata().begin()->second.format; auto updated_outputs_metadata = argmax_op_metadata.get()->outputs_metadata(); updated_outputs_metadata.begin()->second.format = net_flow::ArgmaxOpMetadata::expand_output_format_autos(output_format.second, op_input_format); auto metadata = std::dynamic_pointer_cast(argmax_op_metadata); @@ -535,24 +488,19 @@ hailo_status AsyncPipelineBuilder::add_argmax_flow(std::shared_ptrset_outputs_metadata(updated_outputs_metadata); CHECK_SUCCESS(metadata->validate_format_info()); - auto op_expected = net_flow::ArgmaxPostProcessOp::create(metadata); - CHECK_EXPECTED_AS_STATUS(op_expected); - auto argmax_op = op_expected.release(); - - auto argmax_element = ArgmaxPostProcessElement::create(argmax_op, + TRY(auto argmax_op, net_flow::ArgmaxPostProcessOp::create(metadata)); + TRY(auto argmax_element, ArgmaxPostProcessElement::create(argmax_op, PipelineObject::create_element_name("ArgmaxPPEl", stream_name, stream_info.index), - async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED_AS_STATUS(argmax_element); + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); - async_pipeline->add_element_to_pipeline(argmax_element.value()); - CHECK_SUCCESS(PipelinePad::link_pads(queue_elem.value(), argmax_element.value())); + async_pipeline->add_element_to_pipeline(argmax_element); + CHECK_SUCCESS(PipelinePad::link_pads(queue_elem, argmax_element)); const auto post_transform_frame_size = HailoRTCommon::get_frame_size(updated_outputs_metadata.begin()->second.shape, updated_outputs_metadata.begin()->second.format); - auto last_async_element = add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, - argmax_element.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, + argmax_element)); return HAILO_SUCCESS; } @@ -601,11 +549,11 @@ hailo_status AsyncPipelineBuilder::add_nms_flow(std::shared_ptr a }; outputs_metadata.insert({nms_op->outputs_metadata().begin()->first, output_metadata}); - auto nms_elem = NmsPostProcessMuxElement::create(nms_op, PipelineObject::create_element_name("NmsPPMuxEl", nms_op->get_name(), 0), - async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED_AS_STATUS(nms_elem); + TRY(auto nms_elem, + NmsPostProcessMuxElement::create(nms_op, PipelineObject::create_element_name("NmsPPMuxEl", nms_op->get_name(), 0), + async_pipeline->get_build_params(), PipelineDirection::PUSH, async_pipeline)); - async_pipeline->add_element_to_pipeline(nms_elem.value()); + async_pipeline->add_element_to_pipeline(nms_elem); hailo_format_t nms_src_format = {}; nms_src_format.flags = HAILO_FORMAT_FLAGS_NONE; @@ -620,24 +568,23 @@ hailo_status AsyncPipelineBuilder::add_nms_flow(std::shared_ptr a // TODO (HRT-11052): Fix multi qp for NMS auto stream_quant_infos = std::vector(1, curr_stream_info.quant_info); //output_stream_base->get_quant_infos(); - auto should_transform = OutputTransformContext::is_transformation_required(curr_stream_info.hw_shape, curr_stream_info.format, - curr_stream_info.hw_shape, nms_src_format, stream_quant_infos); - CHECK_EXPECTED_AS_STATUS(should_transform); + TRY(const auto should_transform, + OutputTransformContext::is_transformation_required(curr_stream_info.hw_shape, curr_stream_info.format, + curr_stream_info.hw_shape, nms_src_format, stream_quant_infos)); - CHECK(!(should_transform.value()), HAILO_INVALID_ARGUMENT, "Unexpected transformation required for {}", curr_stream_name); + CHECK(!(should_transform), HAILO_INVALID_ARGUMENT, "Unexpected transformation required for {}", curr_stream_name); - auto source_id = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(curr_stream_name); - CHECK_EXPECTED_AS_STATUS(source_id); + TRY(const auto source_id, + async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(curr_stream_name)); auto is_empty = false; auto interacts_with_hw = true; - auto nms_source_queue_elem = add_push_queue_element(PipelineObject::create_element_name("PushQEl_nms", curr_stream_info.name, + TRY(auto nms_source_queue_elem, add_push_queue_element(PipelineObject::create_element_name("PushQEl_nms", curr_stream_info.name, curr_stream_info.index), async_pipeline, curr_stream_info.hw_frame_size, is_empty, interacts_with_hw, - async_pipeline->get_async_hw_element(), source_id.value()); - CHECK_EXPECTED_AS_STATUS(nms_source_queue_elem); + async_pipeline->get_async_hw_element(), source_id)); - CHECK_SUCCESS(PipelinePad::link_pads(nms_source_queue_elem.value(), nms_elem.value(), 0, i)); - nms_elem.value()->add_sink_name(curr_stream_name); + CHECK_SUCCESS(PipelinePad::link_pads(nms_source_queue_elem, nms_elem, 0, i)); + nms_elem->add_sink_name(curr_stream_name); } uint32_t post_transform_frame_size; if(nms_op_metadata->nms_config().bbox_only){ @@ -645,9 +592,8 @@ hailo_status AsyncPipelineBuilder::add_nms_flow(std::shared_ptr a } else { post_transform_frame_size = HailoRTCommon::get_nms_host_frame_size(vstream_info.nms_shape, output_format.second); } - auto last_async_element = add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, - nms_elem.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, + add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, nms_elem)); return HAILO_SUCCESS; } @@ -664,44 +610,42 @@ hailo_status AsyncPipelineBuilder::add_iou_flow( std::shared_ptr // TODO (HRT-11078): Fix multi qp for PP auto stream_quant_infos = std::vector(1, output_stream_info.quant_info); //output_stream_base->get_quant_infos(); - auto post_infer_element = add_post_infer_element(output_format.second, output_stream_info.nms_info, + TRY(auto post_infer_element, add_post_infer_element(output_format.second, output_stream_info.nms_info, async_pipeline, output_stream_info.hw_shape, output_stream_info.format, output_stream_info.shape, stream_quant_infos, - async_pipeline->get_async_hw_element()); - CHECK_EXPECTED_AS_STATUS(post_infer_element); + async_pipeline->get_async_hw_element())); auto is_empty = false; auto interacts_with_hw = false; const auto post_transform_frame_size = HailoRTCommon::get_nms_host_frame_size(output_stream_info.nms_info, output_format.second); - auto pre_nms_convert_queue_element = add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_nms_convert", output_stream_name, - output_stream_info.index), async_pipeline, post_transform_frame_size, is_empty, interacts_with_hw, post_infer_element.value()); - CHECK_EXPECTED_AS_STATUS(pre_nms_convert_queue_element); + TRY(auto pre_nms_convert_queue_element, + add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_nms_convert", output_stream_name, + output_stream_info.index), async_pipeline, post_transform_frame_size, is_empty, interacts_with_hw, post_infer_element)); - auto nms_to_detections_element = add_nms_to_detections_convert_element(async_pipeline, output_stream_name, output_stream_info.index, - "NmsFormatToDetectionsEl", iou_op_metadata, pre_nms_convert_queue_element.value()); - CHECK_EXPECTED_AS_STATUS(nms_to_detections_element); + TRY(auto nms_to_detections_element, + add_nms_to_detections_convert_element(async_pipeline, output_stream_name, output_stream_info.index, + "NmsFormatToDetectionsEl", iou_op_metadata, pre_nms_convert_queue_element)); - auto pre_remove_overlapping_bboxes_element_queue_element = add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_bboxes_removing", - output_stream_name, output_stream_info.index), async_pipeline, 0, is_empty, interacts_with_hw, nms_to_detections_element.value()); - CHECK_EXPECTED_AS_STATUS(pre_remove_overlapping_bboxes_element_queue_element); + TRY(auto pre_remove_overlapping_bboxes_element_queue_element, + add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_bboxes_removing", + output_stream_name, output_stream_info.index), async_pipeline, 0, is_empty, interacts_with_hw, nms_to_detections_element)); + + TRY(auto remove_overlapping_bboxes_element, + add_remove_overlapping_bboxes_element(async_pipeline, output_stream_name, output_stream_info.index, + "RemoveOverlappingBboxesEl", iou_op_metadata, pre_remove_overlapping_bboxes_element_queue_element)); - auto remove_overlapping_bboxes_element = add_remove_overlapping_bboxes_element(async_pipeline, output_stream_name, output_stream_info.index, - "RemoveOverlappingBboxesEl", iou_op_metadata, pre_remove_overlapping_bboxes_element_queue_element.value()); - CHECK_EXPECTED_AS_STATUS(remove_overlapping_bboxes_element); + TRY(auto pre_fill_nms_format_element_queue_element, + add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_fill_nms_format", + output_stream_name, output_stream_info.index), async_pipeline, 0, is_empty, interacts_with_hw, remove_overlapping_bboxes_element)); - auto pre_fill_nms_format_element_queue_element = add_push_queue_element(PipelineObject::create_element_name("PushQEl_pre_fill_nms_format", - output_stream_name, output_stream_info.index), async_pipeline, 0, is_empty, interacts_with_hw, remove_overlapping_bboxes_element.value()); - CHECK_EXPECTED_AS_STATUS(pre_fill_nms_format_element_queue_element); + TRY(auto fill_nms_format_element, + add_fill_nms_format_element(async_pipeline, output_stream_name, output_stream_info.index, + "FillNmsFormatEl", iou_op_metadata, pre_fill_nms_format_element_queue_element)); - auto fill_nms_format_element = add_fill_nms_format_element(async_pipeline, output_stream_name, output_stream_info.index, - "FillNmsFormatEl", iou_op_metadata, pre_fill_nms_format_element_queue_element.value()); - CHECK_EXPECTED_AS_STATUS(fill_nms_format_element); + TRY(const auto output_vstream_info, iou_op_metadata->get_output_vstream_info()); + const auto final_frame_size = HailoRTCommon::get_frame_size(output_vstream_info, output_format.second); - auto output_vstream_info = iou_op_metadata->get_output_vstream_info(); - CHECK_EXPECTED_AS_STATUS(output_vstream_info); - const auto final_frame_size = HailoRTCommon::get_frame_size(*output_vstream_info, output_format.second); - - auto last_async_element = add_last_async_element(async_pipeline, output_format.first, final_frame_size, fill_nms_format_element.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, + add_last_async_element(async_pipeline, output_format.first, final_frame_size, fill_nms_format_element)); return HAILO_SUCCESS; } @@ -731,19 +675,22 @@ hailo_status AsyncPipelineBuilder::add_nms_flows(std::shared_ptr { auto metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != metadata); - auto op_expected = net_flow::YOLOXPostProcessOp::create(metadata); - CHECK_EXPECTED_AS_STATUS(op_expected); - op = op_expected.release(); + TRY(op, net_flow::YOLOXPostProcessOp::create(metadata)); break; } case net_flow::OperationType::YOLOV8: { auto metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != metadata); - auto op_expected = net_flow::YOLOV8PostProcessOp::create(metadata); - CHECK_EXPECTED_AS_STATUS(op_expected); - op = op_expected.release(); - break; + if (metadata->nms_config().bbox_only) { + auto bbox_only_metadata = std::dynamic_pointer_cast(op_metadata); + assert(nullptr != bbox_only_metadata); + TRY(op, net_flow::YOLOv8BboxOnlyPostProcessOp::create(bbox_only_metadata)); + break; + } else { + TRY(op, net_flow::YOLOV8PostProcessOp::create(metadata)); + break; + } } case net_flow::OperationType::YOLOV5: { @@ -752,14 +699,10 @@ hailo_status AsyncPipelineBuilder::add_nms_flows(std::shared_ptr if (metadata->nms_config().bbox_only) { auto bbox_only_metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != bbox_only_metadata); - auto op_expected = net_flow::YOLOv5BboxOnlyPostProcessOp::create(bbox_only_metadata); - CHECK_EXPECTED(op_expected); - op = op_expected.release(); + TRY(op, net_flow::YOLOv5BboxOnlyPostProcessOp::create(bbox_only_metadata)); break; } else { - auto op_expected = net_flow::YOLOv5PostProcessOp::create(metadata); - CHECK_EXPECTED_AS_STATUS(op_expected); - op = op_expected.release(); + TRY(op, net_flow::YOLOv5PostProcessOp::create(metadata)); break; } } @@ -767,18 +710,14 @@ hailo_status AsyncPipelineBuilder::add_nms_flows(std::shared_ptr { auto metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != metadata); - auto op_expected = net_flow::Yolov5SegPostProcess::create(metadata); - CHECK_EXPECTED_AS_STATUS(op_expected); - op = op_expected.release(); + TRY(op, net_flow::Yolov5SegPostProcess::create(metadata)); break; } case net_flow::OperationType::SSD: { auto metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != metadata); - auto op_expected = net_flow::SSDPostProcessOp::create(metadata); - CHECK_EXPECTED_AS_STATUS(op_expected); - op = op_expected.release(); + TRY(op, net_flow::SSDPostProcessOp::create(metadata)); break; } default: @@ -833,75 +772,71 @@ hailo_status AsyncPipelineBuilder::create_post_async_hw_elements(std::shared_ptr std::unordered_map op_inputs_to_op_name; for (auto &metadata : net_group->get_ops_metadata().release()) { post_process_metadata.insert({metadata->get_name(), metadata}); - for (auto &input_name : metadata->get_input_names()) { + for (const auto &input_name : metadata->get_input_names()) { op_inputs_to_op_name.insert({input_name, metadata->get_name()}); } } for (auto &output_format : expanded_outputs_formats) { - auto stream_names = net_group->get_stream_names_from_vstream_name(output_format.first); - CHECK_EXPECTED_AS_STATUS(stream_names); + TRY(const auto stream_names, net_group->get_stream_names_from_vstream_name(output_format.first)); - if (contains(streams_added, *stream_names->begin())) { + if (contains(streams_added, *stream_names.begin())) { continue; } - for (auto &output_name : stream_names.value()) { + for (auto &output_name : stream_names) { streams_added.push_back(output_name); } - CHECK(contains(named_stream_infos, *stream_names->begin()), HAILO_INTERNAL_FAILURE); - const auto &first_stream_info = named_stream_infos.at(*stream_names->begin()); + CHECK(contains(named_stream_infos, *stream_names.begin()), HAILO_INTERNAL_FAILURE); + const auto &first_stream_info = named_stream_infos.at(*stream_names.begin()); - if (contains(op_inputs_to_op_name, *stream_names->begin())) { - auto &op_name = op_inputs_to_op_name.at(*stream_names->begin()); + if (contains(op_inputs_to_op_name, *stream_names.begin())) { + const auto &op_name = op_inputs_to_op_name.at(*stream_names.begin()); auto &op_metadata = post_process_metadata.at(op_name); - auto output_vstreams_infos = net_group->get_output_vstream_infos(); - CHECK_EXPECTED_AS_STATUS(output_vstreams_infos); + TRY(const auto output_vstreams_infos, net_group->get_output_vstream_infos()); std::pair original_output_format = {output_format.first, original_outputs_formats.at(output_format.first)}; hailo_status status = add_ops_flows(async_pipeline, original_output_format, - op_metadata, stream_names.value(), output_vstreams_infos.value(), named_stream_infos); + op_metadata, stream_names, output_vstreams_infos, named_stream_infos); CHECK_SUCCESS(status); } else if ((HAILO_FORMAT_ORDER_HAILO_NMS == first_stream_info.format.order) && (first_stream_info.nms_info.is_defused)) { // Case defuse NMS - hailo_status status = add_nms_fuse_flow(stream_names.value(), output_format, async_pipeline, named_stream_infos); + hailo_status status = add_nms_fuse_flow(stream_names, output_format, async_pipeline, named_stream_infos); CHECK_SUCCESS(status); } else if (first_stream_info.is_mux) { // case demux in output from NN core (only one output stream is currently supported) - hailo_status status = add_output_demux_flow(*stream_names->begin(), async_pipeline, expanded_outputs_formats, net_group, named_stream_infos); + hailo_status status = add_output_demux_flow(*stream_names.begin(), async_pipeline, expanded_outputs_formats, net_group, named_stream_infos); CHECK_SUCCESS(status); } else { // case simple and single output from NN core to user (and transformation at best) - auto final_elem_source_index = async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(*stream_names->begin()); - CHECK_EXPECTED_AS_STATUS(final_elem_source_index); + TRY(const auto final_elem_source_index, + async_pipeline->get_async_hw_element()->get_source_index_from_output_stream_name(*stream_names.begin())); - auto layer_info = net_group->get_layer_info(first_stream_info.name); - CHECK_EXPECTED_AS_STATUS(layer_info); - auto stream_quant_infos = layer_info.value()->quant_infos; + TRY(const auto layer_info, net_group->get_layer_info(first_stream_info.name)); + auto stream_quant_infos = layer_info->quant_infos; - auto should_transform_expected = should_transform(first_stream_info, stream_quant_infos, output_format.second); - CHECK_EXPECTED_AS_STATUS(should_transform_expected); + TRY(auto should_transform, + should_transform(first_stream_info, stream_quant_infos, output_format.second)); - if (should_transform_expected.value()) { - auto post_infer_elem = add_post_infer_element(output_format.second, first_stream_info.nms_info, async_pipeline, first_stream_info.hw_shape, - first_stream_info.format, first_stream_info.shape, stream_quant_infos, async_pipeline->get_async_hw_element(), final_elem_source_index.value()); - CHECK_EXPECTED_AS_STATUS(post_infer_elem); + if (should_transform) { + TRY(auto post_infer_elem, + add_post_infer_element(output_format.second, first_stream_info.nms_info, async_pipeline, first_stream_info.hw_shape, + first_stream_info.format, first_stream_info.shape, stream_quant_infos, async_pipeline->get_async_hw_element(), + final_elem_source_index)); - auto post_transform_frame_size = (HailoRTCommon::is_nms(first_stream_info.format.order)) ? + const auto post_transform_frame_size = (HailoRTCommon::is_nms(first_stream_info.format.order)) ? HailoRTCommon::get_nms_host_frame_size(first_stream_info.nms_info, output_format.second) : HailoRTCommon::get_frame_size(first_stream_info.shape, output_format.second); - auto last_async_element = add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, - post_infer_elem.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, post_transform_frame_size, + post_infer_elem)); } else { - auto last_async_element = add_last_async_element(async_pipeline, output_format.first, first_stream_info.hw_frame_size, - async_pipeline->get_async_hw_element(), final_elem_source_index.value()); - CHECK_EXPECTED_AS_STATUS(last_async_element); + TRY(auto last_async_element, add_last_async_element(async_pipeline, output_format.first, first_stream_info.hw_frame_size, + async_pipeline->get_async_hw_element(), final_elem_source_index)); } } } @@ -921,54 +856,41 @@ Expected> AsyncPipelineBuilder::create_pipeline(s // Buffer pool sizes for pipeline elements should be: // * The minimum of the maximum queue size of all LL streams (input and output) - for edge elements // * HAILO_DEFAULT_ASYNC_INFER_QUEUE_SIZE - for internal elements - auto buffer_pool_size_expected = net_group->get_min_buffer_pool_size(); - CHECK_EXPECTED(buffer_pool_size_expected); - build_params.buffer_pool_size_edges = buffer_pool_size_expected.release(); - build_params.buffer_pool_size_internal = std::min(static_cast(build_params.buffer_pool_size_edges), static_cast(HAILO_DEFAULT_ASYNC_INFER_QUEUE_SIZE)); + TRY(build_params.buffer_pool_size_edges, net_group->get_min_buffer_pool_size()); + build_params.buffer_pool_size_internal = std::min(static_cast(build_params.buffer_pool_size_edges), + static_cast(HAILO_DEFAULT_ASYNC_INFER_QUEUE_SIZE)); build_params.elem_stats_flags = HAILO_PIPELINE_ELEM_STATS_NONE; build_params.vstream_stats_flags = HAILO_VSTREAM_STATS_NONE; - auto async_pipeline_expected = AsyncPipeline::create_shared(); - CHECK_EXPECTED(async_pipeline_expected); - auto async_pipeline = async_pipeline_expected.release(); - - auto all_stream_infos = net_group->get_all_stream_infos(); - CHECK_EXPECTED(all_stream_infos); + TRY(auto async_pipeline, AsyncPipeline::create_shared()); + TRY(const auto all_stream_infos, net_group->get_all_stream_infos()); std::unordered_map named_stream_infos; - for (const auto &info : all_stream_infos.value()) { + for (const auto &info : all_stream_infos) { named_stream_infos.emplace(info.name, info); } - auto input_expanded_format = expand_auto_input_formats(net_group, inputs_formats, named_stream_infos); - CHECK_EXPECTED(input_expanded_format); - - auto output_expanded_format = expand_auto_output_formats(net_group, outputs_formats, named_stream_infos); - CHECK_EXPECTED(output_expanded_format); - + TRY(const auto input_expanded_format, expand_auto_input_formats(net_group, inputs_formats, named_stream_infos)); + TRY(const auto output_expanded_format, expand_auto_output_formats(net_group, outputs_formats, named_stream_infos)); auto outputs_original_formats = outputs_formats; // The original formats is needed for specific format expanding (required for PP OPs, like argmax) - auto shutdown_event_expected = Event::create_shared(Event::State::not_signalled); - CHECK_EXPECTED(shutdown_event_expected); - - build_params.shutdown_event = shutdown_event_expected.release(); + TRY(build_params.shutdown_event, Event::create_shared(Event::State::not_signalled)); build_params.pipeline_status = pipeline_status; build_params.timeout = std::chrono::milliseconds(timeout); async_pipeline->set_build_params(build_params); - auto async_hw_elem = AsyncHwElement::create(named_stream_infos, build_params.timeout, + TRY(auto async_hw_elem, AsyncHwElement::create(named_stream_infos, build_params.timeout, build_params.elem_stats_flags, "AsyncHwEl", build_params.pipeline_status, net_group, - PipelineDirection::PUSH, async_pipeline); - CHECK_EXPECTED(async_hw_elem); - async_pipeline->add_element_to_pipeline(async_hw_elem.value()); - async_pipeline->set_async_hw_element(async_hw_elem.release()); + PipelineDirection::PUSH, async_pipeline)); + async_pipeline->add_element_to_pipeline(async_hw_elem); + async_pipeline->set_async_hw_element(async_hw_elem); - hailo_status status = create_pre_async_hw_elements(net_group, input_expanded_format.value(), named_stream_infos, + hailo_status status = create_pre_async_hw_elements(net_group, input_expanded_format, named_stream_infos, async_pipeline); CHECK_SUCCESS_AS_EXPECTED(status); - status = create_post_async_hw_elements(net_group, output_expanded_format.value(), outputs_original_formats, named_stream_infos, + status = create_post_async_hw_elements(net_group, output_expanded_format, outputs_original_formats, named_stream_infos, async_pipeline); CHECK_SUCCESS_AS_EXPECTED(status); @@ -995,15 +917,13 @@ Expected> AsyncPipelineBuilder::create_multi_p HAILO_INVALID_ARGUMENT, "The given order ({}) is not a multi-planar order", HailoRTCommon::get_format_order_str(order)); // TODO: Support fps/latency collection for queue elems (HRT-7711) - auto duration_collector_expected = DurationCollector::create(HAILO_PIPELINE_ELEM_STATS_NONE); - CHECK_EXPECTED(duration_collector_expected); + TRY(auto duration_collector, DurationCollector::create(HAILO_PIPELINE_ELEM_STATS_NONE)); - auto planes_splitter = PixBufferElement::create(PipelineObject::create_element_name("PixBufEl", - input_name, 0), std::chrono::milliseconds(HAILO_INFINITE), duration_collector_expected.release(), pipeline_status, order, - async_pipeline); - CHECK_EXPECTED(planes_splitter); + TRY(auto planes_splitter, PixBufferElement::create(PipelineObject::create_element_name("PixBufEl", + input_name, 0), std::chrono::milliseconds(HAILO_INFINITE), std::move(duration_collector), pipeline_status, order, + async_pipeline)); - return planes_splitter.release(); + return planes_splitter; } } /* namespace hailort */ diff --git a/hailort/libhailort/src/net_flow/pipeline/configured_infer_model_hrpc_client.cpp b/hailort/libhailort/src/net_flow/pipeline/configured_infer_model_hrpc_client.cpp new file mode 100644 index 0000000..32ad5d1 --- /dev/null +++ b/hailort/libhailort/src/net_flow/pipeline/configured_infer_model_hrpc_client.cpp @@ -0,0 +1,484 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file configured_infer_model_hrpc_client.cpp + * @brief ConfiguredInferModel HRPC client implementation + **/ + +#include "configured_infer_model_hrpc_client.hpp" +#include "hailo/hailort.h" +#include + +namespace hailort +{ + +Expected InferStreamOnStack::get_buffer() +{ + return MemoryView(m_buffer); +} + +Expected OutputBindingsOnStack::create(ConfiguredInferModel::Bindings bindings, + const std::vector &outputs_names) +{ + std::unordered_map output_streams; + for (const auto &output_name : outputs_names) { + TRY(auto output, bindings.output(output_name)); + TRY(auto buffer, output.get_buffer()); + output_streams.emplace(output_name, InferStreamOnStack(buffer)); + } + return OutputBindingsOnStack(std::move(output_streams)); +} + +Expected OutputBindingsOnStack::output() +{ + CHECK_AS_EXPECTED(1 == m_output_streams.size(), HAILO_INVALID_OPERATION, "Model has more than one output!"); + auto copy = m_output_streams.begin()->second; + return copy; +} + +Expected OutputBindingsOnStack::output(const std::string &name) +{ + CHECK_AS_EXPECTED(contains(m_output_streams, name), HAILO_NOT_FOUND, "Output {}, not found!", name); + auto copy = m_output_streams.at(name); + return copy; +} + +AsyncInferJobHrpcClient::AsyncInferJobHrpcClient(EventPtr event) : m_event(event) +{ +} + +hailo_status AsyncInferJobHrpcClient::wait(std::chrono::milliseconds timeout) +{ + return m_event->wait(timeout); +} + +CallbacksQueue::CallbacksQueue(std::shared_ptr client, const std::vector &outputs_names) : + m_outputs_names(outputs_names) +{ + client->register_custom_reply(HailoRpcActionID::CALLBACK_CALLED, + [this, &outputs_names] (const MemoryView &serialized_reply, hrpc::RpcConnection connection) -> hailo_status { + TRY(auto tuple, CallbackCalledSerializer::deserialize_reply(serialized_reply)); + + auto callback_status = std::get<0>(tuple); + auto callback_handle_id = std::get<1>(tuple); + + { + std::unique_lock lock(m_mutex); + CHECK(contains(m_callbacks, callback_handle_id), HAILO_NOT_FOUND, "Callback handle not found!"); + m_callbacks_status[callback_handle_id] = callback_status; + + if (HAILO_SUCCESS == callback_status) { + CHECK(contains(m_bindings, callback_handle_id), HAILO_NOT_FOUND, "Callback handle not found!"); + for (const auto &output_name : outputs_names) { + TRY(auto buffer, m_bindings.at(callback_handle_id).output(output_name)->get_buffer()); + auto status = connection.read_buffer(buffer); + // TODO: Errors here should be unrecoverable (HRT-14275) + CHECK_SUCCESS(status); + } + } + m_callbacks_queue.push(callback_handle_id); + } + + m_cv.notify_one(); + return HAILO_SUCCESS; + }); + + m_is_running = true; + m_callback_thread = std::thread([this] { + while (true) { + callback_id_t callback_id; + hailo_status info_status = HAILO_UNINITIALIZED; + std::function cb; + { + std::unique_lock lock(m_mutex); + m_cv.wait(lock, [this] { return !m_is_running || !m_callbacks_queue.empty(); }); + if (!m_is_running) { + break; + } + + callback_id = m_callbacks_queue.front(); + m_callbacks_queue.pop(); + + m_cv.wait(lock, [this, callback_id] { return !m_is_running || (m_callbacks.find(callback_id) != m_callbacks.end()); }); + if (!m_is_running) { + break; + } + + info_status = m_callbacks_status[callback_id]; + cb = m_callbacks[callback_id]; + m_callbacks.erase(callback_id); + m_callbacks_status.erase(callback_id); + m_bindings.erase(callback_id); + } + AsyncInferCompletionInfo info(info_status); + cb(info); + } + }); +} + +CallbacksQueue::~CallbacksQueue() +{ + { + std::unique_lock lock(m_mutex); + m_is_running = false; + } + m_cv.notify_one(); + m_callback_thread.join(); +} +Expected> CallbacksQueue::register_callback(callback_id_t id, + ConfiguredInferModel::Bindings bindings, + std::function callback) +{ + TRY(auto event_ptr, Event::create_shared(Event::State::not_signalled)); + + { + std::unique_lock lock(m_mutex); + TRY(auto output_bindings, OutputBindingsOnStack::create(bindings, m_outputs_names)); + m_bindings.emplace(id, output_bindings); + m_callbacks_status[id] = HAILO_SUCCESS; + m_callbacks[id] = [callback, event_ptr] (const AsyncInferCompletionInfo &info) { + auto status = event_ptr->signal(); + if (HAILO_SUCCESS != status) { + LOGGER__CRITICAL("Could not signal event! status = {}", status); + } + callback(info); + }; + } + m_cv.notify_one(); + + auto ptr = make_shared_nothrow(event_ptr); + CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +Expected> ConfiguredInferModelHrpcClient::create(std::shared_ptr client, + rpc_object_handle_t handle_id, std::vector &&input_vstream_infos, + std::vector &&output_vstream_infos, uint32_t max_ongoing_transfers, + std::unique_ptr &&callbacks_queue, rpc_object_handle_t infer_model_id, + const std::unordered_map inputs_frame_sizes, + const std::unordered_map outputs_frame_sizes) +{ + // TODO: consider create a separate client object here - HRT-13687 + auto ptr = make_shared_nothrow(client, handle_id, std::move(input_vstream_infos), + std::move(output_vstream_infos), max_ongoing_transfers, std::move(callbacks_queue), infer_model_id, inputs_frame_sizes, + outputs_frame_sizes); + CHECK_NOT_NULL(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +ConfiguredInferModelHrpcClient::~ConfiguredInferModelHrpcClient() +{ + if (INVALID_HANDLE_ID == m_handle_id) { + return; + } + + auto request = DestroyConfiguredInferModelSerializer::serialize_request(m_handle_id); + if (!request) { + LOGGER__CRITICAL("Failed to serialize ConfiguredInferModel_release request"); + return; + } + + auto client = m_client.lock(); + if (client) { + auto result = client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__DESTROY, MemoryView(*request)); + if (!result) { + LOGGER__CRITICAL("Failed to destroy configured infer model! status = {}", result.status()); + } + + if (HAILO_SUCCESS != DestroyConfiguredInferModelSerializer::deserialize_reply(MemoryView(*result))) { + LOGGER__CRITICAL("Failed to destroy configured infer model! status = {}", result.status()); + } + } +} + +Expected ConfiguredInferModelHrpcClient::create_bindings() +{ + std::unordered_map inputs; + std::unordered_map outputs; + + for (const auto &vstream_info : m_input_vstream_infos) { + TRY(auto stream, ConfiguredInferModelBase::create_infer_stream(vstream_info)); + inputs.emplace(vstream_info.name, std::move(stream)); + } + + for (const auto &vstream_info : m_output_vstream_infos) { + TRY(auto stream, ConfiguredInferModelBase::create_infer_stream(vstream_info)); + outputs.emplace(vstream_info.name, std::move(stream)); + } + + TRY(auto bindings, ConfiguredInferModelBase::create_bindings(std::move(inputs), std::move(outputs))); + return bindings; +} + +hailo_status ConfiguredInferModelHrpcClient::wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count) +{ + std::unique_lock lock(m_ongoing_transfers_mutex); + bool done = m_cv.wait_for(lock, timeout, [this, frames_count] () { + return (m_max_ongoing_transfers - m_ongoing_transfers.load()) >= frames_count; + }); + CHECK(done, HAILO_TIMEOUT); + + return HAILO_SUCCESS; +} + +Expected ConfiguredInferModelHrpcClient::run_async(ConfiguredInferModel::Bindings bindings, + std::function callback) +{ + auto async_job = run_async_impl(bindings, callback); + if (HAILO_SUCCESS != async_job.status()) { + shutdown(); + return make_unexpected(async_job.status()); + } + return async_job.release(); +} + +Expected ConfiguredInferModelHrpcClient::run_async_impl(ConfiguredInferModel::Bindings bindings, + std::function callback) +{ + CHECK_SUCCESS_AS_EXPECTED(validate_bindings(bindings)); + std::unique_lock lock(m_infer_mutex); + m_callbacks_counter++; + auto callback_wrapper = [this, callback] (const AsyncInferCompletionInfo &info) { + { + std::unique_lock transfers_lock(m_ongoing_transfers_mutex); + m_ongoing_transfers--; + } + m_cv.notify_one(); + if (callback) { + callback(info); + } + }; + + TRY(auto job_ptr, m_callbacks_queue->register_callback(m_callbacks_counter, bindings, callback_wrapper)); + + TRY(auto request, RunAsyncSerializer::serialize_request(m_handle_id, m_infer_model_handle_id, + m_callbacks_counter)); + + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use."); + TRY(auto serialized_result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__RUN_ASYNC, + MemoryView(request), [this, &bindings] (hrpc::RpcConnection connection) -> hailo_status { + for (const auto &input_vstream : m_input_vstream_infos) { + TRY(auto input, bindings.input(input_vstream.name)); + auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(input); + switch(buffer_type) { + case BufferType::VIEW: + { + TRY(auto buffer, input.get_buffer()); + auto status = connection.write_buffer(MemoryView(buffer)); + CHECK_SUCCESS(status); + break; + } + case BufferType::PIX_BUFFER: + { + TRY(auto pix_buffer, input.get_pix_buffer()); + for (uint32_t i = 0; i < pix_buffer.number_of_planes; i++) { + auto status = connection.write_buffer(MemoryView(pix_buffer.planes[i].user_ptr, pix_buffer.planes[i].bytes_used)); + CHECK_SUCCESS(status); + } + break; + } + case BufferType::DMA_BUFFER: + LOGGER__CRITICAL("DMA_BUFFER is not supported in HRPC"); + return HAILO_NOT_IMPLEMENTED; + default: + LOGGER__CRITICAL("Unknown buffer type"); + return HAILO_INTERNAL_FAILURE; + } + } + return HAILO_SUCCESS; + })); + auto status = RunAsyncSerializer::deserialize_reply(MemoryView(serialized_result)); + CHECK_SUCCESS_AS_EXPECTED(status); + + { + std::unique_lock transfers_lock(m_ongoing_transfers_mutex); + m_ongoing_transfers++; + } + + return AsyncInferJobBase::create(job_ptr); +} + +hailo_status ConfiguredInferModelHrpcClient::set_scheduler_timeout(const std::chrono::milliseconds &timeout) +{ + TRY(auto serialized_request, SetSchedulerTimeoutSerializer::serialize_request(m_handle_id, timeout)); + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use."); + TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_TIMEOUT, MemoryView(serialized_request))); + CHECK_SUCCESS(SetSchedulerTimeoutSerializer::deserialize_reply(MemoryView(result))); + + return HAILO_SUCCESS; +} + +hailo_status ConfiguredInferModelHrpcClient::set_scheduler_threshold(uint32_t threshold) +{ + TRY(auto serialized_request, SetSchedulerThresholdSerializer::serialize_request(m_handle_id, threshold)); + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use."); + TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_THRESHOLD, MemoryView(serialized_request))); + CHECK_SUCCESS(SetSchedulerThresholdSerializer::deserialize_reply(MemoryView(result))); + + return HAILO_SUCCESS; +} + +hailo_status ConfiguredInferModelHrpcClient::set_scheduler_priority(uint8_t priority) +{ + TRY(auto serialized_request, SetSchedulerPrioritySerializer::serialize_request(m_handle_id, priority)); + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use."); + TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__SET_SCHEDULER_PRIORITY, MemoryView(serialized_request))); + CHECK_SUCCESS(SetSchedulerPrioritySerializer::deserialize_reply(MemoryView(result))); + + return HAILO_SUCCESS; +} + +Expected ConfiguredInferModelHrpcClient::get_hw_latency_measurement() +{ + TRY(auto serialized_request, GetHwLatencyMeasurementSerializer::serialize_request(m_handle_id)); + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use."); + TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__GET_HW_LATENCY_MEASUREMENT, MemoryView(serialized_request))); + + TRY(auto tuple, GetHwLatencyMeasurementSerializer::deserialize_reply(MemoryView(result))); + + auto status = std::get<0>(tuple); + if (HAILO_NOT_AVAILABLE == status) { + return make_unexpected(HAILO_NOT_AVAILABLE); + } + CHECK_SUCCESS(status); + + auto avg_hw_latency = std::get<1>(tuple); + LatencyMeasurementResult latency_measurement_result {avg_hw_latency}; + + return latency_measurement_result; +}; + +hailo_status ConfiguredInferModelHrpcClient::activate() +{ + TRY(auto serialized_request, ActivateSerializer::serialize_request(m_handle_id)); + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use."); + TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__ACTIVATE, MemoryView(serialized_request))); + + CHECK_SUCCESS(ActivateSerializer::deserialize_reply(MemoryView(result))); + + return HAILO_SUCCESS; +}; + +hailo_status ConfiguredInferModelHrpcClient::deactivate() +{ + TRY(auto serialized_request, DeactivateSerializer::serialize_request(m_handle_id)); + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use."); + TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__DEACTIVATE, MemoryView(serialized_request))); + + CHECK_SUCCESS(DeactivateSerializer::deserialize_reply(MemoryView(result))); + + return HAILO_SUCCESS; +}; + +Expected ConfiguredInferModelHrpcClient::get_async_queue_size() +{ + size_t queue_size = m_max_ongoing_transfers; + return queue_size; +} + +hailo_status ConfiguredInferModelHrpcClient::validate_bindings(ConfiguredInferModel::Bindings bindings) +{ + for (const auto &input_vstream : m_input_vstream_infos) { + TRY(auto input, bindings.input(input_vstream.name)); + + auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(input); + switch (buffer_type) { + case BufferType::VIEW: + { + auto buffer = input.get_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + CHECK(buffer->size() == m_inputs_frame_sizes.at(input_vstream.name), HAILO_INVALID_OPERATION, + "Input buffer size {} is different than expected {} for input '{}'", buffer->size(), m_inputs_frame_sizes.at(input_vstream.name), input_vstream.name); + break; + } + case BufferType::PIX_BUFFER: + { + auto buffer = input.get_pix_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + size_t buffer_size = 0; + for (size_t i = 0 ; i < buffer->number_of_planes ; i++) { + buffer_size += buffer->planes[i].bytes_used; + } + + CHECK(buffer_size == m_inputs_frame_sizes.at(input_vstream.name), HAILO_INVALID_OPERATION, + "Input buffer size {} is different than expected {} for input '{}'", buffer_size, m_inputs_frame_sizes.at(input_vstream.name), input_vstream.name); + break; + } + case BufferType::DMA_BUFFER: + { + auto buffer = input.get_dma_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + CHECK(buffer->size == m_inputs_frame_sizes.at(input_vstream.name), HAILO_INVALID_OPERATION, + "Input buffer size {} is different than expected {} for input '{}'", buffer->size, m_inputs_frame_sizes.at(input_vstream.name), input_vstream.name); + break; + } + default: + CHECK(false, HAILO_NOT_FOUND, "Couldnt find input buffer for '{}'", input_vstream.name); + } + } + for (const auto &output_vstream : m_output_vstream_infos) { + TRY(auto output, bindings.output(output_vstream.name)); + auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(output); + switch (buffer_type) { + case BufferType::VIEW: + { + auto buffer = output.get_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + CHECK(buffer->size() == m_outputs_frame_sizes.at(output_vstream.name), HAILO_INVALID_OPERATION, + "Output buffer size {} is different than expected {} for output '{}'", buffer->size(), m_outputs_frame_sizes.at(output_vstream.name), output_vstream.name); + break; + } + case BufferType::PIX_BUFFER: + { + CHECK(false, HAILO_NOT_SUPPORTED, "pix_buffer isn't supported for outputs in '{}'", output_vstream.name); + break; + } + case BufferType::DMA_BUFFER: + { + auto buffer = output.get_dma_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + CHECK(buffer->size == m_outputs_frame_sizes.at(output_vstream.name), HAILO_INVALID_OPERATION, + "Output buffer size {} is different than expected {} for out '{}'", buffer->size, m_outputs_frame_sizes.at(output_vstream.name), output_vstream.name); + break; + } + default: + CHECK(false, HAILO_NOT_FOUND, "Couldnt find output buffer for '{}'", output_vstream.name); + } + } + + return HAILO_SUCCESS; +} + +hailo_status ConfiguredInferModelHrpcClient::shutdown() +{ + TRY(auto serialized_request, ShutdownSerializer::serialize_request(m_handle_id)); + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the ConfiguredInferModel is in use."); + TRY(auto result, client->execute_request(HailoRpcActionID::CONFIGURED_INFER_MODEL__SHUTDOWN, MemoryView(serialized_request))); + + CHECK_SUCCESS(ShutdownSerializer::deserialize_reply(MemoryView(result))); + + return HAILO_SUCCESS; +} + + +} // namespace hailort diff --git a/hailort/libhailort/src/net_flow/pipeline/configured_infer_model_hrpc_client.hpp b/hailort/libhailort/src/net_flow/pipeline/configured_infer_model_hrpc_client.hpp new file mode 100644 index 0000000..9cac520 --- /dev/null +++ b/hailort/libhailort/src/net_flow/pipeline/configured_infer_model_hrpc_client.hpp @@ -0,0 +1,151 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file configured_infer_model_hrpc_client.hpp + * @brief ConfiguredInferModel HRPC client, represents the user's handle to the ConfiguredInferModel object (held in the hailort server) + **/ + +#ifndef _HAILO_CONFIGURED_INFER_MODEL_HRPC_CLIENT_HPP_ +#define _HAILO_CONFIGURED_INFER_MODEL_HRPC_CLIENT_HPP_ + +#include "hailo/infer_model.hpp" +#include "infer_model_internal.hpp" +#include "hrpc/client.hpp" + +namespace hailort +{ + +using callback_id_t = uint32_t; + +class InferStreamOnStack final +{ +public: + InferStreamOnStack(MemoryView buffer) : m_buffer(buffer) {} + Expected get_buffer(); + +private: + MemoryView m_buffer; +}; + +class OutputBindingsOnStack final +{ +public: + static Expected create(ConfiguredInferModel::Bindings bindings, + const std::vector &outputs_names); + Expected output(); + Expected output(const std::string &name); + +private: + OutputBindingsOnStack(std::unordered_map &&output_streams) : + m_output_streams(std::move(output_streams)) {} + + std::unordered_map m_output_streams; +}; + +class AsyncInferJobHrpcClient : public AsyncInferJobBase +{ +public: + AsyncInferJobHrpcClient(EventPtr event); + + virtual hailo_status wait(std::chrono::milliseconds timeout) override; + +private: + EventPtr m_event; +}; + +class CallbacksQueue +{ +public: + CallbacksQueue(std::shared_ptr client, const std::vector &outputs_names); + ~CallbacksQueue(); + + CallbacksQueue(const CallbacksQueue &other) = delete; + CallbacksQueue& operator=(const CallbacksQueue &other) = delete; + CallbacksQueue(CallbacksQueue &&other) = delete; + CallbacksQueue& operator=(CallbacksQueue &&other) = delete; + + Expected> register_callback(callback_id_t id, + ConfiguredInferModel::Bindings bindings, + std::function callback); + +private: + const std::vector m_outputs_names; + std::mutex m_mutex; + std::condition_variable m_cv; + std::queue m_callbacks_queue; + std::unordered_map> m_callbacks; + std::atomic_bool m_is_running; + std::thread m_callback_thread; + std::unordered_map m_bindings; + std::unordered_map m_callbacks_status; +}; + +class ConfiguredInferModelHrpcClient : public ConfiguredInferModelBase +{ +public: + static Expected> create(std::shared_ptr client, + rpc_object_handle_t handle_id, std::vector &&input_vstream_infos, + std::vector &&output_vstream_infos, uint32_t max_ongoing_transfers, + std::unique_ptr &&callbacks_queue, rpc_object_handle_t infer_model_handle_id, + const std::unordered_map inputs_frame_sizes, + const std::unordered_map outputs_frame_sizes); + ConfiguredInferModelHrpcClient(std::shared_ptr client, rpc_object_handle_t handle_id, + std::vector &&input_vstream_infos, std::vector &&output_vstream_infos, + uint32_t max_ongoing_transfers, std::unique_ptr &&callbacks_queue, rpc_object_handle_t infer_model_handle_id, + const std::unordered_map inputs_frame_sizes, + const std::unordered_map outputs_frame_sizes) : + ConfiguredInferModelBase(inputs_frame_sizes, outputs_frame_sizes), + m_client(client), m_handle_id(handle_id), m_input_vstream_infos(std::move(input_vstream_infos)), + m_output_vstream_infos(std::move(output_vstream_infos)), m_max_ongoing_transfers(max_ongoing_transfers), + m_ongoing_transfers(0), m_callbacks_queue(std::move(callbacks_queue)), m_infer_model_handle_id(infer_model_handle_id), + m_callbacks_counter(0) {} + virtual ~ConfiguredInferModelHrpcClient(); + + ConfiguredInferModelHrpcClient(const ConfiguredInferModelHrpcClient &) = delete; + ConfiguredInferModelHrpcClient &operator=(const ConfiguredInferModelHrpcClient &) = delete; + ConfiguredInferModelHrpcClient(ConfiguredInferModelHrpcClient &&) = delete; + ConfiguredInferModelHrpcClient &operator=(ConfiguredInferModelHrpcClient &&) = delete; + + virtual Expected create_bindings() override; + virtual hailo_status wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count) override; + + virtual hailo_status activate() override; + virtual hailo_status deactivate() override; + + virtual Expected run_async(ConfiguredInferModel::Bindings bindings, + std::function callback) override; + + virtual Expected get_hw_latency_measurement() override; + + virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout) override; + virtual hailo_status set_scheduler_threshold(uint32_t threshold) override; + virtual hailo_status set_scheduler_priority(uint8_t priority) override; + + virtual Expected get_async_queue_size() override; + + virtual hailo_status shutdown() override; + +private: + virtual hailo_status validate_bindings(ConfiguredInferModel::Bindings bindings); + Expected run_async_impl(ConfiguredInferModel::Bindings bindings, + std::function callback); + + std::weak_ptr m_client; + rpc_object_handle_t m_handle_id; + std::vector m_input_vstream_infos; + std::vector m_output_vstream_infos; + uint32_t m_max_ongoing_transfers; + std::mutex m_ongoing_transfers_mutex; + std::condition_variable m_cv; + std::atomic_uint32_t m_ongoing_transfers; + std::unique_ptr m_callbacks_queue; + rpc_object_handle_t m_infer_model_handle_id; + std::atomic_uint32_t m_callbacks_counter; + std::mutex m_infer_mutex; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_CONFIGURED_INFER_MODEL_HRPC_CLIENT_HPP_ */ diff --git a/hailort/libhailort/src/net_flow/pipeline/edge_elements.cpp b/hailort/libhailort/src/net_flow/pipeline/edge_elements.cpp index f0b6e2b..9c1e012 100644 --- a/hailort/libhailort/src/net_flow/pipeline/edge_elements.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/edge_elements.cpp @@ -38,18 +38,15 @@ Expected> HwWriteElement::create(std::shared_ptr hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr> pipeline_status, PipelineDirection pipeline_direction) { - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); - - auto got_flush_event = Event::create_shared(Event::State::not_signalled); - CHECK_EXPECTED(got_flush_event); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); + TRY(auto got_flush_event, Event::create_shared(Event::State::not_signalled)); // On HwWriteElement the stream always owns the buffer, hence, we set the mode explicitly. auto status = stream->set_buffer_mode(StreamBufferMode::OWNING); CHECK_SUCCESS_AS_EXPECTED(status); auto hw_write_elem_ptr = make_shared_nothrow(stream, name, - duration_collector.release(), std::move(pipeline_status), got_flush_event.release(), pipeline_direction); + std::move(duration_collector), std::move(pipeline_status), std::move(got_flush_event), pipeline_direction); CHECK_AS_EXPECTED(nullptr != hw_write_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", hw_write_elem_ptr->description()); @@ -175,17 +172,16 @@ Expected> LastAsyncElement::create(const std:: hailo_vstream_stats_flags_t vstream_stats_flags, std::shared_ptr> pipeline_status, size_t queue_size, size_t frame_size, EventPtr shutdown_event, std::shared_ptr async_pipeline) { - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); auto is_empty = true; // LastAsync always holds user buffers, therefore its created empty auto is_dma_able = false; queue_size = queue_size * 2; // Multiplying by 2 to ensure dual-buffering when edge-element is the bottleneck - auto buffer_pool = BufferPool::create(frame_size, queue_size, shutdown_event, elem_flags, vstream_stats_flags, is_empty, is_dma_able); - CHECK_EXPECTED(buffer_pool); + TRY(auto buffer_pool, + BufferPool::create(frame_size, queue_size, shutdown_event, elem_flags, vstream_stats_flags, is_empty, is_dma_able)); auto last_async_elem_ptr = make_shared_nothrow(name, - duration_collector.release(), std::move(pipeline_status), buffer_pool.release(), async_pipeline); + std::move(duration_collector), std::move(pipeline_status), std::move(buffer_pool), async_pipeline); CHECK_NOT_NULL_AS_EXPECTED(last_async_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", last_async_elem_ptr->description()); @@ -228,14 +224,14 @@ hailo_status LastAsyncElement::execute_activate() return HAILO_SUCCESS; } -hailo_status LastAsyncElement::enqueue_execution_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done) +hailo_status LastAsyncElement::enqueue_execution_buffer(PipelineBuffer &&pipeline_buffer) { - return m_pool->enqueue_buffer(mem_view, exec_done); + return m_pool->enqueue_buffer(std::move(pipeline_buffer)); } -Expected LastAsyncElement::can_push_buffer_upstream() +Expected LastAsyncElement::can_push_buffer_upstream(uint32_t frames_count) { - return !m_pool->is_full(); + return (m_pool->num_of_buffers_in_pool() + frames_count) < m_pool->max_capacity(); } SourceElement::SourceElement(const std::string &name, DurationCollector &&duration_collector, std::shared_ptr> &&pipeline_status, @@ -278,16 +274,14 @@ Expected> HwReadElement::create(std::shared_ptrset_buffer_mode(StreamBufferMode::OWNING); CHECK_SUCCESS_AS_EXPECTED(status); - auto duration_collector = DurationCollector::create(build_params.elem_stats_flags); - CHECK_EXPECTED(duration_collector); + TRY(auto duration_collector, DurationCollector::create(build_params.elem_stats_flags)); auto pipeline_status = build_params.pipeline_status; - auto shutdown_event = Event::create_shared(Event::State::not_signalled); - CHECK_EXPECTED(shutdown_event); + TRY(auto shutdown_event, Event::create_shared(Event::State::not_signalled)); auto hw_read_elem_ptr = make_shared_nothrow(stream, name, build_params.timeout, - duration_collector.release(), shutdown_event.release(), std::move(pipeline_status), pipeline_direction); + std::move(duration_collector), std::move(shutdown_event), std::move(pipeline_status), pipeline_direction); CHECK_AS_EXPECTED(nullptr != hw_read_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", hw_read_elem_ptr->description()); @@ -365,11 +359,9 @@ Expected HwReadElement::run_pull(PipelineBuffer &&optional, cons auto pool = next_pad_downstream().element().get_buffer_pool(); assert(pool); - auto buffer = pool->get_available_buffer(std::move(optional), m_timeout); - if (HAILO_SHUTDOWN_EVENT_SIGNALED == buffer.status()) { - return make_unexpected(buffer.status()); - } - CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto buffer, + pool->get_available_buffer(std::move(optional), m_timeout), + "{} (D2H) failed.", name()); while (true) { if (!m_stream->is_scheduled()) { @@ -388,7 +380,8 @@ Expected HwReadElement::run_pull(PipelineBuffer &&optional, cons } } - MemoryView buffer_view(buffer.value().as_view()); + TRY(MemoryView buffer_view, buffer.as_view(BufferProtection::NONE)); + m_duration_collector.start_measurement(); auto status = m_stream->read(buffer_view); if (HAILO_INVALID_FRAME == status) { @@ -406,7 +399,7 @@ Expected HwReadElement::run_pull(PipelineBuffer &&optional, cons CHECK_SUCCESS_AS_EXPECTED(status, "{} (D2H) failed with status={}", name(), status); m_duration_collector.complete_measurement(); - return buffer.release(); + return buffer; } } diff --git a/hailort/libhailort/src/net_flow/pipeline/edge_elements.hpp b/hailort/libhailort/src/net_flow/pipeline/edge_elements.hpp index 090a7ca..dd0e08c 100644 --- a/hailort/libhailort/src/net_flow/pipeline/edge_elements.hpp +++ b/hailort/libhailort/src/net_flow/pipeline/edge_elements.hpp @@ -72,9 +72,9 @@ public: virtual Expected run_pull(PipelineBuffer &&optional, const PipelinePad &source) override; virtual hailo_status execute_activate() override; - virtual hailo_status enqueue_execution_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done) override; + virtual hailo_status enqueue_execution_buffer(PipelineBuffer &&pipeline_buffer) override; - virtual Expected can_push_buffer_upstream() override; + virtual Expected can_push_buffer_upstream(uint32_t frames_count) override; virtual hailo_status execute_post_deactivate(bool /*should_clear_abort*/) override { return HAILO_SUCCESS; }; virtual hailo_status execute_deactivate() override { return HAILO_SUCCESS; }; diff --git a/hailort/libhailort/src/net_flow/pipeline/filter_elements.cpp b/hailort/libhailort/src/net_flow/pipeline/filter_elements.cpp index 03cdc47..4c761c9 100644 --- a/hailort/libhailort/src/net_flow/pipeline/filter_elements.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/filter_elements.cpp @@ -23,13 +23,10 @@ FilterElement::FilterElement(const std::string &name, DurationCollector &&durati hailo_status FilterElement::run_push(PipelineBuffer &&buffer, const PipelinePad &/*sink*/) { - auto output = action(std::move(buffer), PipelineBuffer()); - if (HAILO_SHUTDOWN_EVENT_SIGNALED == output.status()) { - return output.status(); - } - CHECK_EXPECTED_AS_STATUS(output); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto output, + action(std::move(buffer), PipelineBuffer())); - hailo_status status = next_pad().run_push(output.release()); + hailo_status status = next_pad().run_push(std::move(output)); if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) { LOGGER__INFO("run_push of {} was shutdown!", name()); return status; @@ -71,13 +68,9 @@ void FilterElement::run_push_async(PipelineBuffer &&buffer, const PipelinePad &/ Expected FilterElement::run_pull(PipelineBuffer &&optional, const PipelinePad &/*source*/) { - auto buffer = next_pad().run_pull(); - if (HAILO_SHUTDOWN_EVENT_SIGNALED == buffer.status()) { - LOGGER__INFO("run_pull in FilterElement was shutdown!"); - return make_unexpected(buffer.status()); - } - CHECK_EXPECTED(buffer); - return action(buffer.release(), std::move(optional)); + TRY_WITH_ACCEPTABLE_STATUS(HAILO_SHUTDOWN_EVENT_SIGNALED, auto buffer, + next_pad().run_pull()); + return action(std::move(buffer), std::move(optional)); } PipelinePad &FilterElement::next_pad_downstream() @@ -95,15 +88,13 @@ Expected> PreInferElement::create(const hailo_3 const std::string &name, std::chrono::milliseconds timeout, hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr> pipeline_status, PipelineDirection pipeline_direction, std::shared_ptr async_pipeline) { - auto transform_context = InputTransformContext::create(src_image_shape, src_format, dst_image_shape, dst_format, - dst_quant_infos); - CHECK_EXPECTED(transform_context, "Failed Creating InputTransformContext"); - - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); + TRY(auto transform_context, + InputTransformContext::create(src_image_shape, src_format, dst_image_shape, dst_format, + dst_quant_infos), "Failed Creating InputTransformContext"); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); - auto pre_infer_elem_ptr = make_shared_nothrow(transform_context.release(), - name, timeout, duration_collector.release(), std::move(pipeline_status), pipeline_direction, + auto pre_infer_elem_ptr = make_shared_nothrow(std::move(transform_context), + name, timeout, std::move(duration_collector), std::move(pipeline_status), pipeline_direction, async_pipeline); CHECK_AS_EXPECTED(nullptr != pre_infer_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); @@ -176,11 +167,13 @@ Expected PreInferElement::action(PipelineBuffer &&input, Pipelin } CHECK_AS_EXPECTED(HAILO_TIMEOUT != transformed_buffer.status(), HAILO_TIMEOUT, "{} (H2D) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_timeout.count()); - CHECK_EXPECTED(transformed_buffer); + CHECK_EXPECTED(transformed_buffer); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here + + TRY(auto dst, transformed_buffer->as_view(BufferProtection::WRITE)); + TRY(auto src, input.as_view(BufferProtection::READ)); - auto dst = transformed_buffer->as_view(); m_duration_collector.start_measurement(); - const auto status = m_transform_context->transform(input.as_view(), dst); + const auto status = m_transform_context->transform(src, dst); m_duration_collector.complete_measurement(); input.set_action_status(status); @@ -191,7 +184,7 @@ Expected PreInferElement::action(PipelineBuffer &&input, Pipelin CHECK_SUCCESS_AS_EXPECTED(status); // Note: The latency to be measured starts as the input buffer is sent to the InputVStream (via write()) - transformed_buffer->set_metadata(std::move(metadata)); + transformed_buffer->set_metadata_start_time(metadata.get_start_time()); return transformed_buffer.release(); } @@ -201,11 +194,10 @@ Expected> ConvertNmsToDetectionsE std::shared_ptr> pipeline_status, std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr async_pipeline) { - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); auto convert_nms_to_detections_elem_ptr = make_shared_nothrow(std::move(nms_info), - name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); + name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); CHECK_AS_EXPECTED(nullptr != convert_nms_to_detections_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", convert_nms_to_detections_elem_ptr->description()); @@ -257,9 +249,10 @@ Expected ConvertNmsToDetectionsElement::action(PipelineBuffer && if (!buffer) { input.set_action_status(buffer.status()); } - CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); + CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here - buffer->set_metadata(input.get_metadata()); + buffer->set_metadata_start_time(input.get_metadata().get_start_time()); + buffer->set_additional_data(input.get_metadata().get_additional_data()); m_duration_collector.start_measurement(); @@ -277,11 +270,10 @@ Expected> FillNmsFormatElement::create(con const std::string &name, hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr> pipeline_status, std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr async_pipeline) { - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); auto fill_nms_format_element = make_shared_nothrow(std::move(nms_config), - name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); + name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); CHECK_AS_EXPECTED(nullptr != fill_nms_format_element, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", fill_nms_format_element->description()); @@ -332,15 +324,17 @@ Expected FillNmsFormatElement::action(PipelineBuffer &&input, Pi if (!buffer_expected) { input.set_action_status(buffer_expected.status()); } - CHECK_EXPECTED(buffer_expected, "{} (D2H) failed with status={}", name(), buffer_expected.status()); + CHECK_EXPECTED(buffer_expected, + "{} (D2H) failed with status={}", name(),buffer_expected.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here auto buffer = buffer_expected.release(); - buffer.set_metadata(input.get_metadata()); + buffer.set_metadata_start_time(input.get_metadata().get_start_time()); + buffer.set_additional_data(input.get_metadata().get_additional_data()); m_duration_collector.start_measurement(); auto detections = input.get_metadata().get_additional_data(); - auto dst = buffer.as_view(); + TRY(auto dst, buffer.as_view(BufferProtection::WRITE)); net_flow::NmsPostProcessOp::fill_nms_format_buffer(dst, detections->m_detections, detections->m_detections_classes_count, m_nms_config); @@ -355,15 +349,12 @@ Expected> PostInferElement::create(const hailo hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr> pipeline_status, std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr async_pipeline) { - auto transform_context = OutputTransformContext::create(src_image_shape, src_format, dst_image_shape, dst_format, - dst_quant_infos, nms_info); - CHECK_EXPECTED(transform_context, "Failed Creating OutputTransformContext"); + TRY(auto transform_context, OutputTransformContext::create(src_image_shape, src_format, dst_image_shape, dst_format, + dst_quant_infos, nms_info), "Failed creating OutputTransformContext"); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); - - auto post_infer_elem_ptr = make_shared_nothrow(transform_context.release(), name, - duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); + auto post_infer_elem_ptr = make_shared_nothrow(std::move(transform_context), name, + std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); CHECK_AS_EXPECTED(nullptr != post_infer_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", post_infer_elem_ptr->description()); @@ -443,14 +434,16 @@ Expected PostInferElement::action(PipelineBuffer &&input, Pipeli if (!buffer) { input.set_action_status(buffer.status()); } - CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); + CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here // Note: The latency to be measured starts as the buffer is read from the HW (it's 'input' in this case) - buffer->set_metadata(input.get_metadata()); + buffer->set_metadata_start_time(input.get_metadata().get_start_time()); + + TRY(auto src, input.as_view(BufferProtection::READ)); + TRY(auto dst, buffer->as_view(BufferProtection::WRITE)); - auto dst = buffer->as_view(); m_duration_collector.start_measurement(); - const auto status = m_transform_context->transform(input.as_view(), dst); + const auto status = m_transform_context->transform(src, dst); m_duration_collector.complete_measurement(); input.set_action_status(status); @@ -466,11 +459,10 @@ Expected> RemoveOverlappingBboxe std::shared_ptr> pipeline_status, std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr async_pipeline) { - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); auto convert_nms_removed_overlapping_elem_ptr = make_shared_nothrow(std::move(nms_config), - name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); + name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); CHECK_AS_EXPECTED(nullptr != convert_nms_removed_overlapping_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", convert_nms_removed_overlapping_elem_ptr->description()); @@ -530,9 +522,10 @@ Expected RemoveOverlappingBboxesElement::action(PipelineBuffer & if (!buffer) { input.set_action_status(buffer.status()); } - CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); + CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here - buffer->set_metadata(input.get_metadata()); + buffer->set_metadata_start_time(input.get_metadata().get_start_time()); + buffer->set_additional_data(input.get_metadata().get_additional_data()); m_duration_collector.start_measurement(); auto detections_pipeline_data = input.get_metadata().get_additional_data(); @@ -548,10 +541,9 @@ Expected> ArgmaxPostProcessElement::cr const std::string &name, hailo_pipeline_elem_stats_flags_t elem_flags, std::shared_ptr> pipeline_status, std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr async_pipeline) { - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); auto argmax_elem_ptr = make_shared_nothrow(argmax_op, - name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); + name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); CHECK_AS_EXPECTED(nullptr != argmax_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", argmax_elem_ptr->description()); return argmax_elem_ptr; @@ -617,14 +609,18 @@ Expected ArgmaxPostProcessElement::action(PipelineBuffer &&input if (!buffer) { input.set_action_status(buffer.status()); } - CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); + CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here std::map inputs; std::map outputs; auto &input_name = m_argmax_op->inputs_metadata().begin()->first; auto &output_name = m_argmax_op->outputs_metadata().begin()->first; - inputs.insert({input_name, input.as_view()}); - outputs.insert({output_name, buffer->as_view()}); + + TRY(auto src, input.as_view(BufferProtection::READ)); + TRY(auto dst, buffer->as_view(BufferProtection::WRITE)); + + inputs.insert({input_name, src}); + outputs.insert({output_name, dst}); m_duration_collector.start_measurement(); auto post_process_result = m_argmax_op->execute(inputs, outputs); m_duration_collector.complete_measurement(); @@ -642,10 +638,9 @@ Expected> SoftmaxPostProcessElement:: std::shared_ptr> pipeline_status, std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr async_pipeline) { - auto duration_collector = DurationCollector::create(elem_flags); - CHECK_EXPECTED(duration_collector); + TRY(auto duration_collector, DurationCollector::create(elem_flags)); auto softmax_elem_ptr = make_shared_nothrow(softmax_op, - name, duration_collector.release(), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); + name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); CHECK_AS_EXPECTED(nullptr != softmax_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", softmax_elem_ptr->description()); return softmax_elem_ptr; @@ -709,14 +704,18 @@ Expected SoftmaxPostProcessElement::action(PipelineBuffer &&inpu if (!buffer) { input.set_action_status(buffer.status()); } - CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); + CHECK_EXPECTED(buffer, "{} (D2H) failed with status={}", name(), buffer.status()); // TODO (HRT-13278): Figure out how to remove CHECK_EXPECTED here std::map inputs; std::map outputs; auto &input_name = m_softmax_op->inputs_metadata().begin()->first; auto &output_name = m_softmax_op->outputs_metadata().begin()->first; - inputs.insert({input_name, input.as_view()}); - outputs.insert({output_name, buffer->as_view()}); + + TRY(auto src, input.as_view(BufferProtection::READ)); + TRY(auto dst, buffer->as_view(BufferProtection::WRITE)); + + inputs.insert({input_name, src}); + outputs.insert({output_name, dst}); m_duration_collector.start_measurement(); auto post_process_result = m_softmax_op->execute(inputs, outputs); m_duration_collector.complete_measurement(); @@ -733,9 +732,8 @@ Expected> CopyBufferElement::create(const std std::shared_ptr> pipeline_status, std::chrono::milliseconds timeout, PipelineDirection pipeline_direction, std::shared_ptr async_pipeline) { - auto duration_collector = DurationCollector::create(HAILO_PIPELINE_ELEM_STATS_NONE); - CHECK_EXPECTED(duration_collector); - auto elem_ptr = make_shared_nothrow(name, duration_collector.release(), std::move(pipeline_status), + TRY(auto duration_collector, DurationCollector::create(HAILO_PIPELINE_ELEM_STATS_NONE)); + auto elem_ptr = make_shared_nothrow(name, std::move(duration_collector), std::move(pipeline_status), timeout, pipeline_direction, async_pipeline); CHECK_AS_EXPECTED(nullptr != elem_ptr, HAILO_OUT_OF_HOST_MEMORY); diff --git a/hailort/libhailort/src/net_flow/pipeline/infer_model.cpp b/hailort/libhailort/src/net_flow/pipeline/infer_model.cpp index fa19401..65f46bd 100644 --- a/hailort/libhailort/src/net_flow/pipeline/infer_model.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/infer_model.cpp @@ -7,13 +7,10 @@ * @brief Implemention of the async HL infer **/ -#include - #include "common/utils.hpp" #include "hailo/hailort_common.hpp" #include "hailo/vdevice.hpp" #include "hailo/infer_model.hpp" -#include "vdevice/vdevice_internal.hpp" #include "hef/hef_internal.hpp" #include "net_flow/pipeline/infer_model_internal.hpp" #include "net_flow/pipeline/async_infer_runner.hpp" @@ -24,27 +21,27 @@ namespace hailort { -std::string InferModel::InferStream::Impl::name() const +std::string InferModelBase::InferStream::Impl::name() const { return m_vstream_info.name; } -hailo_3d_image_shape_t InferModel::InferStream::Impl::shape() const +hailo_3d_image_shape_t InferModelBase::InferStream::Impl::shape() const { return m_vstream_info.shape; } -hailo_format_t InferModel::InferStream::Impl::format() const +hailo_format_t InferModelBase::InferStream::Impl::format() const { return m_user_buffer_format; } -size_t InferModel::InferStream::Impl::get_frame_size() const +size_t InferModelBase::InferStream::Impl::get_frame_size() const { return HailoRTCommon::get_frame_size(m_vstream_info, m_user_buffer_format); } -Expected InferModel::InferStream::Impl::get_nms_shape() const +Expected InferModelBase::InferStream::Impl::get_nms_shape() const { CHECK_AS_EXPECTED(HailoRTCommon::is_nms(m_vstream_info.format.order), HAILO_INVALID_OPERATION, "Output {} is not NMS", name()); @@ -52,120 +49,184 @@ Expected InferModel::InferStream::Impl::get_nms_shape() const return res; } -std::vector InferModel::InferStream::Impl::get_quant_infos() const +std::vector InferModelBase::InferStream::Impl::get_quant_infos() const { // TODO: Support quant infos vector return {m_vstream_info.quant_info}; } -void InferModel::InferStream::Impl::set_format_type(hailo_format_type_t type) +void InferModelBase::InferStream::Impl::set_format_type(hailo_format_type_t type) { m_user_buffer_format.type = type; } -void InferModel::InferStream::Impl::set_format_order(hailo_format_order_t order) +void InferModelBase::InferStream::Impl::set_format_order(hailo_format_order_t order) { m_user_buffer_format.order = order; } -bool InferModel::InferStream::Impl::is_nms() const +bool InferModelBase::InferStream::Impl::is_nms() const { return HailoRTCommon::is_nms(m_vstream_info.format.order); } -void InferModel::InferStream::Impl::set_nms_score_threshold(float32_t threshold) +void InferModelBase::InferStream::Impl::set_nms_score_threshold(float32_t threshold) { m_nms_score_threshold = threshold; } -void InferModel::InferStream::Impl::set_nms_iou_threshold(float32_t threshold) +void InferModelBase::InferStream::Impl::set_nms_iou_threshold(float32_t threshold) { m_nms_iou_threshold = threshold; } -void InferModel::InferStream::Impl::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class) +void InferModelBase::InferStream::Impl::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class) { m_nms_max_proposals_per_class = max_proposals_per_class; m_vstream_info.nms_shape.max_bboxes_per_class = max_proposals_per_class; } -void InferModel::InferStream::Impl::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size) +void InferModelBase::InferStream::Impl::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size) { m_nms_max_accumulated_mask_size = max_accumulated_mask_size; m_vstream_info.nms_shape.max_accumulated_mask_size = max_accumulated_mask_size; } -InferModel::InferStream::InferStream(std::shared_ptr pimpl) : m_pimpl(pimpl) +float32_t InferModelBase::InferStream::Impl::nms_score_threshold() const { + return m_nms_score_threshold; } -const std::string InferModel::InferStream::name() const +float32_t InferModelBase::InferStream::Impl::nms_iou_threshold() const +{ + return m_nms_iou_threshold; +} + +uint32_t InferModelBase::InferStream::Impl::nms_max_proposals_per_class() const +{ + return m_nms_max_proposals_per_class; +} + +uint32_t InferModelBase::InferStream::Impl::nms_max_accumulated_mask_size() const +{ + return m_nms_max_accumulated_mask_size; +} + +InferModelBase::InferStream::InferStream(std::shared_ptr pimpl) : m_pimpl(pimpl) +{ +} + +const std::string InferModelBase::InferStream::name() const { return m_pimpl->name(); } -hailo_3d_image_shape_t InferModel::InferStream::shape() const +hailo_3d_image_shape_t InferModelBase::InferStream::shape() const { return m_pimpl->shape(); } -hailo_format_t InferModel::InferStream::format() const +hailo_format_t InferModelBase::InferStream::format() const { return m_pimpl->format(); } -size_t InferModel::InferStream::get_frame_size() const +size_t InferModelBase::InferStream::get_frame_size() const { return m_pimpl->get_frame_size(); } -Expected InferModel::InferStream::get_nms_shape() const +Expected InferModelBase::InferStream::get_nms_shape() const { return m_pimpl->get_nms_shape(); } -std::vector InferModel::InferStream::get_quant_infos() const +std::vector InferModelBase::InferStream::get_quant_infos() const { return m_pimpl->get_quant_infos(); } -void InferModel::InferStream::set_format_type(hailo_format_type_t type) +void InferModelBase::InferStream::set_format_type(hailo_format_type_t type) { m_pimpl->set_format_type(type); } -void InferModel::InferStream::set_format_order(hailo_format_order_t order) +void InferModelBase::InferStream::set_format_order(hailo_format_order_t order) { m_pimpl->set_format_order(order); } -bool InferModel::InferStream::is_nms() const +bool InferModelBase::InferStream::is_nms() const { return m_pimpl->is_nms(); } -void InferModel::InferStream::set_nms_score_threshold(float32_t threshold) +void InferModelBase::InferStream::set_nms_score_threshold(float32_t threshold) { m_pimpl->set_nms_score_threshold(threshold); } -void InferModel::InferStream::set_nms_iou_threshold(float32_t threshold) +void InferModelBase::InferStream::set_nms_iou_threshold(float32_t threshold) { m_pimpl->set_nms_iou_threshold(threshold); } -void InferModel::InferStream::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class) +void InferModelBase::InferStream::set_nms_max_proposals_per_class(uint32_t max_proposals_per_class) { m_pimpl->set_nms_max_proposals_per_class(max_proposals_per_class); } -void InferModel::InferStream::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size) +void InferModelBase::InferStream::set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size) { m_pimpl->set_nms_max_accumulated_mask_size(max_accumulated_mask_size); } -InferModel::InferModel(VDevice &vdevice, Hef &&hef, std::unordered_map &&inputs, - std::unordered_map &&outputs) +float32_t InferModelBase::InferStream::nms_score_threshold() const +{ + return m_pimpl->nms_score_threshold(); +} + +float32_t InferModelBase::InferStream::nms_iou_threshold() const +{ + return m_pimpl->nms_iou_threshold(); +} + +uint32_t InferModelBase::InferStream::nms_max_proposals_per_class() const +{ + return m_pimpl->nms_max_proposals_per_class(); +} + +uint32_t InferModelBase::InferStream::nms_max_accumulated_mask_size() const +{ + return m_pimpl->nms_max_accumulated_mask_size(); +} + +Expected> InferModelBase::create(VDevice &vdevice, const std::string &hef_path) +{ + TRY(auto hef, Hef::create(hef_path)); + TRY(auto inputs, create_infer_stream_inputs(hef)); + TRY(auto outputs, create_infer_stream_outputs(hef)); + + auto ptr = make_shared_nothrow(vdevice, std::move(hef), std::move(inputs), std::move(outputs)); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +Expected> InferModelBase::create(VDevice &vdevice, const MemoryView hef_buffer) +{ + TRY(auto hef, Hef::create(hef_buffer)); + TRY(auto inputs, create_infer_stream_inputs(hef)); + TRY(auto outputs, create_infer_stream_outputs(hef)); + + auto ptr = make_shared_nothrow(vdevice, std::move(hef), std::move(inputs), std::move(outputs)); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +InferModelBase::InferModelBase(VDevice &vdevice, Hef &&hef, std::unordered_map &&inputs, + std::unordered_map &&outputs) : m_vdevice(vdevice), m_hef(std::move(hef)), m_inputs(std::move(inputs)), m_outputs(std::move(outputs)), m_config_params(HailoRTDefaults::get_configure_params()) { @@ -184,7 +245,7 @@ InferModel::InferModel(VDevice &vdevice, Hef &&hef, std::unordered_map InferModel::configure() +Expected InferModelBase::configure() { auto configure_params = m_vdevice.get().create_configure_params(m_hef); CHECK_EXPECTED(configure_params); @@ -249,6 +310,8 @@ Expected InferModel::configure() std::unordered_map inputs_formats; std::unordered_map outputs_formats; + std::unordered_map inputs_frame_sizes; + std::unordered_map outputs_frame_sizes; auto input_vstream_infos = network_groups.value()[0]->get_input_vstream_infos(); CHECK_EXPECTED(input_vstream_infos); @@ -256,6 +319,7 @@ Expected InferModel::configure() for (const auto &vstream_info : input_vstream_infos.value()) { assert(contains(m_inputs, std::string(vstream_info.name))); inputs_formats[vstream_info.name] = m_inputs.at(vstream_info.name).format(); + inputs_frame_sizes[vstream_info.name] = m_inputs.at(vstream_info.name).get_frame_size(); } auto output_vstream_infos = network_groups.value()[0]->get_output_vstream_infos(); @@ -264,6 +328,7 @@ Expected InferModel::configure() for (const auto &vstream_info : output_vstream_infos.value()) { assert(contains(m_outputs, std::string(vstream_info.name))); outputs_formats[vstream_info.name] = m_outputs.at(vstream_info.name).format(); + outputs_frame_sizes[vstream_info.name] = m_outputs.at(vstream_info.name).get_frame_size(); } CHECK_AS_EXPECTED(std::all_of(m_inputs.begin(), m_inputs.end(), [](const auto &input_pair) { @@ -300,7 +365,7 @@ Expected InferModel::configure() } auto configured_infer_model_pimpl = ConfiguredInferModelImpl::create(network_groups.value()[0], inputs_formats, outputs_formats, - get_input_names(), get_output_names(), m_vdevice); + get_input_names(), get_output_names(), m_vdevice, inputs_frame_sizes, outputs_frame_sizes); CHECK_EXPECTED(configured_infer_model_pimpl); // The hef buffer is being used only when working with the service. @@ -311,8 +376,9 @@ Expected InferModel::configure() return ConfiguredInferModel(configured_infer_model_pimpl.release()); } -Expected InferModel::configure_for_ut(std::shared_ptr async_infer_runner, +Expected InferModelBase::configure_for_ut(std::shared_ptr async_infer_runner, const std::vector &input_names, const std::vector &output_names, + const std::unordered_map inputs_frame_sizes, const std::unordered_map outputs_frame_sizes, std::shared_ptr net_group) { if (nullptr == net_group) { @@ -337,62 +403,108 @@ Expected InferModel::configure_for_ut(std::shared_ptr InferModel::input() +Expected InferModelBase::input() { CHECK_AS_EXPECTED(1 == m_inputs.size(), HAILO_INVALID_OPERATION, "Model has more than one input!"); auto copy = m_inputs.begin()->second; return copy; } -Expected InferModel::output() +Expected InferModelBase::output() { CHECK_AS_EXPECTED(1 == m_outputs.size(), HAILO_INVALID_OPERATION, "Model has more than one output!"); auto copy = m_outputs.begin()->second; return copy; } -Expected InferModel::input(const std::string &name) +Expected InferModelBase::input(const std::string &name) { CHECK_AS_EXPECTED(contains(m_inputs, name), HAILO_NOT_FOUND, "Input {} not found!", name); auto copy = m_inputs.at(name); return copy; } -Expected InferModel::output(const std::string &name) +Expected InferModelBase::output(const std::string &name) { CHECK_AS_EXPECTED(contains(m_outputs, name), HAILO_NOT_FOUND, "Output {}, not found!", name); auto copy = m_outputs.at(name); return copy; } -const std::vector &InferModel::inputs() const +const std::vector &InferModelBase::inputs() const { return m_inputs_vector; } -const std::vector &InferModel::outputs() const +const std::vector &InferModelBase::outputs() const { return m_outputs_vector; } -const std::vector &InferModel::get_input_names() const +const std::vector &InferModelBase::get_input_names() const { return m_input_names; } -const std::vector &InferModel::get_output_names() const +const std::vector &InferModelBase::get_output_names() const { return m_output_names; } -ConfiguredInferModel::ConfiguredInferModel(std::shared_ptr pimpl) : m_pimpl(pimpl) +Expected> InferModelBase::create_infer_stream_inputs(Hef &hef) { + auto input_vstream_infos = hef.get_input_vstream_infos(); + CHECK_EXPECTED(input_vstream_infos); + + std::unordered_map inputs; + for (const auto &vstream_info : input_vstream_infos.value()) { + auto pimpl = make_shared_nothrow(vstream_info); + CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); + + InferModel::InferStream stream(pimpl); + inputs.emplace(vstream_info.name, std::move(stream)); + } + + return inputs; +} + +Expected> InferModelBase::create_infer_stream_outputs(Hef &hef) +{ + auto output_vstream_infos = hef.get_output_vstream_infos(); + CHECK_EXPECTED(output_vstream_infos); + + std::unordered_map outputs; + for (const auto &vstream_info : output_vstream_infos.value()) { + auto pimpl = make_shared_nothrow(vstream_info); + CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); + + InferModel::InferStream stream(pimpl); + outputs.emplace(vstream_info.name, std::move(stream)); + } + + return outputs; +} + +ConfiguredInferModel::ConfiguredInferModel(std::shared_ptr pimpl) : m_pimpl(pimpl) +{ +} + +ConfiguredInferModelBase::ConfiguredInferModelBase(const std::unordered_map inputs_frame_sizes, + const std::unordered_map outputs_frame_sizes) : + m_inputs_frame_sizes(inputs_frame_sizes), m_outputs_frame_sizes(outputs_frame_sizes) +{ +} + +ConfiguredInferModel ConfiguredInferModelBase::create(std::shared_ptr base) +{ + return ConfiguredInferModel(base); } Expected ConfiguredInferModel::create_bindings() @@ -400,9 +512,9 @@ Expected ConfiguredInferModel::create_bindings() return m_pimpl->create_bindings(); } -hailo_status ConfiguredInferModel::wait_for_async_ready(std::chrono::milliseconds timeout) +hailo_status ConfiguredInferModel::wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count) { - return m_pimpl->wait_for_async_ready(timeout); + return m_pimpl->wait_for_async_ready(timeout, frames_count); } hailo_status ConfiguredInferModel::activate() @@ -410,9 +522,9 @@ hailo_status ConfiguredInferModel::activate() return m_pimpl->activate(); } -void ConfiguredInferModel::deactivate() +hailo_status ConfiguredInferModel::deactivate() { - m_pimpl->deactivate(); + return m_pimpl->deactivate(); } hailo_status ConfiguredInferModel::run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout) @@ -423,7 +535,13 @@ hailo_status ConfiguredInferModel::run(ConfiguredInferModel::Bindings bindings, Expected ConfiguredInferModel::run_async(ConfiguredInferModel::Bindings bindings, std::function callback) { - return m_pimpl->run_async(bindings, callback); + auto async_infer_job = m_pimpl->run_async(bindings, callback); + if (HAILO_SUCCESS != async_infer_job.status()) { + shutdown(); + return make_unexpected(async_infer_job.status()); + } + + return async_infer_job.release(); } Expected ConfiguredInferModel::get_hw_latency_measurement() @@ -451,15 +569,91 @@ Expected ConfiguredInferModel::get_async_queue_size() return m_pimpl->get_async_queue_size(); } -void ConfiguredInferModel::shutdown() +hailo_status ConfiguredInferModel::shutdown() +{ + return m_pimpl->shutdown(); +} + +Expected ConfiguredInferModel::run_async(const std::vector &bindings, + std::function callback) +{ + auto job_pimpl = make_shared_nothrow(static_cast(bindings.size())); + if (nullptr == job_pimpl) { + shutdown(); + return make_unexpected(HAILO_OUT_OF_HOST_MEMORY); + } + + auto transfer_done = [bindings, job_pimpl, callback](const AsyncInferCompletionInfo &completion_info) { + bool should_call_callback = ConfiguredInferModelBase::get_stream_done(completion_info.status, job_pimpl); + if (should_call_callback) { + AsyncInferCompletionInfo final_completion_info(ConfiguredInferModelBase::get_completion_status(job_pimpl)); + callback(final_completion_info); + ConfiguredInferModelBase::mark_callback_done(job_pimpl); + } + }; + + for (const auto &binding : bindings) { + TRY(auto partial_job, run_async(binding, transfer_done)); + partial_job.detach(); + } + + return AsyncInferJobImpl::create(job_pimpl); +} + +Expected ConfiguredInferModelBase::create_bindings( + std::unordered_map &&inputs, + std::unordered_map &&outputs) +{ + return ConfiguredInferModel::Bindings(std::move(inputs), std::move(outputs)); +} + +Expected ConfiguredInferModelBase::create_infer_stream( + const hailo_vstream_info_t &vstream_info) +{ + auto pimpl = make_shared_nothrow(vstream_info); + CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); + + ConfiguredInferModel::Bindings::InferStream stream(pimpl); + return stream; +} + +BufferType ConfiguredInferModelBase::get_infer_stream_buffer_type(ConfiguredInferModel::Bindings::InferStream stream) +{ + return stream.m_pimpl->get_type(); +} + +bool ConfiguredInferModelBase::get_stream_done(hailo_status status, std::shared_ptr job_pimpl) +{ + return job_pimpl->stream_done(status); +} + +hailo_status ConfiguredInferModelBase::get_completion_status(std::shared_ptr job_pimpl) +{ + return job_pimpl->completion_status(); +} + +void ConfiguredInferModelBase::mark_callback_done(std::shared_ptr job_pimpl) +{ + job_pimpl->mark_callback_done(); +} + +hailo_status ConfiguredInferModelBase::run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout) { - m_pimpl->abort(); + auto job = run_async(bindings, [] (const AsyncInferCompletionInfo &) {}); + CHECK_EXPECTED_AS_STATUS(job); + + auto status = job->wait(timeout); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; } Expected> ConfiguredInferModelImpl::create(std::shared_ptr net_group, const std::unordered_map &inputs_formats, const std::unordered_map &outputs_formats, - const std::vector &input_names, const std::vector &output_names, VDevice &vdevice, const uint32_t timeout) + const std::vector &input_names, const std::vector &output_names, VDevice &vdevice, + const std::unordered_map inputs_frame_sizes, const std::unordered_map outputs_frame_sizes, + const uint32_t timeout) { auto async_infer_runner = AsyncInferRunnerImpl::create(net_group, inputs_formats, outputs_formats, timeout); CHECK_EXPECTED(async_infer_runner); @@ -477,33 +671,34 @@ Expected> ConfiguredInferModelImpl::cr } auto configured_infer_model_pimpl = make_shared_nothrow(net_group, async_infer_runner.release(), - input_names, output_names); + input_names, output_names, inputs_frame_sizes, outputs_frame_sizes); CHECK_NOT_NULL_AS_EXPECTED(configured_infer_model_pimpl, HAILO_OUT_OF_HOST_MEMORY); return configured_infer_model_pimpl; } Expected> ConfiguredInferModelImpl::create_for_ut(std::shared_ptr net_group, - std::shared_ptr async_infer_runner, const std::vector &input_names, const std::vector &output_names) + std::shared_ptr async_infer_runner, const std::vector &input_names, const std::vector &output_names, + const std::unordered_map inputs_frame_sizes, const std::unordered_map outputs_frame_sizes) { auto configured_infer_model_pimpl = make_shared_nothrow(net_group, async_infer_runner, - input_names, output_names); + input_names, output_names, inputs_frame_sizes, outputs_frame_sizes); CHECK_NOT_NULL_AS_EXPECTED(configured_infer_model_pimpl, HAILO_OUT_OF_HOST_MEMORY); return configured_infer_model_pimpl; } ConfiguredInferModelImpl::ConfiguredInferModelImpl(std::shared_ptr cng, - std::shared_ptr async_infer_runner, - const std::vector &input_names, - const std::vector &output_names) : m_cng(cng), m_async_infer_runner(async_infer_runner), - m_ongoing_parallel_transfers(0), m_input_names(input_names), m_output_names(output_names) + std::shared_ptr async_infer_runner, const std::vector &input_names, const std::vector &output_names, + const std::unordered_map inputs_frame_sizes, const std::unordered_map outputs_frame_sizes) : + ConfiguredInferModelBase(inputs_frame_sizes, outputs_frame_sizes), + m_cng(cng), m_async_infer_runner(async_infer_runner), m_ongoing_parallel_transfers(0), m_input_names(input_names), m_output_names(output_names) { } ConfiguredInferModelImpl::~ConfiguredInferModelImpl() { - abort(); + shutdown(); } Expected ConfiguredInferModelImpl::create_bindings() @@ -511,37 +706,35 @@ Expected ConfiguredInferModelImpl::create_bindin std::unordered_map inputs; std::unordered_map outputs; - auto input_vstream_infos = m_cng->get_input_vstream_infos(); + auto cng = m_cng.lock(); + CHECK_NOT_NULL_AS_EXPECTED(cng, HAILO_INTERNAL_FAILURE); + + auto input_vstream_infos = cng->get_input_vstream_infos(); CHECK_EXPECTED(input_vstream_infos); for (const auto &vstream_info : input_vstream_infos.value()) { - auto pimpl = make_shared_nothrow(vstream_info); - CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); - - ConfiguredInferModel::Bindings::InferStream stream(pimpl); + TRY(auto stream, ConfiguredInferModelBase::create_infer_stream(vstream_info)); inputs.emplace(vstream_info.name, std::move(stream)); } - auto output_vstream_infos = m_cng->get_output_vstream_infos(); + auto output_vstream_infos = cng->get_output_vstream_infos(); CHECK_EXPECTED(output_vstream_infos); for (const auto &vstream_info : output_vstream_infos.value()) { - auto pimpl = make_shared_nothrow(vstream_info); - CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); - - ConfiguredInferModel::Bindings::InferStream stream(pimpl); + TRY(auto stream, ConfiguredInferModelBase::create_infer_stream(vstream_info)); outputs.emplace(vstream_info.name, std::move(stream)); } - return ConfiguredInferModel::Bindings(std::move(inputs), std::move(outputs)); + TRY(auto bindings, ConfiguredInferModelBase::create_bindings(std::move(inputs), std::move(outputs))); + return bindings; } -hailo_status ConfiguredInferModelImpl::wait_for_async_ready(std::chrono::milliseconds timeout) +hailo_status ConfiguredInferModelImpl::wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count) { std::unique_lock lock(m_mutex); hailo_status status = HAILO_SUCCESS; - bool was_successful = m_cv.wait_for(lock, timeout, [this, &status] () -> bool { - auto pools_are_ready = m_async_infer_runner->can_push_buffers(); + bool was_successful = m_cv.wait_for(lock, timeout, [this, frames_count, &status] () -> bool { + auto pools_are_ready = m_async_infer_runner->can_push_buffers(frames_count); if (HAILO_SUCCESS != pools_are_ready.status()) { status = pools_are_ready.status(); return true; @@ -555,36 +748,32 @@ hailo_status ConfiguredInferModelImpl::wait_for_async_ready(std::chrono::millise return HAILO_SUCCESS; } -void ConfiguredInferModelImpl::abort() +hailo_status ConfiguredInferModelImpl::shutdown() { m_async_infer_runner->abort(); std::unique_lock lock(m_mutex); m_cv.wait_for(lock, WAIT_FOR_ASYNC_IN_DTOR_TIMEOUT, [this] () -> bool { return m_ongoing_parallel_transfers == 0; }); + + return deactivate(); } hailo_status ConfiguredInferModelImpl::activate() { - auto activated_ng = m_cng->activate(); + auto cng = m_cng.lock(); + CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE); + + auto activated_ng = cng->activate(); CHECK_EXPECTED_AS_STATUS(activated_ng); m_ang = activated_ng.release(); return HAILO_SUCCESS; } -void ConfiguredInferModelImpl::deactivate() +hailo_status ConfiguredInferModelImpl::deactivate() { m_ang = nullptr; -} - -hailo_status ConfiguredInferModelImpl::run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout) -{ - auto job = run_async(bindings, [] (const AsyncInferCompletionInfo &) {}); - CHECK_EXPECTED_AS_STATUS(job); - - auto status = job->wait(timeout); - CHECK_SUCCESS(status); return HAILO_SUCCESS; } @@ -592,21 +781,35 @@ hailo_status ConfiguredInferModelImpl::run(ConfiguredInferModel::Bindings bindin hailo_status ConfiguredInferModelImpl::validate_bindings(ConfiguredInferModel::Bindings bindings) { for (const auto &input_name : m_input_names) { - auto buffer_type = bindings.input(input_name)->m_pimpl->get_type(); + TRY(auto input, bindings.input(input_name)); + auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(input); switch (buffer_type) { case BufferType::VIEW: { - CHECK_EXPECTED_AS_STATUS(bindings.input(input_name)->get_buffer()); + auto buffer = input.get_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + CHECK(buffer->size() == m_inputs_frame_sizes.at(input_name), HAILO_INVALID_OPERATION, + "Input buffer size {} is different than expected {} for input '{}'", buffer->size(), m_inputs_frame_sizes.at(input_name), input_name); break; } case BufferType::PIX_BUFFER: { - CHECK_EXPECTED_AS_STATUS(bindings.input(input_name)->get_pix_buffer()); + auto buffer = input.get_pix_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + size_t buffer_size = 0; + for (size_t i = 0 ; i < buffer->number_of_planes ; i++) { + buffer_size += buffer->planes[i].bytes_used; + } + CHECK(buffer_size == m_inputs_frame_sizes.at(input_name), HAILO_INVALID_OPERATION, + "Input buffer size {} is different than expected {} for input '{}'", buffer_size, m_inputs_frame_sizes.at(input_name), input_name); break; } case BufferType::DMA_BUFFER: { - CHECK_EXPECTED_AS_STATUS(bindings.input(input_name)->get_dma_buffer()); + auto buffer = input.get_dma_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + CHECK(buffer->size == m_inputs_frame_sizes.at(input_name), HAILO_INVALID_OPERATION, + "Input buffer size {} is different than expected {} for input '{}'", buffer->size, m_inputs_frame_sizes.at(input_name), input_name); break; } default: @@ -614,11 +817,15 @@ hailo_status ConfiguredInferModelImpl::validate_bindings(ConfiguredInferModel::B } } for (const auto &output_name : m_output_names) { - auto buffer_type = bindings.output(output_name)->m_pimpl->get_type(); + TRY(auto output, bindings.output(output_name)); + auto buffer_type = ConfiguredInferModelBase::get_infer_stream_buffer_type(output); switch (buffer_type) { case BufferType::VIEW: { - CHECK_EXPECTED_AS_STATUS(bindings.output(output_name)->get_buffer()); + auto buffer = output.get_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + CHECK(buffer->size() == m_outputs_frame_sizes.at(output_name), HAILO_INVALID_OPERATION, + "Output buffer size {} is different than expected {} for output '{}'", buffer->size(), m_outputs_frame_sizes.at(output_name), output_name); break; } case BufferType::PIX_BUFFER: @@ -628,7 +835,10 @@ hailo_status ConfiguredInferModelImpl::validate_bindings(ConfiguredInferModel::B } case BufferType::DMA_BUFFER: { - CHECK_EXPECTED_AS_STATUS(bindings.output(output_name)->get_dma_buffer()); + auto buffer = output.get_dma_buffer(); + CHECK_EXPECTED_AS_STATUS(buffer); + CHECK(buffer->size == m_outputs_frame_sizes.at(output_name), HAILO_INVALID_OPERATION, + "Output buffer size {} is different than expected {} for out '{}'", buffer->size, m_outputs_frame_sizes.at(output_name), output_name); break; } default: @@ -644,19 +854,18 @@ Expected ConfiguredInferModelImpl::run_async(ConfiguredInferModel { CHECK_SUCCESS_AS_EXPECTED(validate_bindings(bindings)); - auto job_pimpl = make_shared_nothrow(static_cast(m_input_names.size() + m_output_names.size())); + auto job_pimpl = make_shared_nothrow(static_cast(m_input_names.size() + m_output_names.size())); CHECK_NOT_NULL_AS_EXPECTED(job_pimpl, HAILO_OUT_OF_HOST_MEMORY); TransferDoneCallbackAsyncInfer transfer_done = [this, bindings, job_pimpl, callback](hailo_status status) { - bool should_call_callback = job_pimpl->stream_done(status); + bool should_call_callback = ConfiguredInferModelBase::get_stream_done(status, job_pimpl); if (should_call_callback) { auto final_status = (m_async_infer_runner->get_pipeline_status() == HAILO_SUCCESS) ? - job_pimpl->completion_status() : m_async_infer_runner->get_pipeline_status(); + ConfiguredInferModelBase::get_completion_status(job_pimpl) : m_async_infer_runner->get_pipeline_status(); AsyncInferCompletionInfo completion_info(final_status); callback(completion_info); - job_pimpl->mark_callback_done(); - + ConfiguredInferModelBase::mark_callback_done(job_pimpl); { std::unique_lock lock(m_mutex); m_ongoing_parallel_transfers--; @@ -673,36 +882,50 @@ Expected ConfiguredInferModelImpl::run_async(ConfiguredInferModel } m_cv.notify_all(); - AsyncInferJob job(job_pimpl); - return job; + return AsyncInferJobImpl::create(job_pimpl); } Expected ConfiguredInferModelImpl::get_hw_latency_measurement() { - return m_cng->get_latency_measurement(); + auto cng = m_cng.lock(); + CHECK_NOT_NULL_AS_EXPECTED(cng, HAILO_INTERNAL_FAILURE); + + return cng->get_latency_measurement(); } hailo_status ConfiguredInferModelImpl::set_scheduler_timeout(const std::chrono::milliseconds &timeout) { - return m_cng->set_scheduler_timeout(timeout); + auto cng = m_cng.lock(); + CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE); + + return cng->set_scheduler_timeout(timeout); } hailo_status ConfiguredInferModelImpl::set_scheduler_threshold(uint32_t threshold) { - return m_cng->set_scheduler_threshold(threshold); + auto cng = m_cng.lock(); + CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE); + + return cng->set_scheduler_threshold(threshold); } hailo_status ConfiguredInferModelImpl::set_scheduler_priority(uint8_t priority) { - return m_cng->set_scheduler_priority(priority); + auto cng = m_cng.lock(); + CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE); + + return cng->set_scheduler_priority(priority); } Expected ConfiguredInferModelImpl::get_async_queue_size() { - return m_cng->get_min_buffer_pool_size(); + auto cng = m_cng.lock(); + CHECK_NOT_NULL(cng, HAILO_INTERNAL_FAILURE); + + return cng->get_min_buffer_pool_size(); } -AsyncInferJob::AsyncInferJob(std::shared_ptr pimpl) : m_pimpl(pimpl), m_should_wait_in_dtor(true) +AsyncInferJob::AsyncInferJob(std::shared_ptr pimpl) : m_pimpl(pimpl), m_should_wait_in_dtor(true) { } @@ -748,13 +971,18 @@ void AsyncInferJob::detach() m_should_wait_in_dtor = false; } -AsyncInferJob::Impl::Impl(uint32_t streams_count) : m_job_completion_status(HAILO_SUCCESS) +AsyncInferJob AsyncInferJobBase::create(std::shared_ptr base) +{ + return AsyncInferJob(base); +} + +AsyncInferJobImpl::AsyncInferJobImpl(uint32_t streams_count) : m_job_completion_status(HAILO_SUCCESS) { m_ongoing_transfers = streams_count; m_callback_called = false; } -hailo_status AsyncInferJob::Impl::wait(std::chrono::milliseconds timeout) +hailo_status AsyncInferJobImpl::wait(std::chrono::milliseconds timeout) { std::unique_lock lock(m_mutex); bool was_successful = m_cv.wait_for(lock, timeout, [this] () -> bool { @@ -765,7 +993,7 @@ hailo_status AsyncInferJob::Impl::wait(std::chrono::milliseconds timeout) return HAILO_SUCCESS; } -bool AsyncInferJob::Impl::stream_done(const hailo_status &status) +bool AsyncInferJobImpl::stream_done(const hailo_status &status) { bool should_call_callback = false; { @@ -779,12 +1007,12 @@ bool AsyncInferJob::Impl::stream_done(const hailo_status &status) return should_call_callback; } -hailo_status AsyncInferJob::Impl::completion_status() +hailo_status AsyncInferJobImpl::completion_status() { return m_job_completion_status; } -void AsyncInferJob::Impl::mark_callback_done() +void AsyncInferJobImpl::mark_callback_done() { { std::unique_lock lock(m_mutex); diff --git a/hailort/libhailort/src/net_flow/pipeline/infer_model_hrpc_client.cpp b/hailort/libhailort/src/net_flow/pipeline/infer_model_hrpc_client.cpp new file mode 100644 index 0000000..ce0032d --- /dev/null +++ b/hailort/libhailort/src/net_flow/pipeline/infer_model_hrpc_client.cpp @@ -0,0 +1,149 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file infer_model_hrpc_client.cpp + * @brief InferModel HRPC client implementation + **/ + +#include "infer_model_hrpc_client.hpp" +#include "configured_infer_model_hrpc_client.hpp" + +namespace hailort +{ + +Expected> InferModelHrpcClient::create(Hef &&hef, + std::shared_ptr client, uint32_t infer_model_handle_id, uint32_t vdevice_handle, VDevice &vdevice) +{ + TRY(auto inputs, create_infer_stream_inputs(hef)); + TRY(auto outputs, create_infer_stream_outputs(hef)); + + auto ptr = make_shared_nothrow(client, infer_model_handle_id, + vdevice_handle, vdevice, std::move(hef), std::move(inputs), std::move(outputs)); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +InferModelHrpcClient::InferModelHrpcClient(std::shared_ptr client, uint32_t handle, + uint32_t vdevice_handle, VDevice &vdevice, Hef &&hef, + std::unordered_map &&inputs, std::unordered_map &&outputs) : + InferModelBase(vdevice, std::move(hef), std::move(inputs), std::move(outputs)), + m_client(client), + m_handle(handle), + m_vdevice_handle(vdevice_handle) +{ +} + +InferModelHrpcClient::~InferModelHrpcClient() +{ + if (INVALID_HANDLE_ID == m_handle) { + return; + } + + auto request = DestroyInferModelSerializer::serialize_request(m_handle); + if (!request) { + LOGGER__CRITICAL("Failed to serialize InferModel_release request"); + return; + } + + auto client = m_client.lock(); + if (client) { + auto execute_request_result = client->execute_request(HailoRpcActionID::INFER_MODEL__DESTROY, MemoryView(*request)); + if (!execute_request_result) { + LOGGER__CRITICAL("Failed to destroy infer model! status = {}", execute_request_result.status()); + return; + } + + auto deserialize_reply_result = DestroyInferModelSerializer::deserialize_reply(MemoryView(*execute_request_result)); + if (HAILO_SUCCESS != deserialize_reply_result) { + LOGGER__CRITICAL("Failed to destroy infer model! status = {}", deserialize_reply_result); + return; + } + } +} + +Expected InferModelHrpcClient::configure() +{ + rpc_create_configured_infer_model_request_params_t request_params; + for (const auto &input : m_inputs) { + rpc_stream_params_t current_stream_params; + current_stream_params.format_order = static_cast(input.second.format().order); + current_stream_params.format_type = static_cast(input.second.format().type); + current_stream_params.nms_iou_threshold = input.second.nms_iou_threshold(); + current_stream_params.nms_score_threshold = input.second.nms_score_threshold(); + current_stream_params.nms_max_proposals_per_class = input.second.nms_max_proposals_per_class(); + current_stream_params.nms_max_accumulated_mask_size = input.second.nms_max_accumulated_mask_size(); + + request_params.input_streams_params[input.second.name()] = current_stream_params; + } + + for (const auto &output : m_outputs) { + rpc_stream_params_t current_stream_params; + current_stream_params.format_order = static_cast(output.second.format().order); + current_stream_params.format_type = static_cast(output.second.format().type); + current_stream_params.nms_iou_threshold = output.second.nms_iou_threshold(); + current_stream_params.nms_score_threshold = output.second.nms_score_threshold(); + current_stream_params.nms_max_proposals_per_class = output.second.nms_max_proposals_per_class(); + current_stream_params.nms_max_accumulated_mask_size = output.second.nms_max_accumulated_mask_size(); + + request_params.output_streams_params[output.second.name()] = current_stream_params; + } + + request_params.batch_size = m_config_params.batch_size; + request_params.power_mode = m_config_params.power_mode; + request_params.latency_flag = m_config_params.latency; + request_params.infer_model_handle = m_handle; + request_params.vdevice_handle = m_vdevice_handle; + + TRY(auto request, CreateConfiguredInferModelSerializer::serialize_request(request_params)); + auto client = m_client.lock(); + CHECK_AS_EXPECTED(nullptr != client, HAILO_INTERNAL_FAILURE, + "Lost comunication with the server. This may happen if VDevice is released while the InferModel is in use."); + TRY(auto result, client->execute_request(HailoRpcActionID::INFER_MODEL__CREATE_CONFIGURED_INFER_MODEL, + MemoryView(request))); + TRY(auto tuple, CreateConfiguredInferModelSerializer::deserialize_reply(MemoryView(result))); + CHECK_SUCCESS_AS_EXPECTED(std::get<0>(tuple)); + auto configured_infer_handle = std::get<1>(tuple); + auto async_queue_size = std::get<2>(tuple); + + std::unordered_map inputs_frame_sizes; + std::unordered_map outputs_frame_sizes; + for (const auto &input : m_inputs) { + inputs_frame_sizes.emplace(input.second.name(), input.second.get_frame_size()); + } + for (const auto &output : m_outputs) { + outputs_frame_sizes.emplace(output.second.name(), output.second.get_frame_size()); + } + + auto callbacks_queue = make_unique_nothrow(client, m_output_names); + CHECK_NOT_NULL_AS_EXPECTED(callbacks_queue, HAILO_OUT_OF_HOST_MEMORY); + + TRY(auto input_vstream_infos, m_hef.get_input_vstream_infos()); + TRY(auto output_vstream_infos, m_hef.get_output_vstream_infos()); + TRY(auto cim_client_ptr, ConfiguredInferModelHrpcClient::create(client, + configured_infer_handle, + std::move(input_vstream_infos), std::move(output_vstream_infos), + async_queue_size, std::move(callbacks_queue), m_handle, + inputs_frame_sizes, outputs_frame_sizes)); + + return ConfiguredInferModelBase::create(cim_client_ptr); +} + +Expected InferModelHrpcClient::configure_for_ut(std::shared_ptr async_infer_runner, + const std::vector &input_names, const std::vector &output_names, + const std::unordered_map inputs_frame_sizes, + const std::unordered_map outputs_frame_sizes, + std::shared_ptr net_group) +{ + (void)async_infer_runner; + (void)input_names; + (void)output_names; + (void)net_group; + (void)inputs_frame_sizes; + (void)outputs_frame_sizes; + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/net_flow/pipeline/infer_model_hrpc_client.hpp b/hailort/libhailort/src/net_flow/pipeline/infer_model_hrpc_client.hpp new file mode 100644 index 0000000..7595406 --- /dev/null +++ b/hailort/libhailort/src/net_flow/pipeline/infer_model_hrpc_client.hpp @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file infer_model_hrpc_client.hpp + * @brief Infer model HRPC client, represents the user's handle to the InferModel object + **/ + +#ifndef _HAILO_INFER_MODEL_HRPC_CLIENT_HPP_ +#define _HAILO_INFER_MODEL_HRPC_CLIENT_HPP_ + +#include "hailo/hailort.h" +#include "hailo/infer_model.hpp" +#include "hrpc/client.hpp" +#include "net_flow/pipeline/infer_model_internal.hpp" + +namespace hailort +{ + +class InferModelHrpcClient : public InferModelBase +{ +public: + static Expected> create(Hef &&hef, + std::shared_ptr client, uint32_t infer_model_handle_id, + uint32_t vdevice_handle, VDevice &vdevice); + + InferModelHrpcClient(std::shared_ptr client, uint32_t id, + uint32_t vdevice_handle, VDevice &vdevice, Hef &&hef, std::unordered_map &&inputs, + std::unordered_map &&outputs); + virtual ~InferModelHrpcClient(); + + InferModelHrpcClient(const InferModelHrpcClient &) = delete; + InferModelHrpcClient &operator=(const InferModelHrpcClient &) = delete; + InferModelHrpcClient(InferModelHrpcClient &&) = delete; + InferModelHrpcClient &operator=(InferModelHrpcClient &&) = delete; + + virtual Expected configure() override; + + virtual Expected configure_for_ut(std::shared_ptr async_infer_runner, + const std::vector &input_names, const std::vector &output_names, + const std::unordered_map inputs_frame_sizes = {}, + const std::unordered_map outputs_frame_sizes = {}, + std::shared_ptr net_group = nullptr) override; + +private: + std::weak_ptr m_client; + uint32_t m_handle; + uint32_t m_vdevice_handle; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_INFER_MODEL_HRPC_CLIENT_HPP_ */ diff --git a/hailort/libhailort/src/net_flow/pipeline/infer_model_internal.hpp b/hailort/libhailort/src/net_flow/pipeline/infer_model_internal.hpp index 7f40dea..72b22a2 100644 --- a/hailort/libhailort/src/net_flow/pipeline/infer_model_internal.hpp +++ b/hailort/libhailort/src/net_flow/pipeline/infer_model_internal.hpp @@ -5,15 +5,19 @@ /** * @file infer_model_internal.hpp * @brief Implemention of the infer model + * + * InferModel (Interface) + * |-- InferModelBase (Base implementation) + * |-- InferModelHrpcClient (RPC handle communicating with the server) **/ #ifndef _HAILO_INFER_MODEL_INTERNAL_HPP_ #define _HAILO_INFER_MODEL_INTERNAL_HPP_ #include "hailo/infer_model.hpp" -#include "hailo/vstream.hpp" #include "net_flow/pipeline/async_infer_runner.hpp" #include "net_flow/ops/nms_post_process.hpp" +#include "hrpc/client.hpp" namespace hailort { @@ -43,6 +47,52 @@ private: TransferDoneCallbackAsyncInfer m_stream_callback; }; +class InferModelBase : public InferModel +{ +public: + static Expected> create(VDevice &vdevice, const std::string &hef_path); + static Expected> create(VDevice &vdevice, const MemoryView hef_buffer); + + InferModelBase(VDevice &vdevice, Hef &&hef, std::unordered_map &&inputs, + std::unordered_map &&outputs); + virtual ~InferModelBase() = default; + InferModelBase(InferModelBase &&); + + virtual const Hef &hef() const override; + virtual void set_batch_size(uint16_t batch_size) override; + virtual void set_power_mode(hailo_power_mode_t power_mode) override; + virtual void set_hw_latency_measurement_flags(hailo_latency_measurement_flags_t latency) override; + virtual Expected configure() override; + virtual Expected input() override; + virtual Expected output() override; + virtual Expected input(const std::string &name) override; + virtual Expected output(const std::string &name) override; + virtual const std::vector &inputs() const override; + virtual const std::vector &outputs() const override; + virtual const std::vector &get_input_names() const override; + virtual const std::vector &get_output_names() const override; + + virtual Expected configure_for_ut(std::shared_ptr async_infer_runner, + const std::vector &input_names, const std::vector &output_names, + const std::unordered_map inputs_frame_sizes = {}, + const std::unordered_map outputs_frame_sizes = {}, + std::shared_ptr net_group = nullptr) override; + +protected: + static Expected> create_infer_stream_inputs(Hef &hef); + static Expected> create_infer_stream_outputs(Hef &hef); + + std::reference_wrapper m_vdevice; + Hef m_hef; + std::unordered_map m_inputs; + std::unordered_map m_outputs; + std::vector m_inputs_vector; + std::vector m_outputs_vector; + std::vector m_input_names; + std::vector m_output_names; + ConfigureNetworkParams m_config_params; +}; + class InferModel::InferStream::Impl { public: @@ -68,8 +118,15 @@ public: void set_nms_max_proposals_per_class(uint32_t max_proposals_per_class); void set_nms_max_accumulated_mask_size(uint32_t max_accumulated_mask_size); + float32_t nms_score_threshold() const; + float32_t nms_iou_threshold() const; + uint32_t nms_max_proposals_per_class() const; + uint32_t nms_max_accumulated_mask_size() const; + private: friend class InferModel; + friend class InferModelBase; + friend class VDevice; hailo_vstream_info_t m_vstream_info; hailo_format_t m_user_buffer_format; @@ -80,16 +137,27 @@ private: uint32_t m_nms_max_accumulated_mask_size; }; -class AsyncInferJob::Impl +class AsyncInferJobBase +{ +public: + static AsyncInferJob create(std::shared_ptr base); + virtual ~AsyncInferJobBase() = default; + virtual hailo_status wait(std::chrono::milliseconds timeout) = 0; +}; + +class AsyncInferJobImpl : public AsyncInferJobBase { public: - Impl(uint32_t streams_count); - hailo_status wait(std::chrono::milliseconds timeout); + AsyncInferJobImpl(uint32_t streams_count); + virtual hailo_status wait(std::chrono::milliseconds timeout) override; + +private: + friend class ConfiguredInferModelBase; + bool stream_done(const hailo_status &status); hailo_status completion_status(); void mark_callback_done(); -private: std::condition_variable m_cv; std::mutex m_mutex; std::atomic_uint32_t m_ongoing_transfers; @@ -97,40 +165,87 @@ private: hailo_status m_job_completion_status; }; -class ConfiguredInferModelImpl +/* + * ConfiguredInferModel (interface wrapper - external API) + * |-- ConfiguredInferModelBase (interface) + * |--|-- ConfiguredInferModelImpl (non-RPC implementation) + * |--|-- ConfiguredInferModelHrpcClient (RPC handle communicating with the server) + */ + +class ConfiguredInferModelBase +{ +public: + static ConfiguredInferModel create(std::shared_ptr base); + + ConfiguredInferModelBase(const std::unordered_map inputs_frame_sizes, + const std::unordered_map outputs_frame_sizes); + virtual ~ConfiguredInferModelBase() = default; + virtual Expected create_bindings() = 0; + virtual hailo_status wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count = 1) = 0; + virtual hailo_status activate() = 0; + virtual hailo_status deactivate() = 0; + virtual hailo_status run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout); + virtual Expected run_async(ConfiguredInferModel::Bindings bindings, + std::function callback = ASYNC_INFER_EMPTY_CALLBACK) = 0; + virtual Expected get_hw_latency_measurement() = 0; + virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout) = 0; + virtual hailo_status set_scheduler_threshold(uint32_t threshold) = 0; + virtual hailo_status set_scheduler_priority(uint8_t priority) = 0; + virtual Expected get_async_queue_size() = 0; + virtual hailo_status shutdown() = 0; + + static Expected create_bindings( + std::unordered_map &&inputs, + std::unordered_map &&outputs); + static Expected create_infer_stream(const hailo_vstream_info_t &vstream_info); + static BufferType get_infer_stream_buffer_type(ConfiguredInferModel::Bindings::InferStream expected_stream); + static bool get_stream_done(hailo_status status, std::shared_ptr job_pimpl); + static hailo_status get_completion_status(std::shared_ptr job_pimpl); + static void mark_callback_done(std::shared_ptr job_pimpl); + +private: + virtual hailo_status validate_bindings(ConfiguredInferModel::Bindings bindings) = 0; + +protected: + std::unordered_map m_inputs_frame_sizes; + std::unordered_map m_outputs_frame_sizes; + +}; + +class ConfiguredInferModelImpl : public ConfiguredInferModelBase { public: static Expected> create(std::shared_ptr net_group, const std::unordered_map &inputs_formats, const std::unordered_map &outputs_formats, const std::vector &input_names, const std::vector &output_names, VDevice &vdevice, + const std::unordered_map inputs_frame_sizes, const std::unordered_map outputs_frame_sizes, const uint32_t timeout = HAILO_DEFAULT_VSTREAM_TIMEOUT_MS); - ConfiguredInferModelImpl(std::shared_ptr cng, - std::shared_ptr async_infer_runner, - const std::vector &input_names, - const std::vector &output_names); + ConfiguredInferModelImpl(std::shared_ptr cng, std::shared_ptr async_infer_runner, + const std::vector &input_names, const std::vector &output_names, + const std::unordered_map inputs_frame_sizes, const std::unordered_map outputs_frame_sizes); ~ConfiguredInferModelImpl(); - Expected create_bindings(); - hailo_status wait_for_async_ready(std::chrono::milliseconds timeout); - void abort(); - hailo_status activate(); - void deactivate(); - hailo_status run(ConfiguredInferModel::Bindings bindings, std::chrono::milliseconds timeout); - Expected run_async(ConfiguredInferModel::Bindings bindings, - std::function callback); - Expected get_hw_latency_measurement(); - hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout); - hailo_status set_scheduler_threshold(uint32_t threshold); - hailo_status set_scheduler_priority(uint8_t priority); - Expected get_async_queue_size(); + virtual Expected create_bindings() override; + virtual hailo_status wait_for_async_ready(std::chrono::milliseconds timeout, uint32_t frames_count) override; + virtual hailo_status activate() override; + virtual hailo_status deactivate() override; + virtual Expected run_async(ConfiguredInferModel::Bindings bindings, + std::function callback) override; + virtual Expected get_hw_latency_measurement() override; + virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout) override; + virtual hailo_status set_scheduler_threshold(uint32_t threshold) override; + virtual hailo_status set_scheduler_priority(uint8_t priority) override; + virtual Expected get_async_queue_size() override; + virtual hailo_status shutdown() override; static Expected> create_for_ut(std::shared_ptr net_group, - std::shared_ptr async_infer_runner, const std::vector &input_names, const std::vector &output_names); + std::shared_ptr async_infer_runner, const std::vector &input_names, const std::vector &output_names, + const std::unordered_map inputs_frame_sizes, const std::unordered_map outputs_frame_sizes); private: - hailo_status validate_bindings(ConfiguredInferModel::Bindings bindings); + virtual hailo_status validate_bindings(ConfiguredInferModel::Bindings bindings); - std::shared_ptr m_cng; + std::weak_ptr m_cng; std::unique_ptr m_ang; std::shared_ptr m_async_infer_runner; uint32_t m_ongoing_parallel_transfers; diff --git a/hailort/libhailort/src/net_flow/pipeline/multi_io_elements.cpp b/hailort/libhailort/src/net_flow/pipeline/multi_io_elements.cpp index 54260ee..c4b41bf 100644 --- a/hailort/libhailort/src/net_flow/pipeline/multi_io_elements.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/multi_io_elements.cpp @@ -83,10 +83,13 @@ void BaseMuxElement::run_push_async(PipelineBuffer &&buffer, const PipelinePad & m_barrier->arrive_and_wait(); if (HAILO_SUCCESS == m_pipeline_status->load()) { - std::unique_lock lock(m_mutex); - m_input_buffers[sink.name()] = std::move(buffer); - if (m_input_buffers.size() == m_sink_name_to_index.size()) { // Last sink to set its buffer - + size_t input_buffers_size = 0; + { + std::unique_lock lock(m_mutex); + m_input_buffers[sink.name()] = std::move(buffer); + input_buffers_size = m_input_buffers.size(); + } + if (input_buffers_size == m_sink_name_to_index.size()) { // Last sink to set its buffer for (auto &input_buffer : m_input_buffers) { if (HAILO_SUCCESS != input_buffer.second.action_status()) { handle_non_recoverable_async_error(input_buffer.second.action_status()); @@ -190,7 +193,8 @@ Expected NmsPostProcessMuxElement::action(std::vector inputs; std::map outputs; for (size_t i = 0; i < input_buffers.size(); ++i) { - inputs.insert({m_sinks_names[i], input_buffers[i].as_view()}); + TRY(auto src, input_buffers[i].as_view(BufferProtection::READ)); + inputs.insert({m_sinks_names[i], src}); } auto pool = next_pad_downstream().element().get_buffer_pool(); assert(pool); @@ -206,7 +210,8 @@ Expected NmsPostProcessMuxElement::action(std::vectoras_view()}); // TODO: fill with correct name + TRY(auto dst, acquired_buffer->as_view(BufferProtection::WRITE)); + outputs.insert({"", dst}); // TODO: fill with correct name m_duration_collector.start_measurement(); auto post_process_result = m_nms_op->execute(inputs, outputs); @@ -310,7 +315,8 @@ Expected NmsMuxElement::action(std::vector &&inp input_views.reserve(inputs.size()); for (auto &input_buf : inputs) { - input_views.push_back(input_buf.as_view()); + TRY(auto src, input_buf.as_view(BufferProtection::READ)); + input_views.push_back(src); } auto pool = next_pad_downstream().element().get_buffer_pool(); assert(pool); @@ -330,7 +336,8 @@ Expected NmsMuxElement::action(std::vector &&inp CHECK_EXPECTED(acquired_buffer); m_duration_collector.start_measurement(); - const auto status = fuse_buffers(input_views, m_nms_infos, acquired_buffer.value().as_view()); + TRY(auto dst, acquired_buffer.value().as_view(BufferProtection::WRITE)); + const auto status = fuse_buffers(input_views, m_nms_infos, dst); m_duration_collector.complete_measurement(); for (auto &input : inputs) { @@ -658,11 +665,13 @@ Expected> TransformDemuxElement::action(PipelineBuff } CHECK_EXPECTED(acquired_buffer, "Failed to acquire buffer"); outputs.emplace_back(acquired_buffer.release()); - raw_buffers.push_back(outputs.back().as_view()); + TRY(auto dst, outputs.back().as_view(BufferProtection::WRITE)); + raw_buffers.push_back(dst); } m_duration_collector.start_measurement(); - const auto status = m_demuxer->transform_demux(input.as_view(), raw_buffers); + TRY(auto src, input.as_view(BufferProtection::READ)); + const auto status = m_demuxer->transform_demux(src, raw_buffers); m_duration_collector.complete_measurement(); input.set_action_status(status); @@ -721,13 +730,28 @@ Expected> PixBufferElement::action(PipelineBuffer && CHECK_NOT_NULL_AS_EXPECTED(shared_input_buff, HAILO_OUT_OF_HOST_MEMORY); for (uint32_t i = 0; i < input_pix_buffer.number_of_planes; i++) { - outputs.emplace_back(MemoryView(input_pix_buffer.planes[i].user_ptr, input_pix_buffer.planes[i].bytes_used), - [input_ptr = shared_input_buff](hailo_status status) - { + if (HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR == input_pix_buffer.memory_type) { + outputs.emplace_back(MemoryView(input_pix_buffer.planes[i].user_ptr, input_pix_buffer.planes[i].bytes_used), + [input_ptr = shared_input_buff](hailo_status status) + { + if (HAILO_SUCCESS != status) { + input_ptr->set_action_status(status); + } + }); + } else if (HAILO_PIX_BUFFER_MEMORY_TYPE_DMABUF == input_pix_buffer.memory_type) { + hailo_dma_buffer_t dma_buffer; + dma_buffer.fd = input_pix_buffer.planes[i].fd; + dma_buffer.size = input_pix_buffer.planes[i].plane_size; + TransferDoneCallbackAsyncInfer exec_done = [input_ptr = shared_input_buff](hailo_status status) { if (HAILO_SUCCESS != status) { input_ptr->set_action_status(status); - } - }); + }}; + hailo_status action_status = HAILO_SUCCESS; + BufferPoolPtr pool = m_sources[i].next()->element().get_buffer_pool(); + outputs.emplace_back(PipelineBuffer(dma_buffer, exec_done, action_status, pool->is_holding_user_buffers(), pool)); + } else { + return make_unexpected(HAILO_INVALID_ARGUMENT); + } } } @@ -854,13 +878,59 @@ void AsyncHwElement::action() m_barrier->terminate(); return; } - named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_shared->as_view(), + + BufferRepresentation buffer_representation{}; + if (buffer_shared->get_buffer_type() == BufferType::VIEW) { + auto src = buffer_shared->as_view(BufferProtection::READ); + if (HAILO_SUCCESS != src.status()) { + handle_non_recoverable_async_error(src.status()); + m_input_buffers.clear(); + m_barrier->terminate(); + return; + } + buffer_representation.buffer_type = BufferType::VIEW; + buffer_representation.view = src.value(); + } else if (buffer_shared->get_buffer_type() == BufferType::DMA_BUFFER) { + auto dma_buffer = buffer_shared->get_metadata().get_additional_data(); + buffer_representation.buffer_type = BufferType::DMA_BUFFER; + buffer_representation.dma_buffer = dma_buffer->m_dma_buffer; + } else { + handle_non_recoverable_async_error(HAILO_INVALID_ARGUMENT); + m_input_buffers.clear(); + m_barrier->terminate(); + return; + } + + named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_representation, [buffer_shared](hailo_status status) { buffer_shared->set_action_status(status); })); } for (auto &output_buffer : source_name_to_output_buffer) { const auto &stream_name = m_source_name_to_stream_name.at(output_buffer.first); - named_buffers_callbacks.emplace(stream_name, std::make_pair(output_buffer.second->as_view(), + + BufferRepresentation buffer_representation{}; + if (output_buffer.second->get_buffer_type() == BufferType::VIEW) { + auto dst = output_buffer.second->as_view(BufferProtection::WRITE); + if (HAILO_SUCCESS != dst.status()) { + handle_non_recoverable_async_error(dst.status()); + m_input_buffers.clear(); + m_barrier->terminate(); + return; + } + buffer_representation.buffer_type = BufferType::VIEW; + buffer_representation.view = dst.value(); + } else if (output_buffer.second->get_buffer_type() == BufferType::DMA_BUFFER) { + auto dma_buffer = output_buffer.second->get_metadata().get_additional_data(); + buffer_representation.buffer_type = BufferType::DMA_BUFFER; + buffer_representation.dma_buffer = dma_buffer->m_dma_buffer; + } else { + handle_non_recoverable_async_error(HAILO_INVALID_ARGUMENT); + m_input_buffers.clear(); + m_barrier->terminate(); + return; + } + + named_buffers_callbacks.emplace(stream_name, std::make_pair(buffer_representation, [this, buffer = output_buffer.second, source_name = output_buffer.first](hailo_status status){ buffer->set_action_status(status); if (HAILO_SUCCESS == m_pipeline_status->load()) { @@ -872,7 +942,16 @@ void AsyncHwElement::action() })); } - auto status = m_net_group->wait_for_ongoing_callbacks_count_under(m_max_ongoing_transfers); + auto cng = m_net_group.lock(); + if (!cng) { + // Error - cng (VDevice) was released mid inference + handle_non_recoverable_async_error(HAILO_INTERNAL_FAILURE); + m_input_buffers.clear(); + m_barrier->terminate(); + return; + } + + auto status = cng->wait_for_ongoing_callbacks_count_under(m_max_ongoing_transfers); if (HAILO_SUCCESS != status ) { handle_non_recoverable_async_error(status); m_input_buffers.clear(); @@ -880,7 +959,7 @@ void AsyncHwElement::action() return; } - status = m_net_group->infer_async(named_buffers_callbacks, [](hailo_status){}); + status = cng->infer_async(named_buffers_callbacks, [](hailo_status){}); if (HAILO_SUCCESS != status ) { handle_non_recoverable_async_error(status); m_input_buffers.clear(); @@ -929,11 +1008,12 @@ Expected AsyncHwElement::get_source_index_from_source_name(const std:: return ret_val; } -Expected AsyncHwElement::get_sink_index_from_input_stream_name(const std::string &input_stream_name) +Expected AsyncHwElement::get_sink_index_from_input_stream_name(const std::string &input_stream_name) { for (const auto &name_pair : m_sink_name_to_stream_name) { if (name_pair.second == input_stream_name) { - return Expected(m_sink_name_to_index.at(name_pair.first)); + auto cpy = static_cast(m_sink_name_to_index.at(name_pair.first)); + return cpy; } } return make_unexpected(HAILO_INVALID_ARGUMENT); @@ -962,8 +1042,14 @@ hailo_status AsyncHwElement::execute_terminate(hailo_status error_status) m_barrier->terminate(); + // Best effort - if cng is already released - nothing to shut-down + auto shutdown_status = HAILO_SUCCESS; + auto cng = m_net_group.lock(); + if (cng) { + shutdown_status = cng->shutdown(); + } + // Checking success of shutdown is best effort (terminate should be called even if shutdown fails) - auto shutdown_status = m_net_group->shutdown(); auto terminate_status = PipelineElement::execute_terminate(error_status); CHECK_SUCCESS(shutdown_status); CHECK_SUCCESS(terminate_status); diff --git a/hailort/libhailort/src/net_flow/pipeline/multi_io_elements.hpp b/hailort/libhailort/src/net_flow/pipeline/multi_io_elements.hpp index b7468f1..d10ab4a 100644 --- a/hailort/libhailort/src/net_flow/pipeline/multi_io_elements.hpp +++ b/hailort/libhailort/src/net_flow/pipeline/multi_io_elements.hpp @@ -246,7 +246,7 @@ public: virtual Expected run_pull(PipelineBuffer &&optional, const PipelinePad &source) override; Expected get_source_index_from_output_stream_name(const std::string &output_stream_name); - Expected get_sink_index_from_input_stream_name(const std::string &input_stream_name); + Expected get_sink_index_from_input_stream_name(const std::string &input_stream_name); virtual Expected get_source_index_from_source_name(const std::string &source_name) override; std::vector get_hw_interacted_buffer_pools_h2d(); @@ -260,7 +260,7 @@ private: void handle_error_in_hw_async_elem(hailo_status error_status); std::chrono::milliseconds m_timeout; - std::shared_ptr m_net_group; + std::weak_ptr m_net_group; size_t m_max_ongoing_transfers; std::unordered_map m_sink_name_to_stream_name; diff --git a/hailort/libhailort/src/net_flow/pipeline/pipeline.cpp b/hailort/libhailort/src/net_flow/pipeline/pipeline.cpp index 0dff98f..ee309a5 100644 --- a/hailort/libhailort/src/net_flow/pipeline/pipeline.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/pipeline.cpp @@ -10,6 +10,7 @@ #include "common/utils.hpp" #include "common/runtime_statistics_internal.hpp" #include "common/os_utils.hpp" +#include "utils/dma_buffer_utils.hpp" #include "hailo/expected.hpp" #include "hailo/hailort.h" @@ -43,62 +44,77 @@ void PipelineBuffer::Metadata::set_start_time(PipelineTimePoint val) PipelineBuffer::PipelineBuffer(Type type) : m_type(type), - m_pool(nullptr), m_view(), + m_exec_done([](hailo_status){}), m_metadata(), m_is_user_buffer(false), m_should_call_exec_done(true), - m_action_status(HAILO_SUCCESS) + m_action_status(HAILO_SUCCESS), + m_buffer_type(BufferType::UNINITIALIZED) { - m_exec_done = [buffer_pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer](hailo_status){ - release_buffer(buffer_pool, mem_view, is_user_buffer); - }; } PipelineBuffer::PipelineBuffer(hailo_status action_status, const TransferDoneCallbackAsyncInfer &exec_done) : m_type(Type::DATA), - m_pool(nullptr), m_view(), + m_exec_done(exec_done), m_metadata(), m_is_user_buffer(false), m_should_call_exec_done(true), - m_action_status(action_status) + m_action_status(action_status), + m_buffer_type(BufferType::UNINITIALIZED) { - m_exec_done = [buffer_pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){ - exec_done(status); - release_buffer(buffer_pool, mem_view, is_user_buffer); - }; } -PipelineBuffer::PipelineBuffer(MemoryView view, const TransferDoneCallbackAsyncInfer &exec_done, hailo_status action_status, bool is_user_buffer, BufferPoolPtr pool, - bool should_measure) : +PipelineBuffer::PipelineBuffer(MemoryView view, const TransferDoneCallbackAsyncInfer &exec_done, hailo_status action_status, + bool is_user_buffer, BufferPoolWeakPtr pool, bool should_measure) : m_type(Type::DATA), m_pool(pool), m_view(view), m_metadata(Metadata(add_timestamp(should_measure))), m_is_user_buffer(is_user_buffer), m_should_call_exec_done(true), - m_action_status(action_status) + m_action_status(action_status), + m_buffer_type(BufferType::VIEW) { - m_exec_done = [buffer_pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){ + m_exec_done = [pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){ exec_done(status); - release_buffer(buffer_pool, mem_view, is_user_buffer); + if (auto buffer_pool = pool.lock()) { + return_buffer_to_pool(buffer_pool, mem_view, is_user_buffer); + } }; } PipelineBuffer::PipelineBuffer(hailo_pix_buffer_t buffer, const TransferDoneCallbackAsyncInfer &exec_done) : m_type(Type::DATA), - m_pool(nullptr), m_view(), + m_exec_done(exec_done), m_metadata(), m_is_user_buffer(false), m_should_call_exec_done(true), - m_action_status(HAILO_SUCCESS) + m_action_status(HAILO_SUCCESS), + m_buffer_type(BufferType::PIX_BUFFER) { set_additional_data(std::make_shared(buffer)); - m_exec_done = [buffer_pool = m_pool, mem_view = m_view, is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){ +} + +PipelineBuffer::PipelineBuffer(hailo_dma_buffer_t dma_buffer, const TransferDoneCallbackAsyncInfer &exec_done, hailo_status action_status, + bool is_user_buffer, BufferPoolWeakPtr pool, bool should_measure) : + m_type(Type::DATA), + m_pool(pool), + m_view(), + m_metadata(Metadata(add_timestamp(should_measure))), + m_is_user_buffer(is_user_buffer), + m_should_call_exec_done(true), + m_action_status(action_status), + m_buffer_type(BufferType::DMA_BUFFER) +{ + set_additional_data(std::make_shared(dma_buffer)); + m_exec_done = [pool = m_pool, dma_buffer = get_metadata().get_additional_data(), is_user_buffer = m_is_user_buffer, exec_done = exec_done](hailo_status status){ exec_done(status); - release_buffer(buffer_pool, mem_view, is_user_buffer); + if (auto buffer_pool = pool.lock()) { + return_buffer_to_pool(buffer_pool, dma_buffer->m_dma_buffer, is_user_buffer); + } }; } @@ -110,7 +126,8 @@ PipelineBuffer::PipelineBuffer(PipelineBuffer &&other) : m_metadata(std::move(other.m_metadata)), m_is_user_buffer(std::move(other.m_is_user_buffer)), m_should_call_exec_done(std::exchange(other.m_should_call_exec_done, false)), - m_action_status(std::move(other.m_action_status)) + m_action_status(std::move(other.m_action_status)), + m_buffer_type(other.m_buffer_type) {} PipelineBuffer &PipelineBuffer::operator=(PipelineBuffer &&other) @@ -123,6 +140,7 @@ PipelineBuffer &PipelineBuffer::operator=(PipelineBuffer &&other) m_is_user_buffer = std::move(other.m_is_user_buffer); m_should_call_exec_done = std::exchange(other.m_should_call_exec_done, false); m_action_status = std::move(other.m_action_status); + m_buffer_type = std::move(other.m_buffer_type); return *this; } @@ -145,12 +163,35 @@ uint8_t* PipelineBuffer::data() size_t PipelineBuffer::size() const { - return m_view.size(); + if (BufferType::DMA_BUFFER == m_buffer_type) { + auto dma_buffer = get_metadata().get_additional_data(); + return dma_buffer->m_dma_buffer.size; + } else if (BufferType::PIX_BUFFER == m_buffer_type) { + auto pix_buffer = get_metadata().get_additional_data(); + size_t size = 0; + for (uint32_t i = 0; i < pix_buffer->m_pix_buffer.number_of_planes; i++) { + size += pix_buffer->m_pix_buffer.planes[i].plane_size; + } + return size; + } else { + return m_view.size(); + } } -MemoryView PipelineBuffer::as_view() +Expected PipelineBuffer::as_view(BufferProtection dma_buffer_protection) { - return m_view; + if (BufferType::DMA_BUFFER == m_buffer_type) { + assert(BufferProtection::NONE != dma_buffer_protection); + auto status = set_dma_buf_as_memview(dma_buffer_protection); + CHECK_SUCCESS_AS_EXPECTED(status); + } else if (BufferType::PIX_BUFFER == m_buffer_type) { + LOGGER__ERROR("Can't call as_view for buffer of type pix_buffer."); + return make_unexpected(HAILO_INVALID_ARGUMENT); + } + + auto mem_view = m_view; + + return mem_view; } PipelineBuffer::Type PipelineBuffer::get_type() const @@ -163,12 +204,17 @@ PipelineBuffer::Metadata PipelineBuffer::get_metadata() const return m_metadata; } +BufferType PipelineBuffer::get_buffer_type() const +{ + return m_buffer_type; +} + Expected PipelineBuffer::as_hailo_pix_buffer(hailo_format_order_t order) { auto pix_buffer = get_metadata().get_additional_data(); if (nullptr == pix_buffer) { - auto mem_view = as_view(); + TRY(auto mem_view, as_view(BufferProtection::READ)); return HailoRTCommon::as_hailo_pix_buffer(mem_view, order); } else { uint32_t expected_number_of_planes; @@ -194,9 +240,9 @@ Expected PipelineBuffer::as_hailo_pix_buffer(hailo_format_or } } -void PipelineBuffer::set_metadata(Metadata &&val) +void PipelineBuffer::set_metadata_start_time(PipelineTimePoint val) { - m_metadata = std::move(val); + m_metadata.set_start_time(val); } PipelineTimePoint PipelineBuffer::add_timestamp(bool should_measure) @@ -204,10 +250,30 @@ PipelineTimePoint PipelineBuffer::add_timestamp(bool should_measure) return should_measure ? std::chrono::steady_clock::now() : PipelineTimePoint{}; } -void PipelineBuffer::release_buffer(BufferPoolPtr buffer_pool_ptr, MemoryView mem_view, bool is_user_buffer) +void PipelineBuffer::return_buffer_to_pool(BufferPoolWeakPtr buffer_pool_weak_ptr, MemoryView mem_view, bool is_user_buffer) { - if ((nullptr != buffer_pool_ptr) && (!is_user_buffer)) { - hailo_status status = buffer_pool_ptr->release_buffer(mem_view); + if (is_user_buffer) { + return; + } + + if (auto buffer_pool_ptr = buffer_pool_weak_ptr.lock() ) { + auto pipeline_buffer = PipelineBuffer(mem_view, [](hailo_status){}, HAILO_SUCCESS, false, buffer_pool_ptr, buffer_pool_ptr->should_measure_vstream_latency()); + hailo_status status = buffer_pool_ptr->return_buffer_to_pool(std::move(pipeline_buffer)); + if (HAILO_SUCCESS != status) { + LOGGER__CRITICAL("Releasing buffer in buffer pool failed! status = {}", status); + } + } +} + +void PipelineBuffer::return_buffer_to_pool(BufferPoolWeakPtr buffer_pool_weak_ptr, hailo_dma_buffer_t dma_buffer, bool is_user_buffer) +{ + if (is_user_buffer) { + return; + } + + if (auto buffer_pool_ptr = buffer_pool_weak_ptr.lock() ) { + auto pipeline_buffer = PipelineBuffer(dma_buffer, [](hailo_status){}, HAILO_SUCCESS, false, buffer_pool_ptr, buffer_pool_ptr->should_measure_vstream_latency()); + hailo_status status = buffer_pool_ptr->return_buffer_to_pool(std::move(pipeline_buffer)); if (HAILO_SUCCESS != status) { LOGGER__CRITICAL("Releasing buffer in buffer pool failed! status = {}", status); } @@ -224,6 +290,25 @@ void PipelineBuffer::set_action_status(hailo_status status) m_action_status = status; } +hailo_status PipelineBuffer::set_dma_buf_as_memview(BufferProtection dma_buffer_protection) +{ + auto dma_buffer = get_metadata().get_additional_data(); + + TRY(m_view, DmaBufferUtils::mmap_dma_buffer(dma_buffer->m_dma_buffer, dma_buffer_protection)); + + m_exec_done = [mem_view=m_view, exec_done=m_exec_done, dma_buffer, dma_buffer_protection](hailo_status status) { + auto mumap_status = DmaBufferUtils::munmap_dma_buffer(dma_buffer->m_dma_buffer, mem_view, dma_buffer_protection); + if (HAILO_SUCCESS != mumap_status) { + LOGGER__ERROR("Failed to unmap dma buffer"); + status = HAILO_FILE_OPERATION_FAILURE; + } + exec_done(status); + }; + + m_buffer_type = BufferType::VIEW; + return HAILO_SUCCESS; +} + void PipelineBuffer::call_exec_done() { if (m_should_call_exec_done) { @@ -243,15 +328,16 @@ Expected BufferPool::create(size_t buffer_size, size_t buffer_cou } const bool measure_vstream_latency = (vstream_flags & HAILO_VSTREAM_STATS_MEASURE_LATENCY) != 0; - auto free_mem_views = SpscQueue::create(buffer_count, shutdown_event, BUFFER_POOL_DEFAULT_QUEUE_TIMEOUT); - CHECK_EXPECTED(free_mem_views); - - auto done_cbs = SpscQueue::create(buffer_count, shutdown_event, BUFFER_POOL_DEFAULT_QUEUE_TIMEOUT); - CHECK_EXPECTED(done_cbs); + TRY(auto pipeline_buffers_queue, SpscQueue::create(buffer_count, shutdown_event, BUFFER_POOL_DEFAULT_QUEUE_TIMEOUT)); std::vector buffers; + buffers.reserve(buffer_count); + + auto buffer_pool_ptr = make_shared_nothrow(buffer_size, is_empty, measure_vstream_latency, std::move(buffers), + std::move(pipeline_buffers_queue), std::move(queue_size_accumulator), buffer_count); + CHECK_AS_EXPECTED(nullptr != buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY); + if (!is_empty) { - buffers.reserve(buffer_count); for (size_t i = 0; i < buffer_count; i++) { BufferStorageParams buffer_storage_params; if (is_dma_able) { @@ -260,30 +346,28 @@ Expected BufferPool::create(size_t buffer_size, size_t buffer_cou auto buffer = Buffer::create(buffer_size, buffer_storage_params); CHECK_EXPECTED(buffer); - auto status = free_mem_views->enqueue(MemoryView(buffer.value())); + auto pipeline_buffer = PipelineBuffer(MemoryView(buffer.value()), [](hailo_status){}, HAILO_SUCCESS, false, buffer_pool_ptr, + buffer_pool_ptr->m_measure_vstream_latency); + + auto status = buffer_pool_ptr->enqueue_buffer(std::move(pipeline_buffer)); CHECK_SUCCESS_AS_EXPECTED(status); - buffers.emplace_back(buffer.release()); + buffer_pool_ptr->m_buffers.emplace_back(buffer.release()); } } - auto buffer_pool_ptr = make_shared_nothrow(buffer_size, is_empty, measure_vstream_latency, std::move(buffers), - free_mem_views.release(), done_cbs.release(), std::move(queue_size_accumulator), buffer_count); - CHECK_AS_EXPECTED(nullptr != buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY); - return buffer_pool_ptr; } BufferPool::BufferPool(size_t buffer_size, bool is_holding_user_buffers, bool measure_vstream_latency, std::vector &&buffers, - SpscQueue &&free_mem_views, SpscQueue &&done_cbs, AccumulatorPtr &&queue_size_accumulator, + SpscQueue &&pipeline_buffers_queue, AccumulatorPtr &&queue_size_accumulator, size_t max_buffer_count) : m_buffer_size(buffer_size), m_is_holding_user_buffers(is_holding_user_buffers), m_max_buffer_count(max_buffer_count), m_measure_vstream_latency(measure_vstream_latency), m_buffers(std::move(buffers)), - m_free_mem_views(std::move(free_mem_views)), - m_done_cbs(std::move(done_cbs)), + m_pipeline_buffers_queue(std::move(pipeline_buffers_queue)), m_queue_size_accumulator(std::move(queue_size_accumulator)), m_is_already_running(false) { @@ -295,35 +379,41 @@ size_t BufferPool::buffer_size() return m_buffer_size.load(); } -hailo_status BufferPool::enqueue_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done) +hailo_status BufferPool::enqueue_buffer(PipelineBuffer &&pipeline_buffer) { + // TODO: add support for pix_buffer here (currently enqueue_buffer is called only to add the output_buffers which are never pix_buffers) m_is_already_running = true; auto pool_buffer_size = buffer_size(); - CHECK(mem_view.size() == pool_buffer_size, HAILO_INTERNAL_FAILURE, - "Buffer size is not the same as expected for pool! ({} != {})", mem_view.size(), pool_buffer_size); + CHECK(pipeline_buffer.size() == pool_buffer_size, HAILO_INTERNAL_FAILURE, + "Buffer size is not the same as expected for pool! ({} != {})", pipeline_buffer.size(), pool_buffer_size); std::unique_lock lock(m_enqueue_mutex); - auto status = m_free_mem_views.enqueue(mem_view); + auto status = m_pipeline_buffers_queue.enqueue(std::move(pipeline_buffer)); if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) { return HAILO_SHUTDOWN_EVENT_SIGNALED; } CHECK_SUCCESS(status); - // TODO: Stop using 2 queues, hold a queue of pipeline_buffer instead. - status = m_done_cbs.enqueue(exec_done, true); // we get here only if acquire_free_mem_view succeeded, so we want to push cb to keep sync between the queues - CHECK_SUCCESS(status); - return HAILO_SUCCESS; } +bool BufferPool::should_measure_vstream_latency() +{ + return m_measure_vstream_latency; +} bool BufferPool::is_full() { return (m_max_buffer_count - num_of_buffers_in_pool() == 0); } +size_t BufferPool::max_capacity() +{ + return m_max_buffer_count; +} + size_t BufferPool::num_of_buffers_in_pool() { - return m_free_mem_views.size_approx(); + return m_pipeline_buffers_queue.size_approx(); } bool BufferPool::is_holding_user_buffers() @@ -335,74 +425,25 @@ Expected BufferPool::acquire_buffer(std::chrono::milliseconds ti bool ignore_shutdown_event) { m_is_already_running = true; - std::unique_lock lock(m_dequeue_mutex); - auto mem_view = acquire_free_mem_view(timeout, ignore_shutdown_event); - if ((HAILO_SUCCESS != mem_view.status()) && (m_is_holding_user_buffers)) { - auto done_cb = acquire_on_done_cb(timeout, ignore_shutdown_event); - if (HAILO_SHUTDOWN_EVENT_SIGNALED == done_cb.status()) { - return make_unexpected(HAILO_SHUTDOWN_EVENT_SIGNALED); - } - CHECK_EXPECTED(done_cb); - done_cb.value()(mem_view.status()); - } - if (HAILO_SHUTDOWN_EVENT_SIGNALED == mem_view.status()) { - return make_unexpected(HAILO_SHUTDOWN_EVENT_SIGNALED); - } - CHECK_EXPECTED(mem_view); - - if (m_is_holding_user_buffers) { - auto done_cb = acquire_on_done_cb(timeout, true); // we get here only if acquire_free_mem_view succeeded, so we want to pop cb to keep sync between the queues - if (HAILO_SHUTDOWN_EVENT_SIGNALED == done_cb.status()) { - return make_unexpected(HAILO_SHUTDOWN_EVENT_SIGNALED); - } - CHECK_EXPECTED(done_cb); - - return PipelineBuffer(mem_view.release(), done_cb.release(), HAILO_SUCCESS, m_is_holding_user_buffers, shared_from_this(), m_measure_vstream_latency); - } - - return PipelineBuffer(mem_view.release(), [](hailo_status){}, HAILO_SUCCESS, m_is_holding_user_buffers, shared_from_this(), m_measure_vstream_latency); -} - -Expected BufferPool::acquire_free_mem_view(std::chrono::milliseconds timeout, - bool ignore_shutdown_event) -{ if (nullptr != m_queue_size_accumulator) { - m_queue_size_accumulator->add_data_point(static_cast(m_free_mem_views.size_approx())); + m_queue_size_accumulator->add_data_point(static_cast(m_pipeline_buffers_queue.size_approx())); } - auto mem_view = m_free_mem_views.dequeue(timeout, ignore_shutdown_event); - if (HAILO_SHUTDOWN_EVENT_SIGNALED == mem_view.status()) { - return make_unexpected(mem_view.status()); - } - else if (HAILO_TIMEOUT == mem_view.status()) { - LOGGER__WARNING( - "Failed to acquire buffer because the buffer pool is empty. This could be caused by uneven reading and writing speeds, with a short user-defined timeout. (timeout={}ms)", - timeout.count()); - return make_unexpected(mem_view.status()); - } - CHECK_EXPECTED(mem_view); - - return mem_view.release(); -} - -Expected BufferPool::acquire_on_done_cb(std::chrono::milliseconds timeout, - bool ignore_shutdown_event) -{ - auto done_cb = m_done_cbs.dequeue(timeout, ignore_shutdown_event); - if (HAILO_SHUTDOWN_EVENT_SIGNALED == done_cb.status()) { - return make_unexpected(done_cb.status()); + auto pipeline_buffer = m_pipeline_buffers_queue.dequeue(timeout, ignore_shutdown_event); + if (HAILO_SHUTDOWN_EVENT_SIGNALED == pipeline_buffer.status()) { + return make_unexpected(pipeline_buffer.status()); } - else if (HAILO_TIMEOUT == done_cb.status()) { + else if (HAILO_TIMEOUT == pipeline_buffer.status()) { LOGGER__WARNING( "Failed to acquire buffer because the buffer pool is empty. This could be caused by uneven reading and writing speeds, with a short user-defined timeout. (timeout={}ms)", timeout.count()); - return make_unexpected(done_cb.status()); + return make_unexpected(pipeline_buffer.status()); } - CHECK_EXPECTED(done_cb); + CHECK_EXPECTED(pipeline_buffer); - return done_cb.release(); + return pipeline_buffer.release(); } AccumulatorPtr BufferPool::get_queue_size_accumulator() @@ -430,11 +471,11 @@ Expected BufferPool::get_available_buffer(PipelineBuffer &&optio return acquired_buffer.release(); } -hailo_status BufferPool::release_buffer(MemoryView mem_view) +hailo_status BufferPool::return_buffer_to_pool(PipelineBuffer &&pipeline_buffer) { std::unique_lock lock(m_enqueue_mutex); // This can be called after the shutdown event was signaled so we ignore it here - return m_free_mem_views.enqueue(std::move(mem_view), true); + return m_pipeline_buffers_queue.enqueue(std::move(pipeline_buffer), true); } hailo_status BufferPool::map_to_vdevice(VDevice &vdevice, hailo_dma_buffer_direction_t direction) @@ -836,10 +877,9 @@ void PipelineElement::print_deep_description(std::vector &visited_e } } -hailo_status PipelineElement::enqueue_execution_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done) +hailo_status PipelineElement::enqueue_execution_buffer(PipelineBuffer &&pipeline_buffer) { - (void)mem_view; - (void)exec_done; + (void)pipeline_buffer; LOGGER__ERROR("enqueue_execution_buffer is not implemented for {}!", name()); return HAILO_NOT_IMPLEMENTED; }; @@ -867,12 +907,12 @@ hailo_status PipelineElement::empty_buffer_pool(BufferPoolPtr pool, hailo_status return HAILO_SUCCESS; } -Expected PipelineElement::can_push_buffer_upstream() +Expected PipelineElement::can_push_buffer_upstream(uint32_t /*frames_count*/) { return make_unexpected(HAILO_NOT_IMPLEMENTED); } -Expected PipelineElement::can_push_buffer_downstream() +Expected PipelineElement::can_push_buffer_downstream(uint32_t /*frames_count*/) { return make_unexpected(HAILO_NOT_IMPLEMENTED); } diff --git a/hailort/libhailort/src/net_flow/pipeline/pipeline.hpp b/hailort/libhailort/src/net_flow/pipeline/pipeline.hpp index 67d7d2d..1f6e1f1 100644 --- a/hailort/libhailort/src/net_flow/pipeline/pipeline.hpp +++ b/hailort/libhailort/src/net_flow/pipeline/pipeline.hpp @@ -16,7 +16,7 @@ #include "hailo/runtime_statistics.hpp" #include "hailo/dma_mapped_buffer.hpp" #include "net_flow/ops/nms_post_process.hpp" - +#include "hailo/network_group.hpp" #include "utils/thread_safe_queue.hpp" #include @@ -34,14 +34,15 @@ enum class PipelineDirection PUSH, }; -enum class BufferType +enum class BufferProtection { - UNINITIALIZED, - VIEW, - PIX_BUFFER, - DMA_BUFFER, + NONE, + READ, + WRITE, + READ_WRITE, }; + using TransferDoneCallbackAsyncInfer = std::function; using PipelineTimePoint = std::chrono::steady_clock::time_point; @@ -71,8 +72,15 @@ struct PixBufferPipelineData : AdditionalData hailo_pix_buffer_t m_pix_buffer; }; +struct DmaBufferPipelineData : AdditionalData +{ + DmaBufferPipelineData(const hailo_dma_buffer_t &buffer) : m_dma_buffer(buffer) {}; + hailo_dma_buffer_t m_dma_buffer; +}; + class BufferPool; using BufferPoolPtr = std::shared_ptr; +using BufferPoolWeakPtr = std::weak_ptr; class PipelineBuffer final { @@ -114,9 +122,11 @@ public: PipelineBuffer(Type type); // TODO HRT-12185: remove the option to pass a lambda as a parameter and save it as a member since it increases the memory consumption Significantly PipelineBuffer(hailo_status action_status = HAILO_SUCCESS, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){}); - PipelineBuffer(MemoryView view, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){}, - hailo_status action_status = HAILO_SUCCESS, bool is_user_buffer = true, BufferPoolPtr pool = nullptr, bool should_measure = false); + PipelineBuffer(MemoryView view, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){}, hailo_status action_status = HAILO_SUCCESS, + bool is_user_buffer = true, BufferPoolWeakPtr pool = BufferPoolWeakPtr(), bool should_measure = false); PipelineBuffer(hailo_pix_buffer_t buffer, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){}); + PipelineBuffer(hailo_dma_buffer_t dma_buffer, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){}, + hailo_status action_status = HAILO_SUCCESS, bool is_user_buffer = true, BufferPoolWeakPtr pool = BufferPoolWeakPtr(), bool should_measure = false); ~PipelineBuffer(); @@ -128,58 +138,60 @@ public: uint8_t* data(); size_t size() const; - MemoryView as_view(); + Expected as_view(BufferProtection dma_buffer_protection); Expected as_hailo_pix_buffer(hailo_format_order_t order = HAILO_FORMAT_ORDER_AUTO); Type get_type() const; Metadata get_metadata() const; - void set_metadata(Metadata &&val); + void set_metadata_start_time(PipelineTimePoint val); void set_additional_data(std::shared_ptr data) { m_metadata.set_additional_data(data);} hailo_status action_status(); void set_action_status(hailo_status status); void call_exec_done(); + BufferType get_buffer_type() const; private: Type m_type; - BufferPoolPtr m_pool; + BufferPoolWeakPtr m_pool; MemoryView m_view; TransferDoneCallbackAsyncInfer m_exec_done; Metadata m_metadata; bool m_is_user_buffer; bool m_should_call_exec_done; hailo_status m_action_status; + BufferType m_buffer_type; static PipelineTimePoint add_timestamp(bool should_measure); - static void release_buffer(BufferPoolPtr buffer_pool_ptr, MemoryView mem_view, bool is_user_buffer); + static void return_buffer_to_pool(BufferPoolWeakPtr buffer_pool_weak_ptr, MemoryView mem_view, bool is_user_buffer); + static void return_buffer_to_pool(BufferPoolWeakPtr buffer_pool_weak_ptr, hailo_dma_buffer_t dma_buffer, bool is_user_buffer); + hailo_status set_dma_buf_as_memview(BufferProtection dma_buffer_protection); }; -// The buffer pool has to be created as a shared pointer (via the create function) because we use shared_from_this(), -// which is only allowed if there is already a shared pointer pointing to "this"! -class BufferPool : public std::enable_shared_from_this +class BufferPool { public: static Expected create(size_t buffer_size, size_t buffer_count, EventPtr shutdown_event, hailo_pipeline_elem_stats_flags_t elem_flags, hailo_vstream_stats_flags_t vstream_flags, bool is_empty = false, bool dma_able = false); - BufferPool(size_t buffer_size, bool is_holding_user_buffers, bool measure_vstream_latency, std::vector &&buffers, SpscQueue &&free_mem_views, - SpscQueue &&done_cbs, AccumulatorPtr &&queue_size_accumulator, size_t max_buffer_count); + BufferPool(size_t buffer_size, bool is_holding_user_buffers, bool measure_vstream_latency, std::vector &&buffers, + SpscQueue &&pipeline_buffers_queue, AccumulatorPtr &&queue_size_accumulator, size_t max_buffer_count); virtual ~BufferPool() = default; size_t buffer_size(); - hailo_status enqueue_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done = [](hailo_status){}); + hailo_status enqueue_buffer(PipelineBuffer &&pipeline_buffer); Expected acquire_buffer(std::chrono::milliseconds timeout, bool ignore_shutdown_event = false); AccumulatorPtr get_queue_size_accumulator(); Expected get_available_buffer(PipelineBuffer &&optional, std::chrono::milliseconds timeout); + bool should_measure_vstream_latency(); bool is_full(); + size_t max_capacity(); size_t num_of_buffers_in_pool(); bool is_holding_user_buffers(); hailo_status map_to_vdevice(VDevice &vdevice, hailo_dma_buffer_direction_t direction); hailo_status set_buffer_size(uint32_t buffer_size); private: - Expected acquire_free_mem_view(std::chrono::milliseconds timeout, bool ignore_shutdown_event = false); - Expected acquire_on_done_cb(std::chrono::milliseconds timeout, bool ignore_shutdown_event = false); - hailo_status release_buffer(MemoryView mem_view); + hailo_status return_buffer_to_pool(PipelineBuffer &&pipeline_buffer); std::atomic m_buffer_size; bool m_is_holding_user_buffers; @@ -196,8 +208,7 @@ private: // to the mapping objects. std::vector m_dma_mapped_buffers; - SpscQueue m_free_mem_views; - SpscQueue m_done_cbs; + SpscQueue m_pipeline_buffers_queue; AccumulatorPtr m_queue_size_accumulator; // we have enqueue and dequeue mutex to allow mpmc std::mutex m_enqueue_mutex; @@ -365,10 +376,10 @@ public: std::string links_description() const; void print_deep_description(std::vector &visited_elements); - virtual hailo_status enqueue_execution_buffer(MemoryView mem_view, const TransferDoneCallbackAsyncInfer &exec_done); + virtual hailo_status enqueue_execution_buffer(PipelineBuffer &&pipeline_buffer); hailo_status empty_buffer_pool(BufferPoolPtr pool, hailo_status error_status, std::chrono::milliseconds timeout); - virtual Expected can_push_buffer_upstream(); - virtual Expected can_push_buffer_downstream(); + virtual Expected can_push_buffer_upstream(uint32_t frames_count); + virtual Expected can_push_buffer_downstream(uint32_t frames_count); virtual Expected get_source_index_from_source_name(const std::string &/*source_name*/) { // This function is overriden in multi-srcs elements diff --git a/hailort/libhailort/src/net_flow/pipeline/queue_elements.cpp b/hailort/libhailort/src/net_flow/pipeline/queue_elements.cpp index 3527fde..5c89b60 100644 --- a/hailort/libhailort/src/net_flow/pipeline/queue_elements.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/queue_elements.cpp @@ -570,9 +570,9 @@ hailo_status AsyncPushQueueElement::execute_dequeue_user_buffers(hailo_status er return HAILO_SUCCESS; } -Expected AsyncPushQueueElement::can_push_buffer_downstream() +Expected AsyncPushQueueElement::can_push_buffer_downstream(uint32_t frames_count) { - return !m_queue.is_queue_full(); + return m_queue.size_approx() + frames_count < m_queue.max_capacity(); } Expected> PullQueueElement::create(const std::string &name, std::chrono::milliseconds timeout, @@ -778,7 +778,7 @@ Expected UserBufferQueueElement::run_pull(PipelineBuffer &&optio { CHECK_AS_EXPECTED(optional, HAILO_INVALID_ARGUMENT, "Optional buffer must be valid in {}!", name()); - hailo_status status = m_pool->enqueue_buffer(optional.as_view()); + hailo_status status = m_pool->enqueue_buffer(std::move(optional)); if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) { LOGGER__INFO("Shutdown event was signaled in enqueue of queue element {}!", name()); return make_unexpected(HAILO_SHUTDOWN_EVENT_SIGNALED); diff --git a/hailort/libhailort/src/net_flow/pipeline/queue_elements.hpp b/hailort/libhailort/src/net_flow/pipeline/queue_elements.hpp index 4f86daf..7713587 100644 --- a/hailort/libhailort/src/net_flow/pipeline/queue_elements.hpp +++ b/hailort/libhailort/src/net_flow/pipeline/queue_elements.hpp @@ -114,7 +114,7 @@ public: virtual hailo_status run_push(PipelineBuffer &&buffer, const PipelinePad &sink) override; virtual void run_push_async(PipelineBuffer &&buffer, const PipelinePad &sink) override; virtual hailo_status execute_dequeue_user_buffers(hailo_status error_status) override; - virtual Expected can_push_buffer_downstream() override; + virtual Expected can_push_buffer_downstream(uint32_t frames_count) override; protected: virtual hailo_status run_in_thread() override; diff --git a/hailort/libhailort/src/net_flow/pipeline/vstream.cpp b/hailort/libhailort/src/net_flow/pipeline/vstream.cpp index 02a1c1f..500ed3b 100644 --- a/hailort/libhailort/src/net_flow/pipeline/vstream.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/vstream.cpp @@ -769,7 +769,7 @@ hailo_status InputVStreamImpl::write(const MemoryView &buffer) } assert(1 == m_entry_element->sinks().size()); - auto status = m_entry_element->sinks()[0].run_push(PipelineBuffer(buffer, [](hailo_status){}, HAILO_SUCCESS, false, nullptr, m_measure_pipeline_latency)); + auto status = m_entry_element->sinks()[0].run_push(PipelineBuffer(buffer, [](hailo_status){}, HAILO_SUCCESS, false, BufferPoolWeakPtr(), m_measure_pipeline_latency)); if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) { LOGGER__INFO("Sending to VStream was shutdown!"); status = m_pipeline_status->load(); @@ -1125,7 +1125,7 @@ hailo_status OutputVStreamImpl::read(MemoryView buffer) } assert(1 == m_entry_element->sources().size()); - auto recv_buffer = m_entry_element->sources()[0].run_pull(PipelineBuffer(buffer, [](hailo_status){}, HAILO_SUCCESS, false, nullptr, m_measure_pipeline_latency)); + auto recv_buffer = m_entry_element->sources()[0].run_pull(PipelineBuffer(buffer, [](hailo_status){}, HAILO_SUCCESS, false, BufferPoolWeakPtr(), m_measure_pipeline_latency)); auto status = recv_buffer.status(); if (HAILO_SHUTDOWN_EVENT_SIGNALED == status) { LOGGER__INFO("Receiving to VStream was shutdown!"); diff --git a/hailort/libhailort/src/net_flow/pipeline/vstream_builder.cpp b/hailort/libhailort/src/net_flow/pipeline/vstream_builder.cpp index 522726a..e8b4e74 100644 --- a/hailort/libhailort/src/net_flow/pipeline/vstream_builder.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/vstream_builder.cpp @@ -13,6 +13,7 @@ #include "net_flow/ops/ssd_post_process.hpp" #include "net_flow/ops/yolox_post_process.hpp" #include "net_flow/ops/yolov8_post_process.hpp" +#include "net_flow/ops/yolov8_bbox_only_post_process.hpp" #include "net_flow/ops/yolov5_post_process.hpp" #include "net_flow/ops/yolov5_bbox_only_post_process.hpp" #include "net_flow/ops/argmax_post_process.hpp" @@ -473,10 +474,15 @@ Expected> VStreamsBuilderUtils::create_output_vstream { auto metadata = std::dynamic_pointer_cast(op_metadata); assert(nullptr != metadata); - auto op_expected = net_flow::YOLOV8PostProcessOp::create(metadata); - CHECK_EXPECTED(op_expected); - op = op_expected.release(); - break; + if (metadata->nms_config().bbox_only) { + auto bbox_only_metadata = std::dynamic_pointer_cast(op_metadata); + assert(nullptr != bbox_only_metadata); + TRY(op, net_flow::YOLOv8BboxOnlyPostProcessOp::create(bbox_only_metadata)); + break; + } else { + TRY(op, net_flow::YOLOV8PostProcessOp::create(metadata)); + break; + } } case (net_flow::OperationType::YOLOV5): { diff --git a/hailort/libhailort/src/network_group/network_group.cpp b/hailort/libhailort/src/network_group/network_group.cpp index 569ead8..bc3de09 100644 --- a/hailort/libhailort/src/network_group/network_group.cpp +++ b/hailort/libhailort/src/network_group/network_group.cpp @@ -186,6 +186,16 @@ void ConfiguredNetworkGroup::increase_ongoing_callbacks() m_ongoing_transfers++; } +Expected> ConfiguredNetworkGroupBase::create(const ConfigureNetworkParams &config_params, + std::vector> &&core_ops, NetworkGroupMetadata &&metadata) +{ + auto net_group_ptr = std::shared_ptr(new (std::nothrow) + ConfiguredNetworkGroupBase(config_params, std::move(core_ops), std::move(metadata))); + CHECK_NOT_NULL_AS_EXPECTED(net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); + + return net_group_ptr; +} + Expected> ConfiguredNetworkGroupBase::activate( const hailo_activate_network_group_params_t &network_group_params) { @@ -801,9 +811,17 @@ hailo_status ConfiguredNetworkGroupBase::infer_async(const NamedBuffersCallbacks InferRequest infer_request{}; for (auto &named_buffer_callback : named_buffers_callbacks) { const auto &name = named_buffer_callback.first; - const auto &buffer = named_buffer_callback.second.first; const auto &callback = named_buffer_callback.second.second; - infer_request.transfers.emplace(name, TransferRequest{buffer, callback}); + if (BufferType::VIEW == named_buffer_callback.second.first.buffer_type) { + const auto &buffer = named_buffer_callback.second.first.view; + infer_request.transfers.emplace(name, TransferRequest{buffer, callback}); + } else if (BufferType::DMA_BUFFER == named_buffer_callback.second.first.buffer_type) { + const auto &dma_buffer = named_buffer_callback.second.first.dma_buffer; + infer_request.transfers.emplace(name, TransferRequest{dma_buffer, callback}); + } else { + LOGGER__ERROR("infer_async does not support buffers with type {}", named_buffer_callback.second.first.buffer_type); + return HAILO_INVALID_ARGUMENT; + } } infer_request.callback = [this, infer_request_done_cb](hailo_status status){ if (status == HAILO_STREAM_ABORT) { @@ -829,4 +847,46 @@ hailo_status ConfiguredNetworkGroupBase::infer_async(const NamedBuffersCallbacks return HAILO_SUCCESS; } +Expected ConfiguredNetworkGroupBase::get_cache_read_size() const +{ + CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION, + "get_cache_read_size() is not supported for multi core-op network groups"); + + return m_core_ops[0]->get_cache_read_size(); + +} + +Expected ConfiguredNetworkGroupBase::get_cache_write_size() const +{ + CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION, + "get_cache_write_size() is not supported for multi core-op network groups"); + + return m_core_ops[0]->get_cache_write_size(); +} + + +hailo_status ConfiguredNetworkGroupBase::init_cache(uint32_t read_offset, int32_t write_offset_delta) +{ + CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION, + "init_cache() is not supported for multi core-op network groups"); + + return m_core_ops[0]->init_cache(read_offset, write_offset_delta); +} + +Expected ConfiguredNetworkGroupBase::get_cache_info() const +{ + CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION, + "get_cache_info() is not supported for multi core-op network groups"); + + return m_core_ops[0]->get_cache_info(); +} + +hailo_status ConfiguredNetworkGroupBase::update_cache_offset(int32_t offset_delta_bytes) +{ + CHECK(m_core_ops.size() == 1, HAILO_INVALID_OPERATION, + "update_cache_offset() is not supported for multi core-op network groups"); + + return m_core_ops[0]->update_cache_offset(offset_delta_bytes); +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/network_group/network_group_internal.hpp b/hailort/libhailort/src/network_group/network_group_internal.hpp index 87b4a63..e037912 100644 --- a/hailort/libhailort/src/network_group/network_group_internal.hpp +++ b/hailort/libhailort/src/network_group/network_group_internal.hpp @@ -48,19 +48,15 @@ namespace hailort using stream_name_t = std::string; using op_name_t = std::string; +#define HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR "HAILORT_AUTO_UPDATE_CACHE_OFFSET" +#define HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DEFAULT "default" +#define HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DISABLED "disabled" class ConfiguredNetworkGroupBase : public ConfiguredNetworkGroup { public: static Expected> create(const ConfigureNetworkParams &config_params, - std::vector> &&core_ops, NetworkGroupMetadata &&metadata) - { - auto net_group_ptr = std::shared_ptr(new (std::nothrow) - ConfiguredNetworkGroupBase(config_params, std::move(core_ops), std::move(metadata))); - CHECK_NOT_NULL_AS_EXPECTED(net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); - - return net_group_ptr; - } + std::vector> &&core_ops, NetworkGroupMetadata &&metadata); virtual ~ConfiguredNetworkGroupBase() = default; ConfiguredNetworkGroupBase(const ConfiguredNetworkGroupBase &other) = delete; @@ -207,6 +203,11 @@ public: virtual hailo_status set_nms_max_accumulated_mask_size(const std::string &edge_name, uint32_t max_accumulated_mask_size) override; Expected> get_nms_meta_data(const std::string &edge_name); + + virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override; + virtual Expected get_cache_info() const override; + virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override; + private: ConfiguredNetworkGroupBase(const ConfigureNetworkParams &config_params, std::vector> &&core_ops, NetworkGroupMetadata &&metadata); @@ -217,6 +218,8 @@ private: hailo_status activate_low_level_streams(); hailo_status deactivate_low_level_streams(); + Expected get_cache_read_size() const; + Expected get_cache_write_size() const; const ConfigureNetworkParams m_config_params; std::vector> m_core_ops; @@ -335,6 +338,10 @@ public: virtual hailo_status set_nms_max_bboxes_per_class(const std::string &edge_name, uint32_t max_bboxes_per_class) override; virtual hailo_status set_nms_max_accumulated_mask_size(const std::string &edge_name, uint32_t max_accumulated_mask_size) override; + virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override; + virtual Expected get_cache_info() const override; + virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override; + private: ConfiguredNetworkGroupClient(NetworkGroupIdentifier &&identifier, const std::string &network_group_name); hailo_status create_client(); diff --git a/hailort/libhailort/src/os/posix/linux/dma_buffer_utils.cpp b/hailort/libhailort/src/os/posix/linux/dma_buffer_utils.cpp index 0f9a716..e0fb65a 100644 --- a/hailort/libhailort/src/os/posix/linux/dma_buffer_utils.cpp +++ b/hailort/libhailort/src/os/posix/linux/dma_buffer_utils.cpp @@ -19,59 +19,54 @@ namespace hailort { -Expected DmaBufferUtils::mmap_dma_buffer_write(hailo_dma_buffer_t dma_buffer) +Expected DmaBufferUtils::mmap_dma_buffer(hailo_dma_buffer_t dma_buffer, BufferProtection dma_buffer_protection) { - void* dma_buf_ptr = mmap(NULL, dma_buffer.size, PROT_WRITE, MAP_SHARED, dma_buffer.fd, 0); - CHECK_AS_EXPECTED(MAP_FAILED != dma_buf_ptr, HAILO_INTERNAL_FAILURE, "Failed to run mmap on DMA buffer for writing"); + int prot = 0; + uint64_t dma_buf_sync_flags = 0; + if (BufferProtection::READ == dma_buffer_protection) { + prot = PROT_READ; + dma_buf_sync_flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ; + } else if (BufferProtection::WRITE == dma_buffer_protection) { + prot = PROT_WRITE; + dma_buf_sync_flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE; + } else { + return make_unexpected(HAILO_INVALID_ARGUMENT); + } + + void* dma_buf_ptr = mmap(NULL, dma_buffer.size, prot, MAP_SHARED, dma_buffer.fd, 0); + CHECK_AS_EXPECTED(MAP_FAILED != dma_buf_ptr, HAILO_INTERNAL_FAILURE, "Failed to run mmap on DMA buffer"); struct dma_buf_sync sync = { - .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE, + .flags = dma_buf_sync_flags, }; auto err = ioctl(dma_buffer.fd, DMA_BUF_IOCTL_SYNC, &sync); - CHECK_AS_EXPECTED(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC ioctl, errno {}", err); + CHECK_AS_EXPECTED(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC on FD, size: {}, fd: {}, address: {}, errno {}", dma_buffer.size, + dma_buffer.fd, static_cast(dma_buf_ptr), err); return MemoryView(dma_buf_ptr, dma_buffer.size); } -hailo_status DmaBufferUtils::munmap_dma_buffer_write(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview) +hailo_status DmaBufferUtils::munmap_dma_buffer(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview, BufferProtection dma_buffer_protection) { - struct dma_buf_sync sync = { - .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE, - }; - - auto err = ioctl(dma_buffer.fd, DMA_BUF_IOCTL_SYNC, &sync); - CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC ioctl, errno {}", err); - - err = munmap(dma_buffer_memview.data(), dma_buffer.size); - CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to munmap dma buffer, errno {}", err); - - return HAILO_SUCCESS; -} - -Expected DmaBufferUtils::mmap_dma_buffer_read(hailo_dma_buffer_t dma_buffer) -{ - void* dma_buf_ptr = mmap(NULL, dma_buffer.size, PROT_READ, MAP_SHARED, dma_buffer.fd, 0); - CHECK_AS_EXPECTED(MAP_FAILED != dma_buf_ptr, HAILO_INTERNAL_FAILURE, "Failed to run mmap on DMA buffer for reading"); + uint64_t dma_buf_sync_flags = 0; + if (BufferProtection::READ == dma_buffer_protection) { + dma_buf_sync_flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ; + } else if (BufferProtection::WRITE == dma_buffer_protection) { + dma_buf_sync_flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE; + } else { + return make_unexpected(HAILO_INVALID_ARGUMENT); + } struct dma_buf_sync sync = { - .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ, + .flags = dma_buf_sync_flags, }; - auto err = ioctl(dma_buffer.fd, DMA_BUF_IOCTL_SYNC, &sync); - CHECK_AS_EXPECTED(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC ioctl, errno {}", err); - return MemoryView(dma_buf_ptr, dma_buffer.size); -} - -hailo_status DmaBufferUtils::munmap_dma_buffer_read(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview) -{ - struct dma_buf_sync sync = { - .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ, - }; auto err = ioctl(dma_buffer.fd, DMA_BUF_IOCTL_SYNC, &sync); CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to run DMA_BUF_IOCTL_SYNC ioctl, errno {}", err); - err = munmap(dma_buffer_memview.data(), dma_buffer.size); - CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to unmap dma buffer, errno {}", err); + err = munmap(static_cast(dma_buffer_memview.data()), dma_buffer.size); + CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Failed to munmap dma buffer, size: {}, fd: {}, address: {}, errno {}", dma_buffer.size, dma_buffer.fd, + static_cast(dma_buffer_memview.data()), err); return HAILO_SUCCESS; } diff --git a/hailort/libhailort/src/os/posix/qnx/dma_buffer_utils.cpp b/hailort/libhailort/src/os/posix/qnx/dma_buffer_utils.cpp index 4937a7f..cadbaec 100644 --- a/hailort/libhailort/src/os/posix/qnx/dma_buffer_utils.cpp +++ b/hailort/libhailort/src/os/posix/qnx/dma_buffer_utils.cpp @@ -15,22 +15,13 @@ namespace hailort { -Expected DmaBufferUtils::mmap_dma_buffer_write(hailo_dma_buffer_t /*dma_buffer*/) +Expected DmaBufferUtils::mmap_dma_buffer(hailo_dma_buffer_t /*dma_buffer*/, BufferProtection /*dma_buffer_protection*/) { return make_unexpected(HAILO_NOT_IMPLEMENTED); } -hailo_status DmaBufferUtils::munmap_dma_buffer_write(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/) -{ - return HAILO_NOT_IMPLEMENTED; -} - -Expected DmaBufferUtils::mmap_dma_buffer_read(hailo_dma_buffer_t /*dma_buffer*/) -{ - return make_unexpected(HAILO_NOT_IMPLEMENTED); -} - -hailo_status DmaBufferUtils::munmap_dma_buffer_read(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/) +hailo_status DmaBufferUtils::munmap_dma_buffer(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/, + BufferProtection /*dma_buffer_protection*/) { return HAILO_NOT_IMPLEMENTED; } diff --git a/hailort/libhailort/src/os/windows/dma_buffer_utils.cpp b/hailort/libhailort/src/os/windows/dma_buffer_utils.cpp index b8404bd..45302fe 100644 --- a/hailort/libhailort/src/os/windows/dma_buffer_utils.cpp +++ b/hailort/libhailort/src/os/windows/dma_buffer_utils.cpp @@ -13,22 +13,13 @@ namespace hailort { -Expected DmaBufferUtils::mmap_dma_buffer_write(hailo_dma_buffer_t /*dma_buffer*/) +Expected DmaBufferUtils::mmap_dma_buffer(hailo_dma_buffer_t /*dma_buffer*/, BufferProtection /*dma_buffer_protection*/) { return make_unexpected(HAILO_NOT_IMPLEMENTED); } -hailo_status DmaBufferUtils::munmap_dma_buffer_write(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/) -{ - return HAILO_NOT_IMPLEMENTED; -} - -Expected DmaBufferUtils::mmap_dma_buffer_read(hailo_dma_buffer_t /*dma_buffer*/) -{ - return make_unexpected(HAILO_NOT_IMPLEMENTED); -} - -hailo_status DmaBufferUtils::munmap_dma_buffer_read(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/) +hailo_status DmaBufferUtils::munmap_dma_buffer(hailo_dma_buffer_t /*dma_buffer*/, MemoryView /*dma_buffer_memview*/, + BufferProtection /*dma_buffer_protection*/) { return HAILO_NOT_IMPLEMENTED; } diff --git a/hailort/libhailort/src/service/hailort_rpc_client.cpp b/hailort/libhailort/src/service/hailort_rpc_client.cpp index 85acf5f..db30581 100644 --- a/hailort/libhailort/src/service/hailort_rpc_client.cpp +++ b/hailort/libhailort/src/service/hailort_rpc_client.cpp @@ -12,6 +12,7 @@ #include "hef/hef_internal.hpp" #include "hailort_rpc_client.hpp" #include "net_flow/ops_metadata/yolov8_op_metadata.hpp" +#include "net_flow/ops_metadata/yolov8_bbox_only_op_metadata.hpp" #include "net_flow/ops_metadata/yolox_op_metadata.hpp" #include "net_flow/ops_metadata/ssd_op_metadata.hpp" #include "net_flow/ops_metadata/softmax_op_metadata.hpp" @@ -19,6 +20,7 @@ #include "net_flow/ops_metadata/nms_op_metadata.hpp" #include "net_flow/ops_metadata/yolov5_op_metadata.hpp" #include "net_flow/ops_metadata/yolov5_seg_op_metadata.hpp" +#include "net_flow/ops_metadata/yolov5_bbox_only_op_metadata.hpp" #include @@ -277,7 +279,7 @@ Expected HailoRtRpcClient::VDevice_get_callback_id(cons VDevice_convert_identifier_to_proto(identifier, proto_identifier); VDevice_get_callback_id_Reply reply; - ClientContextWithTimeout context; + grpc::ClientContext context; grpc::Status status = m_stub->VDevice_get_callback_id(&context, request, &reply); CHECK_GRPC_STATUS_AS_EXPECTED(status); assert(reply.status() < HAILO_STATUS_COUNT); @@ -835,31 +837,38 @@ Expected> deserialize_ops_metada nms_config_proto.number_of_classes(), nms_config_proto.background_removal(), nms_config_proto.background_removal_index(), - nms_config_proto.cross_classes()}; + nms_config_proto.cross_classes(), + nms_config_proto.bbox_only()}; } switch (static_cast(op_metadata_proto.type())) { case net_flow::OperationType::YOLOV8: { - auto expected_yolov8_post_process_config = create_yolov8_post_process_config(op_metadata_proto); - CHECK_EXPECTED(expected_yolov8_post_process_config); - auto expteted_yolov8_metadata = hailort::net_flow::Yolov8OpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config, - expected_yolov8_post_process_config.value(), - op_metadata_proto.network_name()); - CHECK_EXPECTED(expteted_yolov8_metadata); - ops_metadata_ptr.push_back(expteted_yolov8_metadata.value()); + TRY(auto yolov8_post_process_config, create_yolov8_post_process_config(op_metadata_proto)); + if (nms_post_process_config.bbox_only) { + TRY(auto yolov8_bbox_only_metadata, hailort::net_flow::Yolov8BboxOnlyOpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config, + yolov8_post_process_config, op_metadata_proto.network_name())); + ops_metadata_ptr.push_back(yolov8_bbox_only_metadata); + } else { + TRY(auto yolov8_metadata, hailort::net_flow::Yolov8OpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config, + yolov8_post_process_config, op_metadata_proto.network_name())); + ops_metadata_ptr.push_back(yolov8_metadata); + } break; } case net_flow::OperationType::YOLOV5: { - auto exected_yolov5_post_process_config = create_yolov5_post_process_config(op_metadata_proto); - CHECK_EXPECTED(exected_yolov5_post_process_config); - auto expteted_yolov5_metadata = hailort::net_flow::Yolov5OpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config, - exected_yolov5_post_process_config.value(), - op_metadata_proto.network_name()); - CHECK_EXPECTED(expteted_yolov5_metadata); - ops_metadata_ptr.push_back(expteted_yolov5_metadata.value()); + TRY(auto yolov5_post_process_config, create_yolov5_post_process_config(op_metadata_proto)); + if (nms_post_process_config.bbox_only) { + TRY(auto yolov5_bbox_only_metadata, hailort::net_flow::Yolov5BboxOnlyOpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config, + yolov5_post_process_config, op_metadata_proto.network_name())); + ops_metadata_ptr.push_back(yolov5_bbox_only_metadata); + } else { + TRY(auto yolov5_metadata, hailort::net_flow::Yolov5OpMetadata::create(inputs_metadata, outputs_metadata, nms_post_process_config, + yolov5_post_process_config, op_metadata_proto.network_name())); + ops_metadata_ptr.push_back(yolov5_metadata); + } break; } diff --git a/hailort/libhailort/src/service/network_group_client.cpp b/hailort/libhailort/src/service/network_group_client.cpp index 06c9dbd..c27b8ff 100644 --- a/hailort/libhailort/src/service/network_group_client.cpp +++ b/hailort/libhailort/src/service/network_group_client.cpp @@ -428,6 +428,22 @@ hailo_status ConfiguredNetworkGroupClient::set_nms_max_accumulated_mask_size(con return m_client->ConfiguredNetworkGroup_set_nms_max_accumulated_mask_size(m_identifier, edge_name, max_accumulated_mask_size); } +// TODO: support kv-cache over service (HRT-13968) +hailo_status ConfiguredNetworkGroupClient::init_cache(uint32_t /* read_offset */, int32_t /* write_offset_delta */) +{ + return HAILO_NOT_IMPLEMENTED; +} + +Expected ConfiguredNetworkGroupClient::get_cache_info() const +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +hailo_status ConfiguredNetworkGroupClient::update_cache_offset(int32_t /* offset_delta_bytes */) +{ + return HAILO_NOT_IMPLEMENTED; +} + hailo_status ConfiguredNetworkGroupClient::execute_callback(const ProtoCallbackIdentifier &cb_id) { if (cb_id.cb_type() == CALLBACK_TYPE_TRANSFER) { @@ -503,12 +519,15 @@ hailo_status ConfiguredNetworkGroupClient::infer_async(const NamedBuffersCallbac std::unique_lock lock(m_mutex); for (const auto &name_buffer_cb : named_buffers_callbacks) { auto cb_idx = get_unique_callback_idx(); - auto name_buffer_cb_tuple = std::make_tuple(name_buffer_cb.first, name_buffer_cb.second.first, name_buffer_cb.second.second); + CHECK(BufferType::VIEW == name_buffer_cb.second.first.buffer_type, HAILO_INVALID_OPERATION, + "Using dmabuf is not supported when working with hailort_service"); + + auto name_buffer_cb_tuple = std::make_tuple(name_buffer_cb.first, name_buffer_cb.second.first.view, name_buffer_cb.second.second); auto tuple_ptr = make_shared_nothrow(name_buffer_cb_tuple); CHECK_NOT_NULL(tuple_ptr, HAILO_OUT_OF_HOST_MEMORY); m_idx_to_callbacks.emplace(cb_idx, tuple_ptr); - cb_idx_to_stream_buffer.emplace_back(std::make_tuple(cb_idx, name_buffer_cb.first, name_buffer_cb.second.first)); + cb_idx_to_stream_buffer.emplace_back(std::make_tuple(cb_idx, name_buffer_cb.first, name_buffer_cb.second.first.view)); } } diff --git a/hailort/libhailort/src/stream_common/nms_stream.cpp b/hailort/libhailort/src/stream_common/nms_stream.cpp index b09be22..79aa29c 100644 --- a/hailort/libhailort/src/stream_common/nms_stream.cpp +++ b/hailort/libhailort/src/stream_common/nms_stream.cpp @@ -372,6 +372,8 @@ hailo_status NmsOutputStream::read_async_impl(TransferRequest &&transfer_request { CHECK(1 == transfer_request.transfer_buffers.size(), HAILO_INVALID_OPERATION, "NMS Reader stream supports only 1 transfer buffer"); + CHECK(TransferBufferType::MEMORYVIEW == transfer_request.transfer_buffers[0].type(), HAILO_INVALID_OPERATION, + "NMS stream doesn't support DMABUF buffer type"); // Currently leave as transfer request - because nms reader uses transfer request queue // TODO HRT-12239: Chagge when support async read with any aligned void ptr return m_reader_thread.launch_transfer(std::move(transfer_request)); @@ -479,8 +481,10 @@ void NmsReaderThread::process_transfer_requests() assert(1 == transfer_request.transfer_buffers.size()); assert(0 == transfer_request.transfer_buffers[0].offset()); + // TODO HRT-13827: Add support for nms stream read with dmabuf auto buffer = transfer_request.transfer_buffers[0].base_buffer(); - auto status = NMSStreamReader::read_nms(*m_base_stream, buffer.data(), 0, buffer.size(), m_stream_interface); + assert(buffer.has_value()); + auto status = NMSStreamReader::read_nms(*m_base_stream, buffer.value().data(), 0, buffer.value().size(), m_stream_interface); if ((HAILO_STREAM_NOT_ACTIVATED == status) || (HAILO_STREAM_ABORT == status)) { // On both deactivation/abort, we want to send HAILO_STREAM_ABORT since it is part of the callback diff --git a/hailort/libhailort/src/stream_common/queued_stream_buffer_pool.cpp b/hailort/libhailort/src/stream_common/queued_stream_buffer_pool.cpp index 3aadb45..885818e 100644 --- a/hailort/libhailort/src/stream_common/queued_stream_buffer_pool.cpp +++ b/hailort/libhailort/src/stream_common/queued_stream_buffer_pool.cpp @@ -62,10 +62,11 @@ hailo_status QueuedStreamBufferPool::enqueue(TransferBuffer &&buffer_info) { CHECK(buffer_info.offset() == 0, HAILO_INTERNAL_FAILURE, "Cant use offset on queued buffer pool"); CHECK(buffer_info.size() == m_storage[0]->size(), HAILO_INTERNAL_FAILURE, "Invalid enqueue buffer size"); - CHECK(buffer_info.base_buffer().data() == m_storage[m_next_enqueue_buffer_index]->data(), HAILO_INTERNAL_FAILURE, + TRY(auto base_buffer, buffer_info.base_buffer()); + CHECK(base_buffer.data() == m_storage[m_next_enqueue_buffer_index]->data(), HAILO_INTERNAL_FAILURE, "Out of order enqueue for queued stream buffer pool"); - m_queue.push(buffer_info.base_buffer()); + m_queue.push(base_buffer); m_next_enqueue_buffer_index = (m_next_enqueue_buffer_index + 1) % (m_storage.size()); return HAILO_SUCCESS; } diff --git a/hailort/libhailort/src/stream_common/remote_process_stream.cpp b/hailort/libhailort/src/stream_common/remote_process_stream.cpp index 4ad2b50..3178a20 100644 --- a/hailort/libhailort/src/stream_common/remote_process_stream.cpp +++ b/hailort/libhailort/src/stream_common/remote_process_stream.cpp @@ -15,13 +15,13 @@ namespace hailort constexpr size_t MIN_QUEUE_SIZE = 2; constexpr size_t DEFAULT_QUEUE_SIZE = 4; +constexpr size_t RemoteProcessBufferPool::BACKING_ARRAY_LENGTH; Expected> RemoteProcessBufferPool::create( hailo_stream_direction_t stream_direction, size_t frame_size, size_t queue_size) { - // queue_size must be some (power-of-2 minus 1) in order to fit CircularArray. - queue_size = get_nearest_powerof_2(static_cast(queue_size + 1), MIN_QUEUE_SIZE) - 1; - queue_size = std::min(queue_size, ONGOING_TRANSFERS_SIZE); + CHECK((queue_size >= MIN_QUEUE_SIZE) && (queue_size < BACKING_ARRAY_LENGTH), HAILO_INVALID_ARGUMENT, + "Queue size must be in the range [{}, {}) (received {})", MIN_QUEUE_SIZE, BACKING_ARRAY_LENGTH, queue_size); hailo_status status = HAILO_UNINITIALIZED; auto buffer_pool = make_unique_nothrow(stream_direction, frame_size, queue_size, diff --git a/hailort/libhailort/src/stream_common/remote_process_stream.hpp b/hailort/libhailort/src/stream_common/remote_process_stream.hpp index d7087dc..f4e6657 100644 --- a/hailort/libhailort/src/stream_common/remote_process_stream.hpp +++ b/hailort/libhailort/src/stream_common/remote_process_stream.hpp @@ -86,7 +86,10 @@ private: // Guards memory allocation. std::vector m_buffers_guard; - using BufferQueue = CircularArray>; + // Note: We use a fixed size array to avoid dynamic memory allocation, needed for shared memory. + // CircularArrays support working with sizes less than the backing array length. + static constexpr size_t BACKING_ARRAY_LENGTH = 1024; + using BufferQueue = CircularArray>; // On input streams - buffers with user data, ready to be sent to the hw. // On output streams - buffers that are ready, the stream can receive into them. diff --git a/hailort/libhailort/src/stream_common/stream_internal.cpp b/hailort/libhailort/src/stream_common/stream_internal.cpp index 520ab93..3bb9768 100644 --- a/hailort/libhailort/src/stream_common/stream_internal.cpp +++ b/hailort/libhailort/src/stream_common/stream_internal.cpp @@ -48,7 +48,7 @@ hailo_status InputStreamBase::write_async(const MemoryView &buffer, const Transf buffer.size(), get_frame_size()); auto wrapped_callback = [buffer, user_callback](hailo_status status) { - user_callback(CompletionInfo{status, buffer.data(), buffer.size()}); + user_callback(CompletionInfo(status, buffer.data(), buffer.size())); }; return write_async(TransferRequest(buffer, wrapped_callback)); } @@ -58,6 +58,16 @@ hailo_status InputStreamBase::write_async(const void *buffer, size_t size, const return write_async(MemoryView::create_const(buffer, size), user_callback); } +hailo_status InputStreamBase::write_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) +{ + CHECK(0 != size, HAILO_INVALID_ARGUMENT, "Invalid size was passed to write_async"); + + auto wrapped_callback = [dmabuf_fd, size, user_callback](hailo_status status) { + user_callback(CompletionInfo(status, dmabuf_fd, size)); + }; + return write_async(TransferRequest(hailo_dma_buffer_t{dmabuf_fd, size}, wrapped_callback)); +} + hailo_status InputStreamBase::write_async(TransferRequest &&) { LOGGER__ERROR("write_async not implemented for sync API"); @@ -129,7 +139,7 @@ hailo_status OutputStreamBase::read_async(MemoryView buffer, const TransferDoneC buffer.size(), get_frame_size()); auto wrapped_callback = [buffer, user_callback](hailo_status status) { - user_callback(CompletionInfo{status, const_cast(buffer.data()), buffer.size()}); + user_callback(CompletionInfo(status, const_cast(buffer.data()), buffer.size())); }; return read_async(TransferRequest(buffer, wrapped_callback)); } @@ -139,6 +149,16 @@ hailo_status OutputStreamBase::read_async(void *buffer, size_t size, const Trans return read_async(MemoryView(buffer, size), user_callback); } +hailo_status OutputStreamBase::read_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) +{ + CHECK(0 != size, HAILO_INVALID_ARGUMENT, "Invalid size was passed to read_async"); + + auto wrapped_callback = [dmabuf_fd, size, user_callback](hailo_status status) { + user_callback(CompletionInfo(status, dmabuf_fd, size)); + }; + return read_async(TransferRequest(hailo_dma_buffer_t{dmabuf_fd, size}, wrapped_callback)); +} + hailo_status OutputStreamBase::read_unaligned_address_async(const MemoryView &, const TransferDoneCallback &) { LOGGER__ERROR("read_unaligned_address_async not implemented OutputStreamBase"); diff --git a/hailort/libhailort/src/stream_common/stream_internal.hpp b/hailort/libhailort/src/stream_common/stream_internal.hpp index 6791267..4228f2f 100644 --- a/hailort/libhailort/src/stream_common/stream_internal.hpp +++ b/hailort/libhailort/src/stream_common/stream_internal.hpp @@ -97,6 +97,7 @@ public: virtual hailo_status write_async(const MemoryView &buffer, const TransferDoneCallback &user_callback) override final; virtual hailo_status write_async(const void *buffer, size_t size, const TransferDoneCallback &user_callback) override final; + virtual hailo_status write_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) override final; virtual hailo_status write_async(TransferRequest &&transfer_request); @@ -160,6 +161,7 @@ public: virtual hailo_status read_async(MemoryView buffer, const TransferDoneCallback &user_callback) override final; virtual hailo_status read_async(void *buffer, size_t size, const TransferDoneCallback &user_callback) override final; + virtual hailo_status read_async(int dmabuf_fd, size_t size, const TransferDoneCallback &user_callback) override final; virtual hailo_status read_async(TransferRequest &&transfer_request); virtual hailo_status read_unaligned_address_async(const MemoryView &buffer, const TransferDoneCallback &user_callback); diff --git a/hailort/libhailort/src/stream_common/transfer_common.cpp b/hailort/libhailort/src/stream_common/transfer_common.cpp index a5fd6a4..ef8db78 100644 --- a/hailort/libhailort/src/stream_common/transfer_common.cpp +++ b/hailort/libhailort/src/stream_common/transfer_common.cpp @@ -17,13 +17,22 @@ namespace hailort TransferBuffer::TransferBuffer() : m_base_buffer(MemoryView{}), m_size(0), - m_offset(0) + m_offset(0), + m_type(TransferBufferType::MEMORYVIEW) +{} + +TransferBuffer::TransferBuffer(hailo_dma_buffer_t dmabuf) : + m_dmabuf(dmabuf), + m_size(dmabuf.size), + m_offset(0), + m_type(TransferBufferType::DMABUF) {} TransferBuffer::TransferBuffer(MemoryView base_buffer, size_t size, size_t offset) : m_base_buffer(base_buffer), m_size(size), - m_offset(offset) + m_offset(offset), + m_type(TransferBufferType::MEMORYVIEW) { assert(m_size <= base_buffer.size()); assert(m_offset < base_buffer.size()); @@ -33,32 +42,50 @@ TransferBuffer::TransferBuffer(MemoryView base_buffer) : TransferBuffer(base_buffer, base_buffer.size(), 0) {} +Expected TransferBuffer::base_buffer() +{ + CHECK(TransferBufferType::DMABUF != m_type, HAILO_INTERNAL_FAILURE, + "base_buffer is not supported for DMABUF type TransferBuffer"); + + return Expected(m_base_buffer); +} + Expected TransferBuffer::map_buffer(HailoRTDriver &driver, HailoRTDriver::DmaDirection direction) { CHECK_AS_EXPECTED(!m_mappings, HAILO_INTERNAL_FAILURE, "Buffer is already mapped"); + if (TransferBufferType::DMABUF == m_type) { + auto mapped_buffer = vdma::MappedBuffer::create_shared_from_dmabuf(m_dmabuf.fd, m_dmabuf.size, driver, direction); + CHECK_EXPECTED(mapped_buffer); - vdma::DmaAbleBufferPtr dma_able_buffer; - const auto storage_key = std::make_pair(m_base_buffer.data(), m_base_buffer.size()); - if (auto storage = BufferStorageResourceManager::get_resource(storage_key)) { - auto dma_able_buffer_exp = storage->get()->get_dma_able_buffer(); - CHECK_EXPECTED(dma_able_buffer_exp); - dma_able_buffer = dma_able_buffer_exp.release(); + m_mappings = mapped_buffer.value(); + return mapped_buffer; } else { - auto dma_able_buffer_exp = vdma::DmaAbleBuffer::create_from_user_address(m_base_buffer.data(), m_base_buffer.size()); - CHECK_EXPECTED(dma_able_buffer_exp); - dma_able_buffer = dma_able_buffer_exp.release(); - } - auto mapped_buffer = vdma::MappedBuffer::create_shared(std::move(dma_able_buffer), driver, direction); - CHECK_EXPECTED(mapped_buffer); - - m_mappings = mapped_buffer.value(); - return mapped_buffer; + vdma::DmaAbleBufferPtr dma_able_buffer; + const auto storage_key = std::make_pair(m_base_buffer.data(), m_base_buffer.size()); + if (auto storage = BufferStorageResourceManager::get_resource(storage_key)) { + auto dma_able_buffer_exp = storage->get()->get_dma_able_buffer(); + CHECK_EXPECTED(dma_able_buffer_exp); + dma_able_buffer = dma_able_buffer_exp.release(); + } else { + auto dma_able_buffer_exp = vdma::DmaAbleBuffer::create_from_user_address(m_base_buffer.data(), m_base_buffer.size()); + CHECK_EXPECTED(dma_able_buffer_exp); + dma_able_buffer = dma_able_buffer_exp.release(); + } + + auto mapped_buffer = vdma::MappedBuffer::create_shared(std::move(dma_able_buffer), driver, direction); + CHECK_EXPECTED(mapped_buffer); + + m_mappings = mapped_buffer.value(); + return mapped_buffer; + } } hailo_status TransferBuffer::copy_to(MemoryView buffer) { CHECK(buffer.size() == m_size, HAILO_INTERNAL_FAILURE, "buffer size {} must be {}", buffer.size(), m_size); + CHECK(TransferBufferType::MEMORYVIEW == m_type, HAILO_INTERNAL_FAILURE, + "copy_to function is only supported in MEMORYVIEW type TransferBuffer"); auto continuous_parts = get_continuous_parts(); memcpy(buffer.data(), continuous_parts.first.data(), continuous_parts.first.size()); @@ -72,6 +99,8 @@ hailo_status TransferBuffer::copy_to(MemoryView buffer) hailo_status TransferBuffer::copy_from(const MemoryView buffer) { CHECK(buffer.size() == m_size, HAILO_INTERNAL_FAILURE, "buffer size {} must be {}", buffer.size(), m_size); + CHECK(TransferBufferType::MEMORYVIEW == m_type, HAILO_INTERNAL_FAILURE, + "copy_from function is only supported in MEMORYVIEW type TransferBuffer"); auto continuous_parts = get_continuous_parts(); memcpy(continuous_parts.first.data(), buffer.data(), continuous_parts.first.size()); diff --git a/hailort/libhailort/src/stream_common/transfer_common.hpp b/hailort/libhailort/src/stream_common/transfer_common.hpp index 66aaf40..572bb53 100644 --- a/hailort/libhailort/src/stream_common/transfer_common.hpp +++ b/hailort/libhailort/src/stream_common/transfer_common.hpp @@ -15,21 +15,27 @@ #include "vdma/driver/hailort_driver.hpp" #include "vdma/memory/mapped_buffer.hpp" +#include "common/os_utils.hpp" namespace hailort { +enum class TransferBufferType { + MEMORYVIEW = 0, + DMABUF +}; + // Contains buffer that can be transferred. The buffer can be circular - // It relies at [m_offset, m_base_buffer.size()) and [0, m_base_buffer.size() - m_size). class TransferBuffer final { public: TransferBuffer(); - + TransferBuffer(hailo_dma_buffer_t dmabuf); TransferBuffer(MemoryView base_buffer); TransferBuffer(MemoryView base_buffer, size_t size, size_t offset); - MemoryView base_buffer() { return m_base_buffer; } + Expected base_buffer(); size_t offset() const { return m_offset; } size_t size() const { return m_size; } @@ -38,6 +44,8 @@ public: hailo_status copy_to(MemoryView buffer); hailo_status copy_from(const MemoryView buffer); + TransferBufferType type () const {return m_type;} + private: bool is_wrap_around() const; @@ -50,9 +58,14 @@ private: // 2. If the buffer is not circular, the first part will contain the buffer, the second will point to nullptr. std::pair get_continuous_parts(); - MemoryView m_base_buffer; + union { + MemoryView m_base_buffer; + hailo_dma_buffer_t m_dmabuf; + }; + size_t m_size; size_t m_offset; + TransferBufferType m_type; // Once map_buffer is called, a MappedBuffer object is stored here to make sure the buffer is mapped. vdma::MappedBufferPtr m_mappings; @@ -86,6 +99,16 @@ struct TransferRequest { } return total_transfer_size; } + + Expected is_request_aligned() { + CHECK(!transfer_buffers.empty(), HAILO_INVALID_ARGUMENT, "TransferRequest is empty"); + CHECK(TransferBufferType::MEMORYVIEW == transfer_buffers[0].type(), HAILO_INVALID_ARGUMENT, + "get_aligned_request is only supported in MEMORYVIEW type TransferBuffer"); + + const auto dma_able_alignment = OsUtils::get_dma_able_alignment(); + TRY(auto base_buffer, transfer_buffers[0].base_buffer()); + return (0 == reinterpret_cast(base_buffer.data()) % dma_able_alignment); + } }; struct InferRequest { diff --git a/hailort/libhailort/src/transform/transform.cpp b/hailort/libhailort/src/transform/transform.cpp index 8a82aae..87fd1f9 100644 --- a/hailort/libhailort/src/transform/transform.cpp +++ b/hailort/libhailort/src/transform/transform.cpp @@ -76,9 +76,20 @@ bool TransformContextUtils::should_transpose(const hailo_format_flags_t &src_fla bool TransformContextUtils::should_reorder(const hailo_3d_image_shape_t &src_image_shape, const hailo_format_t &src_format, const hailo_3d_image_shape_t &dst_image_shape, const hailo_format_t &dst_format) { - /* If shapes and format are different - need to use transform_context */ + /* If the orders are NHCW <-> NHWC, but C == 1, no reordering is needed */ + if ((src_image_shape.features == dst_image_shape.features) && + (src_image_shape.features == 1) && + (src_image_shape.height == dst_image_shape.height) && + (src_image_shape.width == dst_image_shape.width) && + (((src_format.order == HAILO_FORMAT_ORDER_NHCW) && (dst_format.order == HAILO_FORMAT_ORDER_NHWC)) || + ((src_format.order == HAILO_FORMAT_ORDER_NHWC) && (dst_format.order == HAILO_FORMAT_ORDER_NHCW)))) { + return false; + } + + + /* If not all shapes and formats are the same - reordering is needed */ if (!((src_image_shape.features == dst_image_shape.features) && - (src_image_shape.height == dst_image_shape.height) && + (src_image_shape.height == dst_image_shape.height) && (src_image_shape.width == dst_image_shape.width) && (src_format.order == dst_format.order))) { return true; @@ -1674,7 +1685,7 @@ Expected> InputTransformContext::create(c const auto dst_frame_size = HailoRTCommon::get_periph_frame_size(dst_image_shape, dst_format); Buffer quant_buffer; - auto should_quantize = TransformContextUtils::should_quantize(HAILO_H2D_STREAM, src_format, dst_format); + auto should_quantize = TransformContextUtils::should_quantize(HAILO_H2D_STREAM, internal_src_format, dst_format); CHECK_EXPECTED(should_quantize); if (should_quantize.value()) { auto expected_quant_buffer = Buffer::create(src_frame_size, 0); @@ -1683,7 +1694,7 @@ Expected> InputTransformContext::create(c } Buffer transpose_buffer; - bool should_transpose = TransformContextUtils::should_transpose(src_format.flags, dst_format.flags); + bool should_transpose = TransformContextUtils::should_transpose(internal_src_format.flags, dst_format.flags); if (should_transpose) { auto expected_transpose_buffer = Buffer::create(get_transpose_buffer_size(src_image_shape, dst_format.type)); @@ -1691,7 +1702,7 @@ Expected> InputTransformContext::create(c transpose_buffer = expected_transpose_buffer.release(); } - auto should_reorder = TransformContextUtils::should_reorder(src_image_shape, src_format, dst_image_shape, dst_format); + auto should_reorder = TransformContextUtils::should_reorder(src_image_shape, internal_src_format, dst_image_shape, dst_format); auto should_pad_periph = TransformContextUtils::should_pad_periph(dst_image_shape, dst_format); std::unique_ptr transform_context(new (std::nothrow) InputTransformContext(src_frame_size, src_image_shape, @@ -1938,19 +1949,19 @@ Expected> FrameOutputTransformContext::c const auto src_frame_size = HailoRTCommon::get_periph_frame_size(src_image_shape, src_format); const auto dst_frame_size = HailoRTCommon::get_frame_size(dst_image_shape, internal_dst_format); - auto should_quantize = TransformContextUtils::should_quantize(HAILO_D2H_STREAM, src_format, dst_format); + auto should_quantize = TransformContextUtils::should_quantize(HAILO_D2H_STREAM, src_format, internal_dst_format); CHECK_EXPECTED(should_quantize); Buffer transpose_buffer; - auto should_transpose = TransformContextUtils::should_transpose(src_format.flags, dst_format.flags); + auto should_transpose = TransformContextUtils::should_transpose(src_format.flags, internal_dst_format.flags); if (should_transpose) { auto expected_transpose_buffer = Buffer::create(get_transpose_buffer_size(dst_image_shape, src_format.type)); CHECK_EXPECTED(expected_transpose_buffer); transpose_buffer = expected_transpose_buffer.release(); } - auto should_reorder = TransformContextUtils::should_reorder(src_image_shape, src_format, dst_image_shape, dst_format); - auto should_pad_periph = TransformContextUtils::should_pad_periph(dst_image_shape, dst_format); + auto should_reorder = TransformContextUtils::should_reorder(src_image_shape, src_format, dst_image_shape, internal_dst_format); + auto should_pad_periph = TransformContextUtils::should_pad_periph(dst_image_shape, internal_dst_format); std::unique_ptr frame_transform_context = std::make_unique(src_frame_size, src_image_shape, src_format, dst_frame_size, dst_image_shape, internal_dst_format, dst_quant_infos, std::move(transpose_buffer), @@ -1987,7 +1998,7 @@ Expected> NMSOutputTransformContext::cre auto dst_frame_size = HailoRTCommon::get_nms_host_frame_size(nms_info, internal_dst_format); Buffer quant_buffer; - auto should_quantize = TransformContextUtils::should_quantize(HAILO_D2H_STREAM, src_format, dst_format); + auto should_quantize = TransformContextUtils::should_quantize(HAILO_D2H_STREAM, src_format, internal_dst_format); CHECK_EXPECTED(should_quantize); if (*should_quantize) { dst_frame_size = HailoRTCommon::get_nms_host_frame_size(nms_info, internal_dst_format); @@ -1996,7 +2007,7 @@ Expected> NMSOutputTransformContext::cre quant_buffer = expected_nms_quant_buffer.release(); } - auto should_transpose = TransformContextUtils::should_transpose(src_format.flags, dst_format.flags); + auto should_transpose = TransformContextUtils::should_transpose(src_format.flags, internal_dst_format.flags); std::unique_ptr nms_transform_context = std::make_unique(src_frame_size, src_format, dst_frame_size, internal_dst_format, dst_quant_infos, nms_info, std::move(quant_buffer), diff --git a/hailort/libhailort/src/utils/buffer.cpp b/hailort/libhailort/src/utils/buffer.cpp index 054c24f..88c5543 100644 --- a/hailort/libhailort/src/utils/buffer.cpp +++ b/hailort/libhailort/src/utils/buffer.cpp @@ -78,6 +78,9 @@ Buffer::Buffer(Buffer&& other) : Expected Buffer::create(size_t size, const BufferStorageParams ¶ms) { + if (0 == size) { + return Buffer(); + } auto storage = BufferStorage::create(size, params); CHECK_EXPECTED(storage); diff --git a/hailort/libhailort/src/utils/buffer_storage.cpp b/hailort/libhailort/src/utils/buffer_storage.cpp index 877ab7a..009357b 100644 --- a/hailort/libhailort/src/utils/buffer_storage.cpp +++ b/hailort/libhailort/src/utils/buffer_storage.cpp @@ -48,6 +48,10 @@ Expected BufferStorage::create(size_t size, const BufferStorag auto result = DmaStorage::create(size); CHECK_EXPECTED(result); return std::static_pointer_cast(result.release()); + } else if (0 != (params.flags & HAILO_BUFFER_FLAGS_CONTINUOUS)) { + auto result = ContinuousStorage::create(size); + CHECK_EXPECTED(result); + return std::static_pointer_cast(result.release()); } // TODO: HRT-10903 @@ -60,6 +64,11 @@ Expected BufferStorage::get_dma_able_buffer() return make_unexpected(HAILO_NOT_IMPLEMENTED); } +Expected BufferStorage::dma_address() +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + Expected HeapStorage::create(size_t size) { std::unique_ptr data(new (std::nothrow) uint8_t[size]); @@ -125,7 +134,7 @@ void *DmaStorage::user_address() Expected DmaStorage::release() noexcept { - return make_unexpected(HAILO_NOT_IMPLEMENTED); + return make_unexpected(HAILO_INVALID_OPERATION); } Expected DmaStorage::get_dma_able_buffer() @@ -133,4 +142,46 @@ Expected DmaStorage::get_dma_able_buffer() return vdma::DmaAbleBufferPtr{m_dma_able_buffer}; } +Expected ContinuousStorage::create(size_t size) +{ + TRY(auto driver, HailoRTDriver::create_integrated_nnc()); + TRY(auto continuous_buffer, vdma::ContinuousBuffer::create(size, *driver.get())); + + auto result = make_shared_nothrow(std::move(driver), std::move(continuous_buffer)); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + + return result; +} + +ContinuousStorage::ContinuousStorage(std::unique_ptr driver, vdma::ContinuousBuffer &&continuous_buffer) : + m_driver(std::move(driver)), + m_continuous_buffer(std::move(continuous_buffer)) +{} + +ContinuousStorage::ContinuousStorage(ContinuousStorage&& other) noexcept : + BufferStorage(std::move(other)), + m_driver(std::move(other.m_driver)), + m_continuous_buffer(std::move(other.m_continuous_buffer)) +{} + +size_t ContinuousStorage::size() const +{ + return m_continuous_buffer.size(); +} + +void *ContinuousStorage::user_address() +{ + return m_continuous_buffer.user_address(); +} + +Expected ContinuousStorage::dma_address() +{ + return m_continuous_buffer.dma_address(); +} + +Expected ContinuousStorage::release() noexcept +{ + return make_unexpected(HAILO_INVALID_OPERATION); +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/utils/buffer_storage.hpp b/hailort/libhailort/src/utils/buffer_storage.hpp index b277c98..c074153 100644 --- a/hailort/libhailort/src/utils/buffer_storage.hpp +++ b/hailort/libhailort/src/utils/buffer_storage.hpp @@ -15,6 +15,7 @@ #include "hailo/buffer.hpp" #include "utils/exported_resource_manager.hpp" +#include "vdma/memory/continuous_buffer.hpp" #include #include @@ -35,6 +36,7 @@ class VdmaDevice; class BufferStorage; class HeapStorage; class DmaStorage; +class ContinuousStorage; class HailoRTDriver; class Buffer; @@ -85,6 +87,7 @@ public: // Internal functions virtual Expected get_dma_able_buffer(); + virtual Expected dma_address(); BufferStorage() = default; }; @@ -144,6 +147,32 @@ private: }; +using ContinuousStoragePtr = std::shared_ptr; + +/** + * Storage class for buffer that is continuous + */ +class ContinuousStorage : public BufferStorage +{ +public: + static Expected create(size_t size); + ContinuousStorage(std::unique_ptr driver, vdma::ContinuousBuffer &&continuous_buffer); + ContinuousStorage(ContinuousStorage&& other) noexcept; + ContinuousStorage(const ContinuousStorage &) = delete; + ContinuousStorage &operator=(ContinuousStorage &&) = delete; + ContinuousStorage &operator=(const ContinuousStorage &) = delete; + virtual ~ContinuousStorage() = default; + + virtual size_t size() const override; + virtual void *user_address() override; + virtual Expected dma_address() override; + virtual Expected release() noexcept override; + +private: + std::unique_ptr m_driver; + vdma::ContinuousBuffer m_continuous_buffer; +}; + } /* namespace hailort */ #endif /* _HAILO_BUFFER_STORAGE_HPP_ */ diff --git a/hailort/libhailort/src/utils/dma_buffer_utils.hpp b/hailort/libhailort/src/utils/dma_buffer_utils.hpp index 7ed8019..8d1f830 100644 --- a/hailort/libhailort/src/utils/dma_buffer_utils.hpp +++ b/hailort/libhailort/src/utils/dma_buffer_utils.hpp @@ -13,6 +13,7 @@ #include "hailo/hailort.h" #include "hailo/expected.hpp" #include "utils/buffer_storage.hpp" +#include "net_flow/pipeline/pipeline.hpp" /** hailort namespace */ namespace hailort @@ -21,15 +22,8 @@ namespace hailort class HAILORTAPI DmaBufferUtils { public: - - static Expected mmap_dma_buffer_write(hailo_dma_buffer_t dma_buffer); - - static hailo_status munmap_dma_buffer_write(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview); - - static Expected mmap_dma_buffer_read(hailo_dma_buffer_t dma_buffer); - - static hailo_status munmap_dma_buffer_read(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview); - + static Expected mmap_dma_buffer(hailo_dma_buffer_t dma_buffer, BufferProtection dma_buffer_protection); + static hailo_status munmap_dma_buffer(hailo_dma_buffer_t dma_buffer, MemoryView dma_buffer_memview, BufferProtection dma_buffer_protection); }; } /* namespace hailort */ diff --git a/hailort/libhailort/src/utils/hailort_common.cpp b/hailort/libhailort/src/utils/hailort_common.cpp index ddca83a..be19c89 100644 --- a/hailort/libhailort/src/utils/hailort_common.cpp +++ b/hailort/libhailort/src/utils/hailort_common.cpp @@ -63,7 +63,7 @@ uint32_t HailoRTCommon::get_nms_host_frame_size(const hailo_nms_shape_t &nms_sha } } -Expected HailoRTCommon::as_hailo_pix_buffer(MemoryView &memory_view, hailo_format_order_t order) +Expected HailoRTCommon::as_hailo_pix_buffer(MemoryView memory_view, hailo_format_order_t order) { switch(order){ case HAILO_FORMAT_ORDER_NV12: @@ -108,4 +108,30 @@ Expected HailoRTCommon::as_hailo_pix_buffer(MemoryView &memo } } +bool HailoRTCommon::is_power_measurement_supported(const hailo_device_architecture_t &hw_arch) +{ + switch(hw_arch) { + case HAILO_ARCH_HAILO8: + return true; + default: + return false; + } +} + +bool HailoRTCommon::is_current_measurement_supported(const hailo_device_architecture_t &hw_arch) +{ + return is_power_measurement_supported(hw_arch); +} + +bool HailoRTCommon::is_temp_measurement_supported(const hailo_device_architecture_t &hw_arch) +{ + switch(hw_arch) { + case HAILO_ARCH_HAILO8: + case HAILO_ARCH_HAILO8L: + return true; + default: + return false; + } +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/utils/hailort_logger.cpp b/hailort/libhailort/src/utils/hailort_logger.cpp index 57fb592..39ef76f 100644 --- a/hailort/libhailort/src/utils/hailort_logger.cpp +++ b/hailort/libhailort/src/utils/hailort_logger.cpp @@ -48,11 +48,6 @@ namespace hailort #define HAILORT_LOGGER_FLUSH_EVERY_PRINT_ENV_VAR ("HAILORT_LOGGER_FLUSH_EVERY_PRINT") #define PERIODIC_FLUSH_INTERVAL_IN_SECONDS (5) -#ifdef _WIN32 -#define PATH_SEPARATOR "\\" -#else -#define PATH_SEPARATOR "/" -#endif std::string HailoRTLogger::parse_log_path(const char *log_path) { @@ -148,8 +143,11 @@ std::shared_ptr HailoRTLogger::create_file_sink(const std:: return make_shared_nothrow(); } if (!is_dir.value()) { - std::cerr << "HailoRT warning: Cannot create log file " << filename << "! Path " << dir_path << " is not a directory." << std::endl; - return make_shared_nothrow(); + auto status = Filesystem::create_directory(dir_path); + if (status != HAILO_SUCCESS) { + std::cerr << "HailoRT warning: Cannot create log file " << filename << "! Path " << dir_path << " is not valid." << std::endl; + return make_shared_nothrow(); + } } if (!Filesystem::is_path_accesible(dir_path)) { @@ -237,5 +235,18 @@ void HailoRTLogger::set_levels(spdlog::level::level_enum console_level, spdlog:: spdlog::flush_every(std::chrono::seconds(PERIODIC_FLUSH_INTERVAL_IN_SECONDS)); } +Expected HailoRTLogger::get_console_logger_level_from_string(const std::string &user_console_logger_level) +{ + static const std::unordered_map log_level_map = { + {"info", spdlog::level::info}, + {"warning", spdlog::level::warn}, + {"error", spdlog::level::err}, + {"critical", spdlog::level::critical} + }; + if(log_level_map.find(user_console_logger_level) != log_level_map.end()) { + return Expected(log_level_map.at(user_console_logger_level)); + } + return make_unexpected(HAILO_INVALID_ARGUMENT); +} } /* namespace hailort */ diff --git a/hailort/libhailort/src/utils/hailort_logger.hpp b/hailort/libhailort/src/utils/hailort_logger.hpp index 3641b87..4120f10 100644 --- a/hailort/libhailort/src/utils/hailort_logger.hpp +++ b/hailort/libhailort/src/utils/hailort_logger.hpp @@ -21,6 +21,13 @@ namespace hailort { +#define HAILORT_CONSOLE_LOGGER_LEVEL ("HAILORT_CONSOLE_LOGGER_LEVEL") + +#ifdef _WIN32 +#define PATH_SEPARATOR "\\" +#else +#define PATH_SEPARATOR "/" +#endif class HailoRTLogger { public: @@ -33,6 +40,16 @@ public: #endif { static std::unique_ptr instance = nullptr; + auto user_console_logger_level = std::getenv(HAILORT_CONSOLE_LOGGER_LEVEL); + if ((nullptr != user_console_logger_level) && (std::strlen(user_console_logger_level) > 0)){ + auto expected_console_level = get_console_logger_level_from_string(user_console_logger_level); + if (expected_console_level) { + console_level = expected_console_level.release(); + } else { + LOGGER__WARNING("Failed to parse console logger level from environment variable: {}, status: {}", + user_console_logger_level, expected_console_level.status()); + } + } if (nullptr == instance) { instance = make_unique_nothrow(console_level, file_level, flush_level); } @@ -52,6 +69,7 @@ public: private: static std::string parse_log_path(const char *log_path); void set_levels(spdlog::level::level_enum console_level, spdlog::level::level_enum file_level, spdlog::level::level_enum flush_level); + static Expected get_console_logger_level_from_string(const std::string &user_console_logger_level); std::shared_ptr m_console_sink; diff --git a/hailort/libhailort/src/utils/profiler/handler.hpp b/hailort/libhailort/src/utils/profiler/handler.hpp index 7b82ef6..4b46a42 100644 --- a/hailort/libhailort/src/utils/profiler/handler.hpp +++ b/hailort/libhailort/src/utils/profiler/handler.hpp @@ -157,13 +157,15 @@ struct FrameEnqueueD2HTrace : Trace struct ActivateCoreOpTrace : Trace { - ActivateCoreOpTrace(const device_id_t &device_id, vdevice_core_op_handle_t handle, double duration) - : Trace("activate_core_op"), device_id(device_id), core_op_handle(handle), duration(duration) + ActivateCoreOpTrace(const device_id_t &device_id, vdevice_core_op_handle_t handle, double duration, + int dynamic_batch_size) + : Trace("activate_core_op"), device_id(device_id), core_op_handle(handle), duration(duration), dynamic_batch_size(dynamic_batch_size) {} device_id_t device_id; vdevice_core_op_handle_t core_op_handle; double duration; + int dynamic_batch_size; }; // Currently, activate and switch are the same trace to make scheduler and fast-switch flow similar (although in the diff --git a/hailort/libhailort/src/utils/profiler/scheduler_profiler_handler.cpp b/hailort/libhailort/src/utils/profiler/scheduler_profiler_handler.cpp index c3c56ab..c86f6a9 100644 --- a/hailort/libhailort/src/utils/profiler/scheduler_profiler_handler.cpp +++ b/hailort/libhailort/src/utils/profiler/scheduler_profiler_handler.cpp @@ -25,17 +25,28 @@ #include #include -#define PROFILER_DEFAULT_FILE_NAME ("hailo.tracer") + #define SCHEDULER_PROFILER_NAME ("SchedulerProfiler") -#define PROFILER_FILE_ENV_VAR ("HAILO_TRACE_FILE") +#define PROFILER_FILE_ENV_VAR ("HAILO_TRACE_PATH") #define SCHEDULER_PROFILER_LOGGER_FILENAME ("scheduler_profiler.json") #define SCHEDULER_PROFILER_LOGGER_PATTERN ("%v") #define SCHEDULER_PROFILER_LOGGER_PATH ("SCHEDULER_PROFILER_LOGGER_PATH") +static const std::string PROFILER_DEFAULT_FILE_NAME_PREFIX("hailort"); +static const std::string PROFILER_DEFAULT_FILE_NAME_SUFFIX(".hrtt"); + namespace hailort { +std::string get_current_datetime() { + auto now = std::chrono::system_clock::now(); + auto tp = std::chrono::system_clock::to_time_t(now); + std::stringstream ss; + ss << std::put_time(std::localtime(&tp), "%Y-%m-%d_%H-%M-%S"); + return ss.str(); +} + SchedulerProfilerHandler::SchedulerProfilerHandler(int64_t &start_time) #ifndef __ANDROID__ : m_file_sink(HailoRTLogger::create_file_sink(HailoRTLogger::get_log_path(SCHEDULER_PROFILER_LOGGER_PATH), SCHEDULER_PROFILER_LOGGER_FILENAME, false)), @@ -63,9 +74,9 @@ SchedulerProfilerHandler::~SchedulerProfilerHandler() void SchedulerProfilerHandler::serialize_and_dump_proto() { auto file_env_var = std::getenv(PROFILER_FILE_ENV_VAR); - std::string file_name = PROFILER_DEFAULT_FILE_NAME; + std::string file_name = PROFILER_DEFAULT_FILE_NAME_PREFIX + "_" + get_current_datetime() + PROFILER_DEFAULT_FILE_NAME_SUFFIX; if (nullptr != file_env_var) { - file_name = std::string(file_env_var); + file_name = std::string(file_env_var) + PATH_SEPARATOR + file_name; } std::ofstream output_file(std::string(file_name), std::ios::out |std::ios::binary); @@ -331,6 +342,7 @@ void SchedulerProfilerHandler::handle_trace(const ActivateCoreOpTrace &trace) added_trace->mutable_activate_core_op()->set_new_core_op_handle(trace.core_op_handle); added_trace->mutable_activate_core_op()->set_time_stamp(trace.timestamp); added_trace->mutable_activate_core_op()->set_duration(trace.duration); + added_trace->mutable_activate_core_op()->set_dynamic_batch_size(trace.dynamic_batch_size); } void SchedulerProfilerHandler::handle_trace(const DeactivateCoreOpTrace &trace) diff --git a/hailort/libhailort/src/utils/thread_safe_queue.hpp b/hailort/libhailort/src/utils/thread_safe_queue.hpp index cd24448..2c3c906 100644 --- a/hailort/libhailort/src/utils/thread_safe_queue.hpp +++ b/hailort/libhailort/src/utils/thread_safe_queue.hpp @@ -287,6 +287,11 @@ public: return m_inner.size_approx(); } + size_t max_capacity() + { + return m_inner.max_capacity(); + } + bool is_queue_full() { return (m_inner.size_approx() == m_inner.max_capacity()); diff --git a/hailort/libhailort/src/vdevice/CMakeLists.txt b/hailort/libhailort/src/vdevice/CMakeLists.txt index c3c6cb7..1a0179f 100644 --- a/hailort/libhailort/src/vdevice/CMakeLists.txt +++ b/hailort/libhailort/src/vdevice/CMakeLists.txt @@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/vdevice.cpp ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_core_op.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_hrpc_client.cpp ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_native_stream.cpp ${CMAKE_CURRENT_SOURCE_DIR}/callback_reorder_queue.cpp @@ -14,4 +15,6 @@ set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/scheduler/infer_request_accumulator.cpp ) +set(SRC_FILES ${SRC_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_hrpc_client.cpp) + set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.cpp b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.cpp index 66a35ee..7e157cf 100644 --- a/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.cpp +++ b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.cpp @@ -114,11 +114,9 @@ std::shared_ptr ScheduledCoreOp::get_core_op() return m_core_op; } -std::shared_ptr ScheduledCoreOp::get_vdma_core_op(const device_id_t &device_id) +Expected> ScheduledCoreOp::get_vdma_core_op(const device_id_t &device_id) { - auto vdma_core_op = m_core_op->get_core_op_by_device_id(device_id); - assert(vdma_core_op); - return vdma_core_op.release(); + return m_core_op->get_core_op_by_device_id(device_id); } std::chrono::time_point ScheduledCoreOp::get_last_run_timestamp() diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.hpp b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.hpp index 08b9991..b9711e5 100644 --- a/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.hpp +++ b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.hpp @@ -48,7 +48,7 @@ public: std::shared_ptr get_core_op(); - std::shared_ptr get_vdma_core_op(const device_id_t &device_id); + Expected> get_vdma_core_op(const device_id_t &device_id); uint32_t get_max_ongoing_frames_per_device() const; diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduler.cpp b/hailort/libhailort/src/vdevice/scheduler/scheduler.cpp index 060d3b2..ad987ac 100644 --- a/hailort/libhailort/src/vdevice/scheduler/scheduler.cpp +++ b/hailort/libhailort/src/vdevice/scheduler/scheduler.cpp @@ -117,11 +117,11 @@ hailo_status CoreOpsScheduler::switch_core_op(const scheduler_core_op_handle_t & curr_device_info->current_batch_size = hw_batch_size; if ((core_op_handle != curr_device_info->current_core_op_handle) || (!has_same_hw_batch_size_as_previous)) { - auto next_core_op = get_vdma_core_op(core_op_handle, device_id); + TRY(auto next_core_op, get_vdma_core_op(core_op_handle, device_id)); std::shared_ptr current_core_op = nullptr; if (curr_device_info->current_core_op_handle != INVALID_CORE_OP_HANDLE) { - current_core_op = get_vdma_core_op(curr_device_info->current_core_op_handle, device_id); + TRY(current_core_op, get_vdma_core_op(curr_device_info->current_core_op_handle, device_id)); } auto status = VdmaConfigManager::set_core_op(device_id, current_core_op, next_core_op, hw_batch_size); @@ -144,7 +144,7 @@ hailo_status CoreOpsScheduler::deactivate_core_op(const device_id_t &device_id) return HAILO_SUCCESS; } - auto vdma_core_op = get_vdma_core_op(core_op_handle, device_id); + TRY (auto vdma_core_op, get_vdma_core_op(core_op_handle, device_id)); auto status = VdmaConfigManager::deactivate_core_op(vdma_core_op); CHECK_SUCCESS(status, "Scheduler failed deactivate core op on {}", device_id); @@ -180,7 +180,7 @@ hailo_status CoreOpsScheduler::infer_async(const scheduler_core_op_handle_t &cor auto current_device_info = m_devices[device_id]; assert(core_op_handle == current_device_info->current_core_op_handle); auto scheduled_core_op = m_scheduled_core_ops.at(core_op_handle); - auto vdma_core_op = get_vdma_core_op(core_op_handle, device_id); + TRY (auto vdma_core_op, get_vdma_core_op(core_op_handle, device_id)); auto infer_request = dequeue_infer_request(core_op_handle); CHECK_EXPECTED_AS_STATUS(infer_request); @@ -336,7 +336,7 @@ uint16_t CoreOpsScheduler::get_frames_ready_to_transfer(scheduler_core_op_handle return static_cast(std::min(requested_frames, max_ongoing_frames - ongoing_frames)); } -std::shared_ptr CoreOpsScheduler::get_vdma_core_op(scheduler_core_op_handle_t core_op_handle, +Expected> CoreOpsScheduler::get_vdma_core_op(scheduler_core_op_handle_t core_op_handle, const device_id_t &device_id) { return m_scheduled_core_ops.at(core_op_handle)->get_vdma_core_op(device_id); diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduler.hpp b/hailort/libhailort/src/vdevice/scheduler/scheduler.hpp index 3d25220..6d3c5e8 100644 --- a/hailort/libhailort/src/vdevice/scheduler/scheduler.hpp +++ b/hailort/libhailort/src/vdevice/scheduler/scheduler.hpp @@ -81,7 +81,7 @@ private: Expected dequeue_infer_request(scheduler_core_op_handle_t core_op_handle); uint16_t get_frames_ready_to_transfer(scheduler_core_op_handle_t core_op_handle, const device_id_t &device_id) const; - std::shared_ptr get_vdma_core_op(scheduler_core_op_handle_t core_op_handle, + Expected> get_vdma_core_op(scheduler_core_op_handle_t core_op_handle, const device_id_t &device_id); void shutdown_core_op(scheduler_core_op_handle_t core_op_handle); diff --git a/hailort/libhailort/src/vdevice/vdevice.cpp b/hailort/libhailort/src/vdevice/vdevice.cpp index c51db77..c3d0763 100644 --- a/hailort/libhailort/src/vdevice/vdevice.cpp +++ b/hailort/libhailort/src/vdevice/vdevice.cpp @@ -17,6 +17,7 @@ #include "vdevice/vdevice_internal.hpp" #include "vdevice/vdevice_core_op.hpp" +#include "vdevice/vdevice_hrpc_client.hpp" #include "vdma/pcie/pcie_device.hpp" #include "vdma/integrated/integrated_device.hpp" @@ -31,6 +32,9 @@ #include "rpc/rpc_definitions.hpp" #endif // HAILO_SUPPORT_MULTI_PROCESS +#define HAILO_FORCE_HRPC_CLIENT_ENV_VAR "HAILO_FORCE_HRPC" +#define HAILO_FORCE_HRPC_CLIENT_ON "1" + namespace hailort { @@ -212,6 +216,24 @@ hailo_status VDeviceHandle::dma_unmap(void *address, size_t size, hailo_dma_buff return vdevice.value()->dma_unmap(address, size, direction); } +hailo_status VDeviceHandle::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + auto &manager = SharedResourceManager::get_instance(); + auto vdevice = manager.resource_lookup(m_handle); + CHECK_EXPECTED_AS_STATUS(vdevice); + + return vdevice.value()->dma_map_dmabuf(dmabuf_fd, size, direction); +} + +hailo_status VDeviceHandle::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + auto &manager = SharedResourceManager::get_instance(); + auto vdevice = manager.resource_lookup(m_handle); + CHECK_EXPECTED_AS_STATUS(vdevice); + + return vdevice.value()->dma_unmap_dmabuf(dmabuf_fd, size, direction); +} + bool VDevice::service_over_ip_mode() { #ifdef HAILO_SUPPORT_MULTI_PROCESS @@ -221,6 +243,13 @@ bool VDevice::service_over_ip_mode() return false; // no service -> no service over ip } +bool VDevice::force_hrpc_client() +{ + // The env var HAILO_FORCE_HRPC_CLIENT_ENV_VAR is supported for debug purposes + char *pcie_service_var = std::getenv(HAILO_FORCE_HRPC_CLIENT_ENV_VAR); // TODO: Remove duplication + return (nullptr != pcie_service_var) && (HAILO_FORCE_HRPC_CLIENT_ON == std::string(pcie_service_var)); +} + #ifdef HAILO_SUPPORT_MULTI_PROCESS VDeviceClient::VDeviceClient(std::unique_ptr client, VDeviceIdentifier &&identifier, @@ -453,7 +482,7 @@ hailo_status VDeviceClient::dma_map(void *address, size_t size, hailo_dma_buffer (void) size; (void) direction; // It is ok to do nothing on service, because the buffer is copied anyway to the service. - LOGGER__TRACE("VDevice `dma_map()` is doing nothing on service"); + LOGGER__TRACE("VDevice `dma_map()` does nothing in service"); return HAILO_SUCCESS; } @@ -463,7 +492,27 @@ hailo_status VDeviceClient::dma_unmap(void *address, size_t size, hailo_dma_buff (void) size; (void) direction; // It is ok to do nothing on service, because the buffer is copied anyway to the service. - LOGGER__TRACE("VDevice `dma_map()` is doing nothing on service"); + LOGGER__TRACE("VDevice `dma_map()` does nothing in service"); + return HAILO_SUCCESS; +} + +hailo_status VDeviceClient::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + (void) dmabuf_fd; + (void) size; + (void) direction; + // It is ok to do nothing on service, because the buffer is copied anyway to the service. + LOGGER__TRACE("VDevice `dma_map_dmabuf()` does nothing in service"); + return HAILO_SUCCESS; +} + +hailo_status VDeviceClient::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + (void) dmabuf_fd; + (void) size; + (void) direction; + // It is ok to do nothing on service, because the buffer is copied anyway to the service. + LOGGER__TRACE("VDevice `dma_unmap_dmabuf()` does nothing in service"); return HAILO_SUCCESS; } @@ -475,7 +524,7 @@ Expected> VDevice::create(const hailo_vdevice_params_t auto status = VDeviceBase::validate_params(params); CHECK_SUCCESS_AS_EXPECTED(status); - std::unique_ptr vdevice; + std::unique_ptr vdevice = nullptr; if (params.multi_process_service) { #ifdef HAILO_SUPPORT_MULTI_PROCESS @@ -489,9 +538,22 @@ Expected> VDevice::create(const hailo_vdevice_params_t return make_unexpected(HAILO_INVALID_OPERATION); #endif // HAILO_SUPPORT_MULTI_PROCESS } else { - auto expected_vdevice = VDeviceHandle::create(params); - CHECK_EXPECTED(expected_vdevice); - vdevice = expected_vdevice.release(); + auto acc_type = HailoRTDriver::AcceleratorType::ACC_TYPE_MAX_VALUE; + if (nullptr != params.device_ids) { + TRY(auto device_ids_contains_eth, VDeviceBase::device_ids_contains_eth(params)); + if (!device_ids_contains_eth) { + TRY(acc_type, VDeviceBase::get_accelerator_type(params.device_ids, params.device_count)); + } + } else { + TRY(acc_type, VDeviceBase::get_accelerator_type(params.device_ids, params.device_count)); + } + if ((acc_type == HailoRTDriver::AcceleratorType::SOC_ACCELERATOR) || force_hrpc_client()) { + // Creating VDeviceClient + TRY(vdevice, VDeviceHrpcClient::create(params)); + } else { + // Creating VDeviceHandle + TRY(vdevice, VDeviceHandle::create(params)); + } } // Upcasting to VDevice unique_ptr auto vdevice_ptr = std::unique_ptr(vdevice.release()); @@ -517,21 +579,45 @@ Expected> VDevice::create(const std::vector VDeviceBase::get_accelerator_type(hailo_device_id_t *device_ids, size_t device_count) +{ + auto acc_type = HailoRTDriver::AcceleratorType::ACC_TYPE_MAX_VALUE; + TRY(auto device_infos, HailoRTDriver::scan_devices()); + if (nullptr != device_ids) { + // device_ids are provided - check that all ids are of the same type + that the id exists in the scan from device_infos + for (uint32_t i = 0; i < device_count; i++) { + const auto &id = device_ids[i].id; + auto device_info = std::find_if(device_infos.begin(), device_infos.end(), [&](const auto &device_info) { + return Device::device_ids_equal(device_info.device_id, id); + }); + CHECK(device_info != device_infos.end(), HAILO_INVALID_ARGUMENT, + "VDevice creation failed. device_id {} not found", id); + CHECK(acc_type == HailoRTDriver::AcceleratorType::ACC_TYPE_MAX_VALUE || acc_type == device_info->accelerator_type, HAILO_INVALID_ARGUMENT, + "VDevice creation failed. device_ids of devices with different types are provided (e.g. Hailo8 and Hailo10). Please provide device_ids of the same device types"); + acc_type = device_info->accelerator_type; + } + } else { + // No device_id is provided - check that all devices are of the same type + for (const auto &device_info : device_infos) { + CHECK(acc_type == HailoRTDriver::AcceleratorType::ACC_TYPE_MAX_VALUE || acc_type == device_info.accelerator_type, HAILO_INVALID_ARGUMENT, + "VDevice creation failed. Devices of different types are found and no device_id is provided. Please provide device_ids"); + acc_type = device_info.accelerator_type; + } + } + return acc_type; +} + hailo_status VDeviceBase::validate_params(const hailo_vdevice_params_t ¶ms) { CHECK(0 != params.device_count, HAILO_INVALID_ARGUMENT, "VDevice creation failed. invalid device_count ({}).", params.device_count); - if (params.device_ids != nullptr) { - for (uint32_t i = 0; i < params.device_count; i++) { - auto dev_type = Device::get_device_type(params.device_ids[i].id); - CHECK_EXPECTED_AS_STATUS(dev_type); - CHECK((Device::Type::ETH != dev_type.value() || (1 == params.device_count)), HAILO_INVALID_ARGUMENT, - "VDevice over ETH is supported for 1 device. Passed device_count: {}", params.device_count); - CHECK((Device::Type::ETH != dev_type.value() || (HAILO_SCHEDULING_ALGORITHM_NONE == params.scheduling_algorithm)), HAILO_INVALID_ARGUMENT, - "VDevice over ETH is not supported when scheduler is enabled."); - } - } + TRY(auto device_ids_contains_eth, device_ids_contains_eth(params)); + CHECK(!(device_ids_contains_eth && (1 != params.device_count)), HAILO_INVALID_ARGUMENT, + "VDevice over ETH is supported for 1 device. Passed device_count: {}", params.device_count); + CHECK(!(device_ids_contains_eth && (HAILO_SCHEDULING_ALGORITHM_NONE != params.scheduling_algorithm)), HAILO_INVALID_ARGUMENT, + "VDevice over ETH is not supported when scheduler is enabled."); + return HAILO_SUCCESS; } @@ -670,40 +756,15 @@ Expected VDeviceBase::configure(Hef &hef, Expected> VDevice::create_infer_model(const std::string &hef_path, const std::string &network_name) { CHECK_AS_EXPECTED(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Passing network name is not supported yet!"); + TRY(auto infer_model_base, InferModelBase::create(*this, hef_path)); + return std::shared_ptr(std::move(infer_model_base)); +} - auto hef_expected = Hef::create(hef_path); - CHECK_EXPECTED(hef_expected); - auto hef = hef_expected.release(); - - std::unordered_map inputs; - std::unordered_map outputs; - - auto input_vstream_infos = hef.get_input_vstream_infos(); - CHECK_EXPECTED(input_vstream_infos); - - for (const auto &vstream_info : input_vstream_infos.value()) { - auto pimpl = make_shared_nothrow(vstream_info); - CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); - - InferModel::InferStream stream(pimpl); - inputs.emplace(vstream_info.name, std::move(stream)); - } - - auto output_vstream_infos = hef.get_output_vstream_infos(); - CHECK_EXPECTED(output_vstream_infos); - - for (const auto &vstream_info : output_vstream_infos.value()) { - auto pimpl = make_shared_nothrow(vstream_info); - CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); - - InferModel::InferStream stream(pimpl); - outputs.emplace(vstream_info.name, std::move(stream)); - } - - auto res = make_shared_nothrow(InferModel(*this, std::move(hef), std::move(inputs), std::move(outputs))); - CHECK_NOT_NULL_AS_EXPECTED(res, HAILO_OUT_OF_HOST_MEMORY); - - return res; +Expected> VDevice::create_infer_model(const MemoryView hef_buffer, const std::string &network_name) +{ + CHECK_AS_EXPECTED(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Passing network name is not supported yet!"); + TRY(auto infer_model_base, InferModelBase::create(*this, hef_buffer)); + return std::shared_ptr(std::move(infer_model_base)); } Expected VDeviceBase::get_default_streams_interface() const @@ -744,6 +805,8 @@ Expected>> VDeviceBase::create_dev "VDevice with multiple devices is not supported on HAILO_ARCH_HAILO8L. device {} is HAILO_ARCH_HAILO8L", device_id); CHECK_AS_EXPECTED(HAILO_ARCH_HAILO15M != device_arch.value(), HAILO_INVALID_OPERATION, "VDevice with multiple devices is not supported on HAILO_ARCH_HAILO15M. device {} is HAILO_ARCH_HAILO15M", device_id); + CHECK_AS_EXPECTED(HAILO_ARCH_HAILO10H != device_arch.value(), HAILO_INVALID_OPERATION, + "VDevice with multiple devices is not supported on HAILO_ARCH_HAILO10H. device {} is HAILO_ARCH_HAILO10H", device_id); } auto dev_type = Device::get_device_type(device_id); @@ -890,4 +953,17 @@ bool VDeviceBase::should_use_multiplexer() return (!disabled_by_flag && m_core_ops_scheduler); } +Expected VDeviceBase::device_ids_contains_eth(const hailo_vdevice_params_t ¶ms) +{ + if (params.device_ids != nullptr) { + for (uint32_t i = 0; i < params.device_count; i++) { + TRY(auto dev_type, Device::get_device_type(params.device_ids[i].id)); + if (Device::Type::ETH == dev_type) { + return true; + } + } + } + return false; // in case no device_ids were provided, we assume there's no ETH device +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdevice/vdevice_core_op.cpp b/hailort/libhailort/src/vdevice/vdevice_core_op.cpp index d0ca583..ac8e83d 100644 --- a/hailort/libhailort/src/vdevice/vdevice_core_op.cpp +++ b/hailort/libhailort/src/vdevice/vdevice_core_op.cpp @@ -93,7 +93,7 @@ VDeviceCoreOp::VDeviceCoreOp(VDevice &vdevice, CoreOpsSchedulerWeakPtr core_ops_scheduler, vdevice_core_op_handle_t core_op_handle, const std::string &hef_hash, size_t max_queue_size, hailo_status &status) : - CoreOp(configure_params, core_ops.begin()->second->m_metadata, active_core_op_holder, status), + CoreOp(configure_params, core_ops.begin()->second->m_metadata, active_core_op_holder, status, !core_ops_scheduler.expired()), m_vdevice(vdevice), m_core_ops(std::move(core_ops)), m_core_ops_scheduler(core_ops_scheduler), @@ -129,6 +129,26 @@ VDeviceCoreOp::VDeviceCoreOp(VDevice &vdevice, m_infer_requests_accumulator = infer_request_accumulator; } + + if (has_caches() && is_scheduled()) { + // TODO: caches only work with a batch size of one currently (HRT-13628) + if (is_default_batch_size()) { + // Default batch size with the sched leads to dynamic batches with the max size supported by the streams + LOGGER__ERROR("Caches are not supported with the scheduler when using the default batch size"); + status = HAILO_INVALID_OPERATION; + return; + } + + const auto configured_batch_size = get_smallest_configured_batch_size(configure_params); + if (configured_batch_size > 1) { + LOGGER__ERROR("Caches are not supported with the scheduler when using a batch size greater than 1 (received {})", + configured_batch_size); + status = HAILO_INVALID_OPERATION; + return; + } + } + + status = HAILO_SUCCESS; } Expected VDeviceCoreOp::get_default_streams_interface() @@ -438,6 +458,66 @@ Expected VDeviceCoreOp::get_intermediate_buffer(const IntermediateBuffer return m_core_ops.begin()->second->get_intermediate_buffer(key); } +Expected VDeviceCoreOp::get_cache_buffer(uint32_t cache_id) +{ + CHECK_AS_EXPECTED(1 == m_core_ops.size(), HAILO_INVALID_OPERATION, + "get_cache_buffer function is not supported on more than 1 physical device."); + return m_core_ops.begin()->second->get_cache_buffer(cache_id); +} + +Expected> VDeviceCoreOp::get_cache_buffers() +{ + CHECK_AS_EXPECTED(1 == m_core_ops.size(), HAILO_INVALID_OPERATION, + "get_cache_buffers function is not supported on more than 1 physical device."); + return m_core_ops.begin()->second->get_cache_buffers(); +} + +Expected VDeviceCoreOp::get_cache_read_size() const +{ + CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION, + "get_cache_read_size function is not supported on more than 1 physical device."); + return m_core_ops.begin()->second->get_cache_read_size(); +} + +Expected VDeviceCoreOp::get_cache_write_size() const +{ + CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION, + "get_cache_write_size function is not supported on more than 1 physical device."); + return m_core_ops.begin()->second->get_cache_write_size(); +} + +bool VDeviceCoreOp::has_caches() const +{ + for (const auto &core_op : m_core_ops) { + if (core_op.second->has_caches()) { + return true; + } + } + + return false; +} + +hailo_status VDeviceCoreOp::init_cache(uint32_t read_offset, int32_t write_offset_delta) +{ + CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION, + "init_cache function is not supported on more than 1 physical device."); + return m_core_ops.begin()->second->init_cache(read_offset, write_offset_delta); +} + +Expected VDeviceCoreOp::get_cache_info() const +{ + CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION, + "get_cache_info function is not supported on more than 1 physical device."); + return m_core_ops.begin()->second->get_cache_info(); +} + +hailo_status VDeviceCoreOp::update_cache_offset(int32_t offset_delta_bytes) +{ + CHECK(1 == m_core_ops.size(), HAILO_INVALID_OPERATION, + "update_cache_offset function is not supported on more than 1 physical device."); + return m_core_ops.begin()->second->update_cache_offset(offset_delta_bytes); +} + hailo_status VDeviceCoreOp::add_to_trace() { const auto batch_size = get_stream_batch_size(m_config_params.stream_params_by_name.begin()->first); diff --git a/hailort/libhailort/src/vdevice/vdevice_core_op.hpp b/hailort/libhailort/src/vdevice/vdevice_core_op.hpp index 9482880..f057d47 100644 --- a/hailort/libhailort/src/vdevice/vdevice_core_op.hpp +++ b/hailort/libhailort/src/vdevice/vdevice_core_op.hpp @@ -92,6 +92,14 @@ public: virtual Expected run_hw_infer_estimator() override; virtual Expected get_intermediate_buffer(const IntermediateBufferKey &) override; + virtual Expected get_cache_buffer(uint32_t cache_id) override; + virtual Expected> get_cache_buffers() override; + virtual bool has_caches() const override; + virtual Expected get_cache_read_size() const override; + virtual Expected get_cache_write_size() const override; + virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override; + virtual Expected get_cache_info() const; + virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override; VDeviceCoreOp(VDevice &vdevice, ActiveCoreOpHolder &active_core_op_holder, diff --git a/hailort/libhailort/src/vdevice/vdevice_hrpc_client.cpp b/hailort/libhailort/src/vdevice/vdevice_hrpc_client.cpp new file mode 100644 index 0000000..628a7ad --- /dev/null +++ b/hailort/libhailort/src/vdevice/vdevice_hrpc_client.cpp @@ -0,0 +1,155 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file vdevice_hrpc_client.cpp + * @brief VDevice HRPC client implementation + **/ + +#include "vdevice_hrpc_client.hpp" +#include "hailo/hailort.h" +#include "hrpc_protocol/serializer.hpp" +#include "net_flow/pipeline/infer_model_hrpc_client.hpp" + +namespace hailort +{ + +Expected> VDeviceHrpcClient::create(const hailo_vdevice_params_t ¶ms) +{ + CHECK_AS_EXPECTED(params.device_count == 1, HAILO_OUT_OF_PHYSICAL_DEVICES, "Only single device is supported!"); + + auto client = make_shared_nothrow(); + CHECK_NOT_NULL(client, HAILO_INTERNAL_FAILURE); + + auto status = client->connect(); + CHECK_SUCCESS_AS_EXPECTED(status, "Failed to connect to server"); + + TRY(auto request, CreateVDeviceSerializer::serialize_request(params)); + TRY(auto result, client->execute_request(HailoRpcActionID::VDEVICE__CREATE, MemoryView(request))); + TRY(auto tuple, CreateVDeviceSerializer::deserialize_reply(MemoryView(result))); + status = std::get<0>(tuple); + CHECK_SUCCESS_AS_EXPECTED(status); + + auto vdevice_handle = std::get<1>(tuple); + auto vdevice_client = make_unique_nothrow(std::move(client), vdevice_handle); + CHECK_NOT_NULL(vdevice_client, HAILO_OUT_OF_HOST_MEMORY); + + return std::unique_ptr(std::move(vdevice_client)); +} + +VDeviceHrpcClient::~VDeviceHrpcClient() +{ + if (INVALID_HANDLE_ID == m_handle) { + return; + } + + auto request = DestroyVDeviceSerializer::serialize_request(m_handle); + if (!request) { + LOGGER__CRITICAL("Failed to serialize VDevice_release request"); + return; + } + + auto result = m_client->execute_request(HailoRpcActionID::VDEVICE__DESTROY, MemoryView(*request)); + if (!result) { + LOGGER__CRITICAL("Failed to destroy VDevice! status = {}", result.status()); + } + + if (HAILO_SUCCESS != DestroyVDeviceSerializer::deserialize_reply(MemoryView(*result))) { + LOGGER__CRITICAL("Failed to destroy VDevice! status = {}", result.status()); + } +} + +Expected> VDeviceHrpcClient::create_infer_model(const std::string &hef_path, const std::string &network_name) +{ + CHECK_AS_EXPECTED(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Passing network name is not supported yet!"); + + FileReader hef_reader(hef_path); + auto status = hef_reader.open(); + CHECK_SUCCESS(status); + + TRY(auto hef_size, hef_reader.get_size()); + TRY(auto hef_buffer, Buffer::create(hef_size)); + status = hef_reader.read(hef_buffer.data(), hef_size); + CHECK_SUCCESS(status); + + status = hef_reader.close(); + CHECK_SUCCESS(status); + + TRY(auto request, CreateInferModelSerializer::serialize_request(m_handle, hef_size)); + TRY(auto result, m_client->execute_request(HailoRpcActionID::VDEVICE__CREATE_INFER_MODEL, + MemoryView(request), [&hef_buffer] (hrpc::RpcConnection connection) -> hailo_status { + // TODO: change write to accept uint64_t, or accept file stream instead or write in chunks + auto status = connection.write_buffer(MemoryView(hef_buffer)); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; + })); + TRY(auto tuple, CreateInferModelSerializer::deserialize_reply(MemoryView(result))); + + CHECK_SUCCESS_AS_EXPECTED(std::get<0>(tuple)); + auto infer_model_handle = std::get<1>(tuple); + + TRY(auto hef, Hef::create(MemoryView(hef_buffer))); + TRY(auto infer_model, InferModelHrpcClient::create(std::move(hef), m_client, infer_model_handle, m_handle, *this)); + + return std::shared_ptr(std::move(infer_model)); +} + +Expected VDeviceHrpcClient::configure(Hef &hef, const NetworkGroupsParamsMap &configure_params) +{ + (void)m_handle; + (void)hef; + (void)configure_params; + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +Expected>> VDeviceHrpcClient::get_physical_devices() const +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +Expected> VDeviceHrpcClient::get_physical_devices_ids() const +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +// Currently only homogeneous vDevice is allow (= all devices are from the same type) +Expected VDeviceHrpcClient::get_default_streams_interface() const +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +hailo_status VDeviceHrpcClient::dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) +{ + (void)address; + (void)size; + (void)direction; + return HAILO_SUCCESS; // TODO: implement this (HRT-13689) +} + +hailo_status VDeviceHrpcClient::dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) +{ + (void)address; + (void)size; + (void)direction; + return HAILO_SUCCESS; // TODO: implement this (HRT-13689) +} + +hailo_status VDeviceHrpcClient::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + (void)dmabuf_fd; + (void)size; + (void)direction; + return HAILO_SUCCESS; // TODO: implement this (HRT-13689) +} + +hailo_status VDeviceHrpcClient::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) +{ + (void)dmabuf_fd; + (void)size; + (void)direction; + return HAILO_SUCCESS; // TODO: implement this (HRT-13689) +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdevice/vdevice_hrpc_client.hpp b/hailort/libhailort/src/vdevice/vdevice_hrpc_client.hpp new file mode 100644 index 0000000..bcccdd6 --- /dev/null +++ b/hailort/libhailort/src/vdevice/vdevice_hrpc_client.hpp @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file vdevice_hrpc_client.hpp + * @brief VDevice HRPC client, represents the user's handle to the VDevice object (held in the hailort server) + **/ + +#ifndef _HAILO_VDEVICE_HRPC_CLIENT_HPP_ +#define _HAILO_VDEVICE_HRPC_CLIENT_HPP_ + +#include "hailo/hailort.h" +#include "hrpc/client.hpp" +#include "vdevice/vdevice_internal.hpp" + +namespace hailort +{ + +class VDeviceHrpcClient : public VDevice +{ +public: + static Expected> create(const hailo_vdevice_params_t ¶ms); + + VDeviceHrpcClient(std::shared_ptr client, uint32_t handle) + : m_client(client), m_handle(handle) {} + + VDeviceHrpcClient(VDeviceHrpcClient &&) = delete; + VDeviceHrpcClient(const VDeviceHrpcClient &) = delete; + VDeviceHrpcClient &operator=(VDeviceHrpcClient &&) = delete; + VDeviceHrpcClient &operator=(const VDeviceHrpcClient &) = delete; + virtual ~VDeviceHrpcClient(); + + virtual Expected> create_infer_model(const std::string &hef_path, + const std::string &network_name = "") override; + virtual Expected configure(Hef &hef, const NetworkGroupsParamsMap &configure_params={}) override; + virtual Expected>> get_physical_devices() const override; + virtual Expected> get_physical_devices_ids() const override; + virtual Expected get_default_streams_interface() const override; + virtual hailo_status dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override; + +private: + std::shared_ptr m_client; + uint32_t m_handle; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_VDEVICE_HRPC_CLIENT_HPP_ */ diff --git a/hailort/libhailort/src/vdevice/vdevice_internal.hpp b/hailort/libhailort/src/vdevice/vdevice_internal.hpp index 5920b90..3c272d6 100644 --- a/hailort/libhailort/src/vdevice/vdevice_internal.hpp +++ b/hailort/libhailort/src/vdevice/vdevice_internal.hpp @@ -110,7 +110,37 @@ public: return status; } + virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override + { + for (const auto &pair : m_devices) { + auto &device = pair.second; + const auto status = device->dma_map_dmabuf(dmabuf_fd, size, direction); + CHECK_SUCCESS(status); + } + return HAILO_SUCCESS; + } + + virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override + { + hailo_status status = HAILO_SUCCESS; + for (const auto &pair : m_devices) { + auto &device = pair.second; + // Best effort, propagate first error + const auto unmap_status = device->dma_unmap_dmabuf(dmabuf_fd, size, direction); + if (HAILO_SUCCESS != unmap_status) { + LOGGER__ERROR("Failed unmapping dmabuf {} with status {}", dmabuf_fd, unmap_status); + if (HAILO_SUCCESS == status) { + status = unmap_status; + } + } + } + + return status; + } + + static Expected get_accelerator_type(hailo_device_id_t *device_ids, size_t device_count); static hailo_status validate_params(const hailo_vdevice_params_t ¶ms); + static Expected device_ids_contains_eth(const hailo_vdevice_params_t ¶ms); private: VDeviceBase(std::map> &&devices, CoreOpsSchedulerPtr core_ops_scheduler, @@ -165,6 +195,8 @@ public: virtual hailo_status after_fork_in_child() override; virtual hailo_status dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) override; virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override; private: VDeviceClient(std::unique_ptr client, VDeviceIdentifier &&identifier, std::vector> &&devices); @@ -209,6 +241,8 @@ public: const std::string &network_name = "") override; virtual hailo_status dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) override; virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override; private: VDeviceHandle(uint32_t handle); diff --git a/hailort/libhailort/src/vdma/CMakeLists.txt b/hailort/libhailort/src/vdma/CMakeLists.txt index 5641ff6..e2be112 100644 --- a/hailort/libhailort/src/vdma/CMakeLists.txt +++ b/hailort/libhailort/src/vdma/CMakeLists.txt @@ -11,6 +11,7 @@ elseif(UNIX) else() message(FATAL_ERROR "Unexpeced platform target, stopping build") endif() +set(DRIVER_OS_DIR ${DRIVER_OS_DIR} PARENT_SCOPE) set(DRIVER_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/driver/hailort_driver.cpp @@ -24,12 +25,15 @@ set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/vdma_stream.cpp ${CMAKE_CURRENT_SOURCE_DIR}/circular_stream_buffer_pool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pcie_session.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/dma_mapped_buffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pcie/pcie_device.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integrated/integrated_device.cpp ${CMAKE_CURRENT_SOURCE_DIR}/channel/boundary_channel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/channel/channels_group.cpp ${CMAKE_CURRENT_SOURCE_DIR}/channel/interrupts_dispatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/channel/transfer_launcher.cpp diff --git a/hailort/libhailort/src/vdma/channel/boundary_channel.cpp b/hailort/libhailort/src/vdma/channel/boundary_channel.cpp index 8c33105..5cdb82a 100644 --- a/hailort/libhailort/src/vdma/channel/boundary_channel.cpp +++ b/hailort/libhailort/src/vdma/channel/boundary_channel.cpp @@ -22,32 +22,38 @@ namespace hailort { namespace vdma { - - -Expected BoundaryChannel::create(vdma::ChannelId channel_id, Direction direction, - HailoRTDriver &driver, uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name, - LatencyMeterPtr latency_meter) +Expected BoundaryChannel::create(HailoRTDriver &driver, vdma::ChannelId channel_id, + Direction direction, vdma::DescriptorList &&desc_list, TransferLauncher &transfer_launcher, + size_t ongoing_transfers, size_t pending_transfers, const std::string &stream_name, LatencyMeterPtr latency_meter) { hailo_status status = HAILO_UNINITIALIZED; - auto channel_ptr = make_shared_nothrow(channel_id, direction, driver, descs_count, - desc_page_size, stream_name, latency_meter, status); + auto channel_ptr = make_shared_nothrow(driver, channel_id, direction, std::move(desc_list), + transfer_launcher, ongoing_transfers, pending_transfers, stream_name, latency_meter, status); CHECK_NOT_NULL_AS_EXPECTED(channel_ptr, HAILO_OUT_OF_HOST_MEMORY); CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating BoundaryChannel"); return channel_ptr; } -BoundaryChannel::BoundaryChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, - uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name, - LatencyMeterPtr latency_meter, hailo_status &status) : +BoundaryChannel::BoundaryChannel(HailoRTDriver &driver, vdma::ChannelId channel_id, Direction direction, + DescriptorList &&desc_list, TransferLauncher &transfer_launcher, + size_t ongoing_transfers_queue_size, size_t pending_transfers_queue_size, + const std::string &stream_name, LatencyMeterPtr latency_meter, hailo_status &status) : m_channel_id(channel_id), m_direction(direction), m_driver(driver), - m_desc_list(nullptr), + m_transfer_launcher(transfer_launcher), + m_desc_list(std::move(desc_list)), m_stream_name(stream_name), + m_descs(m_desc_list.count()), m_is_channel_activated(false), - m_ongoing_transfers((latency_meter != nullptr) ? ONGOING_TRANSFERS_SIZE/2 : ONGOING_TRANSFERS_SIZE), + m_channel_mutex(), + // CircularArrays with storage_size x can store x-1 elements, hence the +1 + m_ongoing_transfers(ongoing_transfers_queue_size + 1), + m_pending_transfers(pending_transfers_queue_size + 1), m_latency_meter(latency_meter), - m_pending_latency_measurements(ONGOING_TRANSFERS_SIZE) // Make sure there will always be place for latency measure + m_pending_latency_measurements(ONGOING_TRANSFERS_SIZE), // Make sure there will always be place for latency measure + m_last_timestamp_num_processed(0), + m_bounded_buffer(nullptr) { if (Direction::BOTH == direction) { LOGGER__ERROR("Boundary channels must be unidirectional"); @@ -67,22 +73,30 @@ BoundaryChannel::BoundaryChannel(vdma::ChannelId channel_id, Direction direction return; } - CB_INIT(m_descs, descs_count); - - status = allocate_descriptor_list(descs_count, desc_page_size); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to allocate Vdma buffer for channel transfer! status={}", status); - return; - } - status = HAILO_SUCCESS; } -hailo_status BoundaryChannel::trigger_channel_completion(uint16_t hw_num_processed) +// Function that based off the irq data returns the status to be sent to the callbak functions +static hailo_status get_callback_status(vdma::ChannelId channel_id, const ChannelIrqData &irq_data) { - // NOTE: right now, we can retake the 'completion' descriptor for a new transfer before handling the interrupt. - // we should have our own pointers indicating whats free instead of reading from HW. + hailo_status status = HAILO_UNINITIALIZED; + if (!irq_data.is_active) { + status = HAILO_STREAM_ABORT; + } else if (!irq_data.validation_success) { + LOGGER__WARNING("Channel {} validation failed", channel_id); + status = HAILO_INTERNAL_FAILURE; + } else if ((0 != irq_data.host_error) || (0 != irq_data.device_error)) { + LOGGER__WARNING("Channel {} completed with errors: host_error {} device_error {}", + channel_id, irq_data.host_error, irq_data.device_error); + status = HAILO_INTERNAL_FAILURE; + } else { + status = HAILO_SUCCESS; + } + return status; +} +hailo_status BoundaryChannel::trigger_channel_completion(const ChannelIrqData &irq_data) +{ std::unique_lock lock(m_channel_mutex); if (!m_is_channel_activated) { @@ -93,17 +107,44 @@ hailo_status BoundaryChannel::trigger_channel_completion(uint16_t hw_num_process CHECK_SUCCESS(update_latency_meter()); } - while (!m_ongoing_transfers.empty()) { - // Reading previous_num_processed inside the loop since on_transfer_complete may increase this value. - const auto previous_num_processed = static_cast(CB_TAIL(m_descs)); - if (!is_transfer_complete(m_ongoing_transfers.front(), previous_num_processed, hw_num_processed)) { - break; - } - + CHECK(irq_data.transfers_completed <= m_ongoing_transfers.size(), HAILO_INTERNAL_FAILURE, + "Invalid amount of completed transfers {} max {}", irq_data.transfers_completed, m_ongoing_transfers.size()); + + auto callback_status = get_callback_status(m_channel_id, irq_data); + // If channel is no longer active - all transfers should be completed + const size_t num_transfers_to_trigger = (HAILO_SUCCESS == callback_status) ? irq_data.transfers_completed : + m_ongoing_transfers.size(); + for (size_t i = 0; i < num_transfers_to_trigger; i++) { auto transfer = std::move(m_ongoing_transfers.front()); m_ongoing_transfers.pop_front(); - on_transfer_complete(lock, transfer, HAILO_SUCCESS); + // We increase desc num_proc (can happen only in this flow). After it is increased - + // 1. On D2H channels - the output can be read by the user. + // 2. On H2D channels - new input can be written to the buffer. + m_descs.set_tail((transfer.last_desc + 1) & m_descs.size_mask()); + + // We've freed up room in the descriptor list, so we can launch another transfer + if (!m_pending_transfers.empty()) { + m_transfer_launcher.enqueue_transfer([this]() { + std::unique_lock lock(m_channel_mutex); + if (m_pending_transfers.empty()) { + return; + } + auto transfer_request = std::move(m_pending_transfers.front()); + m_pending_transfers.pop_front(); + const auto status = launch_transfer_impl(std::move(transfer_request)); + if (status != HAILO_SUCCESS) { + on_request_complete(lock, transfer_request, status); + } + }); + } + + // Call the user callback + // We want to do this after launching transfers queued in m_pending_transfers, in order to keep the + // callback order consistent. + // Also, we want to make sure that the callbacks are called after the descriptors can be reused (so the user + // will be able to start new transfer). + on_request_complete(lock, transfer.request, callback_status); } return HAILO_SUCCESS; @@ -112,8 +153,8 @@ hailo_status BoundaryChannel::trigger_channel_completion(uint16_t hw_num_process CONTROL_PROTOCOL__host_buffer_info_t BoundaryChannel::get_boundary_buffer_info(uint32_t transfer_size) const { // Boundary channels always have scatter gather buffers - return VdmaEdgeLayer::get_host_buffer_info(VdmaEdgeLayer::Type::SCATTER_GATHER, m_desc_list->dma_address(), - m_desc_list->desc_page_size(), m_desc_list->count(), transfer_size); + return VdmaEdgeLayer::get_host_buffer_info(VdmaEdgeLayer::Type::SCATTER_GATHER, m_desc_list.dma_address(), + m_desc_list.desc_page_size(), m_desc_list.count(), transfer_size); } hailo_status BoundaryChannel::activate() @@ -125,35 +166,52 @@ hailo_status BoundaryChannel::activate() m_is_channel_activated = true; assert(m_ongoing_transfers.empty()); m_last_timestamp_num_processed = 0; - CB_RESET(m_descs); + m_descs.reset(); return HAILO_SUCCESS; } -hailo_status BoundaryChannel::deactivate() +void BoundaryChannel::deactivate() { std::unique_lock lock(m_channel_mutex); m_is_channel_activated = false; +} + +hailo_status BoundaryChannel::launch_transfer(TransferRequest &&transfer_request) +{ + std::unique_lock lock(m_channel_mutex); + if (!m_is_channel_activated) { + return HAILO_STREAM_NOT_ACTIVATED; + } + + if ((m_ongoing_transfers.size() < m_ongoing_transfers.capacity()) && (m_pending_transfers.size() == 0)) { + // There's room in the desc list and there are no pending transfers => execute on user's thread + // We can't use the user thread to launch the transfer if there are pending transfers, because we need to + // preserve the order of the transfers. + return launch_transfer_impl(std::move(transfer_request)); + } - // Note: OngoingTransfers held by m_ongoing_transfers may still hold copies of the current callback - // which in turn holds a reference to *this. Since we deactivate the channel there's no risk that - // these callbacks will be called and we don't need to reset this callback. + if (m_pending_transfers.size() >= m_pending_transfers.capacity()) { + return HAILO_QUEUE_IS_FULL; + } + // Defer to the transfer launcher + m_pending_transfers.push_back(std::move(transfer_request)); return HAILO_SUCCESS; } -hailo_status BoundaryChannel::launch_transfer(TransferRequest &&transfer_request) +// Assumes that the m_channel_mutex is locked! +hailo_status BoundaryChannel::launch_transfer_impl(TransferRequest &&transfer_request) { - std::unique_lock lock(m_channel_mutex); if (!m_is_channel_activated) { return HAILO_STREAM_NOT_ACTIVATED; } - if (m_ongoing_transfers.size() >= get_max_ongoing_transfers(transfer_request.get_total_transfer_size())) { + if (m_ongoing_transfers.size() >= m_ongoing_transfers.capacity()) { return HAILO_QUEUE_IS_FULL; } - auto num_available = static_cast(CB_HEAD(m_descs)); + auto num_available = static_cast(m_descs.head()); const uint16_t first_desc = num_available; uint16_t last_desc = std::numeric_limits::max(); uint16_t total_descs_count = 0; @@ -174,14 +232,14 @@ hailo_status BoundaryChannel::launch_transfer(TransferRequest &&transfer_request transfer_buffer.size() }); - const auto desired_desc_num = m_desc_list->descriptors_in_buffer(transfer_buffer.size()); + const auto desired_desc_num = m_desc_list.descriptors_in_buffer(transfer_buffer.size()); CHECK(desired_desc_num <= MAX_SG_DESCS_COUNT, HAILO_INTERNAL_FAILURE); const uint16_t desc_num = static_cast(desired_desc_num); assert(total_descs_count + desc_num < MAX_SG_DESCS_COUNT); total_descs_count = static_cast(total_descs_count + desc_num); - last_desc = static_cast((current_num_available + desc_num - 1) & m_descs.size_mask); - current_num_available = static_cast((last_desc + 1) & m_descs.size_mask); + last_desc = static_cast((current_num_available + desc_num - 1) & m_descs.size_mask()); + current_num_available = static_cast((last_desc + 1) & m_descs.size_mask()); } auto first_desc_interrupts = InterruptsDomain::NONE; @@ -191,22 +249,21 @@ hailo_status BoundaryChannel::launch_transfer(TransferRequest &&transfer_request } const auto last_desc_interrupts = InterruptsDomain::HOST; - int num_processed = CB_TAIL(m_descs); - int num_free = CB_AVAIL(m_descs, num_available, num_processed); + int num_processed = m_descs.tail(); + int num_free = m_descs.avail(num_available, num_processed); if (total_descs_count > num_free) { return HAILO_OUT_OF_DESCRIPTORS; } - m_ongoing_transfers.push_back(OngoingTransfer{std::move(transfer_request), last_desc}); if (m_latency_meter) { assert(!m_pending_latency_measurements.full()); m_pending_latency_measurements.push_back(m_direction == Direction::H2D ? first_desc : last_desc); } - CB_ENQUEUE(m_descs, total_descs_count); + m_descs.enqueue(total_descs_count); - TRY(const auto desc_programmed, m_driver.launch_transfer( + TRY_WITH_ACCEPTABLE_STATUS(HAILO_STREAM_ABORT, const auto desc_programmed, m_driver.launch_transfer( m_channel_id, - m_desc_list->handle(), + m_desc_list.handle(), num_available, driver_transfer_buffers, should_bind, @@ -215,6 +272,7 @@ hailo_status BoundaryChannel::launch_transfer(TransferRequest &&transfer_request )); CHECK(total_descs_count == desc_programmed, HAILO_INTERNAL_FAILURE, "Inconsistent desc programed expecting {} got {}", total_descs_count, desc_programmed); + m_ongoing_transfers.push_back(OngoingTransfer{std::move(transfer_request), last_desc}); return HAILO_SUCCESS; } @@ -223,12 +281,12 @@ hailo_status BoundaryChannel::bind_buffer(MappedBufferPtr buffer) { CHECK(m_bounded_buffer == nullptr, HAILO_INTERNAL_FAILURE, "Buffer is already bound to channel {}", m_channel_id); - const auto expected_size = static_cast(m_desc_list->desc_page_size()) * m_desc_list->count(); + const auto expected_size = static_cast(m_desc_list.desc_page_size()) * m_desc_list.count(); CHECK(buffer->size() == expected_size, HAILO_INVALID_ARGUMENT, - "Buffer size {} does not feet in desc list - descs count {} desc page size {}", buffer->size(), - m_desc_list->count(), m_desc_list->desc_page_size()); + "Buffer size {} does not fit in desc list - descs count {} desc page size {}", buffer->size(), + m_desc_list.count(), m_desc_list.desc_page_size()); static const size_t DEFAULT_BUFFER_OFFSET = 0; - CHECK_SUCCESS(m_desc_list->configure_to_use_buffer(*buffer, buffer->size(), DEFAULT_BUFFER_OFFSET, m_channel_id)); + CHECK_SUCCESS(m_desc_list.program(*buffer, buffer->size(), DEFAULT_BUFFER_OFFSET, m_channel_id)); m_bounded_buffer = buffer; return HAILO_SUCCESS; } @@ -236,22 +294,35 @@ hailo_status BoundaryChannel::bind_buffer(MappedBufferPtr buffer) void BoundaryChannel::cancel_pending_transfers() { std::unique_lock lock(m_channel_mutex); + // Cancel all ongoing transfers while (!m_ongoing_transfers.empty()) { auto transfer = std::move(m_ongoing_transfers.front()); m_ongoing_transfers.pop_front(); - on_transfer_complete(lock, transfer, HAILO_STREAM_ABORT); + on_request_complete(lock, transfer.request, HAILO_STREAM_ABORT); + } + + // Then cancel all pending transfers (which were to happen after the ongoing transfers are done) + while (!m_pending_transfers.empty()) { + auto pending_transfer = std::move(m_pending_transfers.front()); + m_pending_transfers.pop_front(); + + on_request_complete(lock, pending_transfer, HAILO_STREAM_ABORT); } } -size_t BoundaryChannel::get_max_ongoing_transfers(size_t transfer_size) const +size_t BoundaryChannel::get_max_ongoing_transfers(size_t /* transfer_size */) const { - // Add desc for boundary channel because might need extra for non aligned async API - const auto descs_in_transfer = m_desc_list->descriptors_in_buffer(transfer_size) + 1; - const auto descs_count = CB_SIZE(m_descs); - size_t max_transfers_in_buffer = (descs_count - 1) / descs_in_transfer; + // TODO: Remove transfer_size from get_max_ongoing_transfers (HRT-13419) + return std::max(m_pending_transfers.capacity(), m_ongoing_transfers.capacity()); +} - return std::min(max_transfers_in_buffer, m_ongoing_transfers.capacity()); +// TODO: try and get rid of this func and merge with get_max_ongoing_transfers (HRT-13557) +size_t BoundaryChannel::get_max_aligned_transfers_in_desc_list(size_t transfer_size) const +{ + // Since this calc if for aligned transfers, we don't need to factor in the bounce buffer + static const auto NO_BOUNCE_BUFFER = false; + return m_desc_list.max_transfers(static_cast(transfer_size), NO_BOUNCE_BUFFER); } hailo_status BoundaryChannel::update_latency_meter() @@ -295,28 +366,11 @@ hailo_status BoundaryChannel::update_latency_meter() return HAILO_SUCCESS; } -bool BoundaryChannel::is_transfer_complete(const OngoingTransfer &transfer, uint16_t previous_num_processed, - uint16_t current_num_processed) const -{ - // Transfer is complete if its last descriptor is in [previous_num_processed, current_num_processed) or - // the the buffer is empty (previous_num_processed == CB_HEAD(m_descs)) - return is_desc_between(previous_num_processed, current_num_processed, transfer.last_desc) || - (current_num_processed == CB_HEAD(m_descs)); -} - -void BoundaryChannel::on_transfer_complete(std::unique_lock &lock, - OngoingTransfer &transfer, hailo_status complete_status) +void BoundaryChannel::on_request_complete(std::unique_lock &lock, TransferRequest &request, + hailo_status complete_status) { - // We increase desc num_proc (can happen only in this flow). After it is increased - - // 1. On D2H channels - the output can be read by the user. - // 2. On H2D channels - new input can be written to the buffer. - _CB_SET(m_descs.tail, (transfer.last_desc + 1) & m_descs.size_mask); - - // Finally, we notify user callbacks registered with the transfer. - // We want to make sure that the callbacks are called after the descriptors can be reused (So the user will - // be able to start new transfer). lock.unlock(); - transfer.request.callback(complete_status); + request.callback(complete_status); lock.lock(); } @@ -336,18 +390,6 @@ bool BoundaryChannel::is_desc_between(uint16_t begin, uint16_t end, uint16_t des } } -hailo_status BoundaryChannel::allocate_descriptor_list(uint32_t descs_count, uint16_t desc_page_size) -{ - static const bool CIRCULAR = true; - auto desc_list_exp = DescriptorList::create(descs_count, desc_page_size, CIRCULAR, m_driver); - CHECK_EXPECTED_AS_STATUS(desc_list_exp); - - m_desc_list = make_shared_nothrow(desc_list_exp.release()); - CHECK_NOT_NULL(m_desc_list, HAILO_OUT_OF_HOST_MEMORY); - - return HAILO_SUCCESS; -} - hailo_status BoundaryChannel::validate_bound_buffer(TransferRequest &transfer_request) { assert(m_bounded_buffer); @@ -355,14 +397,15 @@ hailo_status BoundaryChannel::validate_bound_buffer(TransferRequest &transfer_re "When bound buffer is used, transfer request must contain only one buffer"); auto &transfer_buffer = transfer_request.transfer_buffers[0]; - const auto num_available = CB_HEAD(m_descs); - const auto expected_offset = static_cast(m_desc_list->desc_page_size()) * num_available; + const auto num_available = m_descs.head(); + const auto expected_offset = static_cast(m_desc_list.desc_page_size()) * num_available; CHECK(transfer_buffer.offset() == expected_offset, HAILO_INTERNAL_FAILURE, "Unexpected buffer offset, expected {} actual {}", expected_offset, transfer_buffer.offset()); - CHECK(transfer_buffer.base_buffer().data() == reinterpret_cast(m_bounded_buffer->user_address()), HAILO_INTERNAL_FAILURE, + TRY(auto base_buffer, transfer_buffer.base_buffer()); + CHECK(base_buffer.data() == reinterpret_cast(m_bounded_buffer->user_address()), HAILO_INTERNAL_FAILURE, "Got the wrong buffer"); - CHECK(transfer_buffer.base_buffer().size() == m_bounded_buffer->size(), HAILO_INTERNAL_FAILURE, - "Got invalid buffer size {}, expected {}", transfer_buffer.base_buffer().size(), m_bounded_buffer->size()); + CHECK(base_buffer.size() == m_bounded_buffer->size(), HAILO_INTERNAL_FAILURE, + "Got invalid buffer size {}, expected {}", base_buffer.size(), m_bounded_buffer->size()); return HAILO_SUCCESS; } diff --git a/hailort/libhailort/src/vdma/channel/boundary_channel.hpp b/hailort/libhailort/src/vdma/channel/boundary_channel.hpp index 8b13613..55e5596 100644 --- a/hailort/libhailort/src/vdma/channel/boundary_channel.hpp +++ b/hailort/libhailort/src/vdma/channel/boundary_channel.hpp @@ -11,6 +11,7 @@ #define _HAILO_VDMA_BOUNDARY_CHANNEL_HPP_ #include "vdma/channel/channel_id.hpp" +#include "vdma/channel/transfer_launcher.hpp" #include "vdma/memory/descriptor_list.hpp" #include "stream_common/transfer_common.hpp" @@ -36,25 +37,28 @@ class BoundaryChannel final public: using Direction = HailoRTDriver::DmaDirection; - static Expected create(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, - uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name = "", LatencyMeterPtr latency_meter = nullptr); + static Expected create(HailoRTDriver &driver, vdma::ChannelId channel_id, Direction direction, + vdma::DescriptorList &&desc_list, TransferLauncher &transfer_launcher, size_t ongoing_transfers, + size_t pending_transfers = 0, const std::string &stream_name = "", LatencyMeterPtr latency_meter = nullptr); - BoundaryChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, uint32_t descs_count, - uint16_t desc_page_size, const std::string &stream_name, LatencyMeterPtr latency_meter, - hailo_status &status); + BoundaryChannel(HailoRTDriver &driver, vdma::ChannelId channel_id, Direction direction, DescriptorList &&desc_list, + TransferLauncher &transfer_launcher, size_t ongoing_transfers_queue_size, size_t pending_transfers_queue_size, + const std::string &stream_name, LatencyMeterPtr latency_meter, hailo_status &status); BoundaryChannel(const BoundaryChannel &other) = delete; BoundaryChannel &operator=(const BoundaryChannel &other) = delete; BoundaryChannel(BoundaryChannel &&other) = delete; BoundaryChannel &operator=(BoundaryChannel &&other) = delete; virtual ~BoundaryChannel() = default; - // Called after the FW activated the channel. + /** + * Activates the channel object, assume the vDMA channel registers are already in activated state. + */ hailo_status activate(); - // Called before the FW deactivated the channel. - hailo_status deactivate(); - - hailo_status trigger_channel_completion(uint16_t hw_num_processed); + /** + * Deactivates the channel object, assume the vDMA channel registers are already in deactivated state. + */ + void deactivate(); // Calls all pending transfer callbacks (if they exist), marking them as canceled by passing // HAILO_STREAM_ABORT as a status to the callbacks. @@ -62,13 +66,20 @@ public: // unexpected results void cancel_pending_transfers(); + /** + * Called when some transfer (or transfers) is completed. + */ + hailo_status trigger_channel_completion(const ChannelIrqData &irq_data); + hailo_status launch_transfer(TransferRequest &&transfer_request); // To avoid buffer bindings, one can call this function to statically bind a full buffer to the channel. The buffer // size should be exactly desc_page_size() * descs_count() of current descriptors list. hailo_status bind_buffer(MappedBufferPtr buffer); + // TODO: rename BoundaryChannel::get_max_ongoing_transfers to BoundaryChannel::get_max_parallel_transfers (HRT-13513) size_t get_max_ongoing_transfers(size_t transfer_size) const; + size_t get_max_aligned_transfers_in_desc_list(size_t transfer_size) const; CONTROL_PROTOCOL__host_buffer_info_t get_boundary_buffer_info(uint32_t transfer_size) const; @@ -82,33 +93,40 @@ public: return m_stream_name; } - std::shared_ptr get_desc_list() + DescriptorList &get_desc_list() { return m_desc_list; } -private: + bool should_measure_timestamp() const { return m_latency_meter != nullptr; } +private: hailo_status update_latency_meter(); - bool is_transfer_complete(const OngoingTransfer &transfer, uint16_t previous_num_processed, - uint16_t current_num_processed) const; - void on_transfer_complete(std::unique_lock &lock, OngoingTransfer &transfer, + void on_request_complete(std::unique_lock &lock, TransferRequest &request, hailo_status complete_status); + hailo_status launch_transfer_impl(TransferRequest &&transfer_request); static bool is_desc_between(uint16_t begin, uint16_t end, uint16_t desc); - hailo_status allocate_descriptor_list(uint32_t descs_count, uint16_t desc_page_size); hailo_status validate_bound_buffer(TransferRequest &transfer_request); const vdma::ChannelId m_channel_id; const Direction m_direction; HailoRTDriver &m_driver; - std::shared_ptr m_desc_list; // Host side descriptor list + TransferLauncher &m_transfer_launcher; + DescriptorList m_desc_list; // Host side descriptor list const std::string m_stream_name; - circbuf_t m_descs; + // Since all desc list sizes are a power of 2, we can use IsPow2Tag to optimize the circular buffer + CircularBuffer m_descs; bool m_is_channel_activated; std::mutex m_channel_mutex; - CircularArray m_ongoing_transfers; + // * m_pending_transfers holds transfers that are waiting to be bound to the descriptor list. + // * m_ongoing_transfers holds transfers that have been bound to the descriptor list and + // are waiting to be completed. + // * Note that the capacity of the pending_transfers and ongoing_transfers circular + // buffers may not be a power of 2, hence the IsNotPow2Tag + CircularArray m_ongoing_transfers; + CircularArray m_pending_transfers; // About HW latency measurements: // - For each ongoing transfer, we push some num-proc value to the pending_latency_measurements array. When this diff --git a/hailort/libhailort/src/vdma/channel/channels_group.cpp b/hailort/libhailort/src/vdma/channel/channels_group.cpp new file mode 100644 index 0000000..a206bd0 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/channels_group.cpp @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file channels_group.cpp + **/ + +#include "channels_group.hpp" + +namespace hailort { +namespace vdma { + +ChannelsGroup::ChannelsGroup(std::initializer_list channels) +{ + for (const auto &channel : channels) { + add_channel(channel); + } +} + +void ChannelsGroup::add_channel(BoundaryChannelPtr channel) +{ + const auto id = channel->get_channel_id(); + assert(nullptr == m_channels[id.engine_index][id.channel_index]); + m_channels[id.engine_index][id.channel_index] = channel; +} + +ChannelsBitmap ChannelsGroup::bitmap() const +{ + ChannelsBitmap bitmap{}; + for (size_t i = 0; i < m_channels.size(); i++) { + for (size_t j = 0; j < m_channels[i].size(); j++) { + if (m_channels[i][j]) { + bitmap[i] |= (1 << j); + } + } + } + return bitmap; +} + +bool ChannelsGroup::should_measure_timestamp() const +{ + for (const auto &engine : m_channels) { + for (const auto &channel : engine) { + if (channel && channel->should_measure_timestamp()) { + return true; + } + } + } + return false; +} + +Expected ChannelsGroup::get_by_id(vdma::ChannelId channel_id) +{ + auto channel = m_channels[channel_id.engine_index][channel_id.channel_index]; + if (!channel) { + return make_unexpected(HAILO_NOT_FOUND); + } + return channel; +} + +Expected ChannelsGroup::get_by_name(const std::string &stream_name) +{ + for (const auto &engine : m_channels) { + for (const auto &channel : engine) { + if (channel && (channel->stream_name() == stream_name)) { + return BoundaryChannelPtr{channel}; + } + } + } + return make_unexpected(HAILO_NOT_FOUND); +} + +void ChannelsGroup::process_interrupts(IrqData &&irq_data) +{ + assert(irq_data.channels_count <= ARRAY_ENTRIES(irq_data.channels_irq_data)); + for (uint8_t irq_index = 0; irq_index < irq_data.channels_count; irq_index++) { + const auto &channel_irq_data = irq_data.channels_irq_data[irq_index]; + auto status = process_channel_interrupt(channel_irq_data); // TODO: TODO: HRT-9429 done + if ((status != HAILO_SUCCESS) && (status != HAILO_STREAM_NOT_ACTIVATED)) { + LOGGER__ERROR("Trigger channel completion failed on channel {} with status {}", channel_irq_data.channel_id, status); + } + } +} + +hailo_status ChannelsGroup::process_channel_interrupt(const ChannelIrqData &channel_irq_data) +{ + TRY(auto channel, get_by_id(channel_irq_data.channel_id), "Channel {} not found", channel_irq_data.channel_id); + return channel->trigger_channel_completion(channel_irq_data); +} + +} /* namespace vdma */ +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/channel/channels_group.hpp b/hailort/libhailort/src/vdma/channel/channels_group.hpp new file mode 100644 index 0000000..03830c5 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/channels_group.hpp @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file channels_group.hpp + * @brief Contains a group of channels that are used together. + **/ + +#ifndef _HAILO_CHANNELS_GROUP_HPP_ +#define _HAILO_CHANNELS_GROUP_HPP_ + +#include "vdma/channel/boundary_channel.hpp" + +namespace hailort { +namespace vdma { + +class ChannelsGroup final { +public: + ChannelsGroup() = default; + ChannelsGroup(std::initializer_list channels); + + void add_channel(BoundaryChannelPtr channel); + + ChannelsBitmap bitmap() const; + bool should_measure_timestamp() const; + Expected get_by_id(vdma::ChannelId channel_id); + Expected get_by_name(const std::string &stream_name); + + void process_interrupts(IrqData &&irq_data); + +private: + + hailo_status process_channel_interrupt(const ChannelIrqData &irq_data); + + std::array< + std::array, + MAX_VDMA_ENGINES_COUNT + > m_channels; +}; + +} /* namespace vdma */ +} /* namespace hailort */ + +#endif /* _HAILO_CHANNELS_GROUP_HPP_ */ diff --git a/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.cpp b/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.cpp index 0f71c2c..36fba73 100644 --- a/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.cpp +++ b/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.cpp @@ -52,14 +52,22 @@ hailo_status InterruptsDispatcher::start(const ChannelsBitmap &channels_bitmap, CHECK_NOT_NULL(wait_context, HAILO_OUT_OF_HOST_MEMORY); m_wait_context = std::move(wait_context); - auto status = m_driver.get().vdma_interrupts_enable(m_wait_context->bitmap, enable_timestamp_measure); - CHECK_SUCCESS(status, "Failed to enable vdma interrupts"); + auto status = m_driver.get().vdma_enable_channels(m_wait_context->bitmap, enable_timestamp_measure); + CHECK_SUCCESS(status, "Failed to enable vdma channels"); } m_cond.notify_one(); return HAILO_SUCCESS; } +hailo_status InterruptsDispatcher::start(const ChannelsGroup &channels_group) +{ + return start(channels_group.bitmap(), channels_group.should_measure_timestamp(), + [channels_group=channels_group](IrqData &&irq_data) mutable { + channels_group.process_interrupts(std::move(irq_data)); + }); +} + hailo_status InterruptsDispatcher::stop() { std::unique_lock lock(m_mutex); @@ -74,7 +82,7 @@ hailo_status InterruptsDispatcher::stop() m_wait_context = nullptr; // Calling disable interrupts will cause the vdma_interrupts_wait to return. - auto status = m_driver.get().vdma_interrupts_disable(bitmap); + auto status = m_driver.get().vdma_disable_channels(bitmap); CHECK_SUCCESS(status, "Failed to disable vdma interrupts"); // Needs to make sure that the interrupts thread is disabled. @@ -105,7 +113,7 @@ void InterruptsDispatcher::wait_interrupts() // vdma_interrupts_wait is a blocking function that returns in this scenarios: // 1. We got a new interrupts, irq_data will be passed to the process_irq callback - // 2. vdma_interrupts_disable will be called, vdma_interrupts_wait will return with an empty list. + // 2. vdma_disable_channels will be called, vdma_interrupts_wait will return with an empty list. // 3. Other error returns - shouldn't really happen, we exit the interrupt thread. lock.unlock(); auto irq_data = m_driver.get().vdma_interrupts_wait(wait_context.bitmap); diff --git a/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.hpp b/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.hpp index 50b0c49..8a7288e 100644 --- a/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.hpp +++ b/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.hpp @@ -11,6 +11,8 @@ #define _HAILO_VDMA_INTERRUPTS_DISPATCHER_HPP_ #include "vdma/driver/hailort_driver.hpp" +#include "vdma/channel/channels_group.hpp" + #include #include #include @@ -35,6 +37,7 @@ public: hailo_status start(const ChannelsBitmap &channels_bitmap, bool enable_timestamp_measure, const ProcessIrqCallback &process_irq); + hailo_status start(const ChannelsGroup &channels_group); hailo_status stop(); private: diff --git a/hailort/libhailort/src/vdma/channel/transfer_launcher.cpp b/hailort/libhailort/src/vdma/channel/transfer_launcher.cpp index 96c5c93..469bd54 100644 --- a/hailort/libhailort/src/vdma/channel/transfer_launcher.cpp +++ b/hailort/libhailort/src/vdma/channel/transfer_launcher.cpp @@ -79,13 +79,8 @@ hailo_status TransferLauncher::stop() m_thread_active = false; while (!m_queue.empty()) { + // No need signal that the transfer was aborted, it'll be done in BoundaryChannel::cancel_pending_transfers m_queue.pop(); - // TODO: need to call the callbacks to signal that they were aborted? (HRT-13110) - // like this: - // auto transfer_request = m_queue.front(); - // m_queue.pop(); - // transfer_request.callback(HAILO_STREAM_ABORT); - // or can it be done in BoundaryChannel::cancel_pending_transfers? } // TODO: Keep stop flow used in interrupt thread? (HRT-13110) diff --git a/hailort/libhailort/src/vdma/channel/transfer_launcher.hpp b/hailort/libhailort/src/vdma/channel/transfer_launcher.hpp index 8713684..1857e14 100644 --- a/hailort/libhailort/src/vdma/channel/transfer_launcher.hpp +++ b/hailort/libhailort/src/vdma/channel/transfer_launcher.hpp @@ -26,7 +26,6 @@ namespace vdma { class TransferLauncher final { public: - // TODO: fix this to be a proper transfer object (HRT-13110) using Transfer = std::function; static Expected> create(); diff --git a/hailort/libhailort/src/vdma/circular_stream_buffer_pool.cpp b/hailort/libhailort/src/vdma/circular_stream_buffer_pool.cpp index 6be6287..ea504bd 100644 --- a/hailort/libhailort/src/vdma/circular_stream_buffer_pool.cpp +++ b/hailort/libhailort/src/vdma/circular_stream_buffer_pool.cpp @@ -43,22 +43,22 @@ CircularStreamBufferPool::CircularStreamBufferPool(size_t desc_page_size, size_t m_transfer_size(transfer_size), m_base_buffer(std::move(base_buffer)), m_mappings(std::move(mappings)), + m_queue(static_cast(descs_count)), m_next_enqueue_desc_offset(0) { assert(is_powerof2(descs_count) && (descs_count > 0)); assert(m_base_buffer.size() == (m_desc_page_size * descs_count)); - CB_INIT(m_queue, descs_count); - m_queue.head = static_cast(descs_count - 1); + m_queue.set_head(static_cast(descs_count) - 1); } size_t CircularStreamBufferPool::max_queue_size() const { - return (m_queue.size - 1) / DIV_ROUND_UP(m_transfer_size, m_desc_page_size); + return (m_queue.size() - 1) / DIV_ROUND_UP(m_transfer_size, m_desc_page_size); } size_t CircularStreamBufferPool::buffers_ready_to_dequeue() const { - const size_t descs_available = CB_PROG(m_queue, CB_HEAD(m_queue), CB_TAIL(m_queue)); + const size_t descs_available = m_queue.prog(m_queue.head(), m_queue.tail()); return descs_available / descs_in_transfer(); } @@ -66,8 +66,8 @@ Expected CircularStreamBufferPool::dequeue() { CHECK_AS_EXPECTED(buffers_ready_to_dequeue() > 0, HAILO_INTERNAL_FAILURE, "CircularStreamBufferPool is empty"); - const size_t offset_in_buffer = CB_TAIL(m_queue) * m_desc_page_size; - CB_DEQUEUE(m_queue, descs_in_transfer()); + const size_t offset_in_buffer = m_queue.tail() * m_desc_page_size; + m_queue.dequeue(static_cast(descs_in_transfer())); return TransferBuffer { MemoryView(m_base_buffer), m_transfer_size, @@ -78,9 +78,10 @@ Expected CircularStreamBufferPool::dequeue() hailo_status CircularStreamBufferPool::enqueue(TransferBuffer &&buffer_info) { const size_t descs_required = descs_in_transfer(); - const size_t descs_available = CB_AVAIL(m_queue, CB_HEAD(m_queue), CB_TAIL(m_queue)); + const size_t descs_available = m_queue.avail(m_queue.head(), m_queue.tail()); CHECK(descs_available >= descs_required, HAILO_INTERNAL_FAILURE, "Can enqueue without previous dequeue"); - CHECK(buffer_info.base_buffer().data() == m_base_buffer.data(), HAILO_INTERNAL_FAILURE, "Got the wrong buffer"); + TRY(auto base_buffer, buffer_info.base_buffer()); + CHECK(base_buffer.data() == m_base_buffer.data(), HAILO_INTERNAL_FAILURE, "Got the wrong buffer"); CHECK(buffer_info.size() == m_transfer_size, HAILO_INTERNAL_FAILURE, "Got invalid buffer size {}, expected {}", buffer_info.size(), m_transfer_size); @@ -89,15 +90,15 @@ hailo_status CircularStreamBufferPool::enqueue(TransferBuffer &&buffer_info) "Out of order enqueue is not supported in CircularStreamBufferPool. Got offset {}, expected {}", buffer_info.offset(), expected_offset); - CB_ENQUEUE(m_queue, descs_required); - m_next_enqueue_desc_offset = (m_next_enqueue_desc_offset + descs_required) & m_queue.size_mask; + m_queue.enqueue(static_cast(descs_required)); + m_next_enqueue_desc_offset = (m_next_enqueue_desc_offset + descs_required) & m_queue.size_mask(); return HAILO_SUCCESS; } void CircularStreamBufferPool::reset_pointers() { - CB_RESET(m_queue); - m_queue.head = static_cast(m_queue.size - 1); + m_queue.reset(); + m_queue.set_head(static_cast(m_queue.size()) - 1); m_next_enqueue_desc_offset = 0; } diff --git a/hailort/libhailort/src/vdma/circular_stream_buffer_pool.hpp b/hailort/libhailort/src/vdma/circular_stream_buffer_pool.hpp index 4fd8765..17ba1cb 100644 --- a/hailort/libhailort/src/vdma/circular_stream_buffer_pool.hpp +++ b/hailort/libhailort/src/vdma/circular_stream_buffer_pool.hpp @@ -56,19 +56,19 @@ private: const size_t m_transfer_size; - // m_mapped_buffer.size() must be CB_SIZE(m_queue) * m_desc_page_size + // m_mapped_buffer.size() must be m_queue.size() * m_desc_page_size Buffer m_base_buffer; DmaMappedBuffer m_mappings; // Head/tail based queue that manages the buffer pool. // The head and tail are in m_desc_page_size granularity. // - // If CB_HEAD(m_queue) == CB_TAIL(m_queue) the pool is empty. + // If m_queue.head() == m_queue.tail() the pool is empty. // Otherwise, the buffers that can be in use starts from - // CB_TAIL(m_queue) * m_desc_page_size (inclusive) + // m_queue.tail() * m_desc_page_size (inclusive) // until - // CB_HEAD(m_queue) * m_desc_page_size (exclusive) - circbuf_t m_queue; + // m_queue.head() * m_desc_page_size (exclusive) + CircularBuffer m_queue; // Used to validate that the buffers are enqueued in order. size_t m_next_enqueue_desc_offset; diff --git a/hailort/libhailort/src/vdma/driver/hailort_driver.cpp b/hailort/libhailort/src/vdma/driver/hailort_driver.cpp index 36f0c20..891eb92 100755 --- a/hailort/libhailort/src/vdma/driver/hailort_driver.cpp +++ b/hailort/libhailort/src/vdma/driver/hailort_driver.cpp @@ -41,24 +41,25 @@ static_assert(static_cast(InterruptsDomain::BOTH) == (HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE | HAILO_VDMA_INTERRUPTS_DOMAIN_HOST), "Driver and libhailort parameters mismatch"); +static const std::string INTEGRATED_NNC_DRIVER_PATH = "/dev/hailo_integrated_nnc"; +static const std::string PCIE_EP_DRIVER_PATH = "/dev/hailo_pci_ep"; + + #define CHECK_IOCTL_RESULT(err, message) do { \ auto __err = (err); \ CHECK(0 == __err, HAILO_DRIVER_FAIL, message " errno: {}", __err); \ } while (0) -static hailo_dma_data_direction direction_to_dma_data_direction(HailoRTDriver::DmaDirection direction) { - switch (direction) { - case HailoRTDriver::DmaDirection::H2D: - return HAILO_DMA_TO_DEVICE; - case HailoRTDriver::DmaDirection::D2H: - return HAILO_DMA_FROM_DEVICE; - case HailoRTDriver::DmaDirection::BOTH: - return HAILO_DMA_BIDIRECTIONAL; +static hailo_dma_buffer_type driver_dma_buffer_type_to_dma_buffer_type(HailoRTDriver::DmaBufferType buffer_type) { + switch (buffer_type) { + case HailoRTDriver::DmaBufferType::USER_PTR_BUFFER: + return HAILO_DMA_USER_PTR_BUFFER; + case HailoRTDriver::DmaBufferType::DMABUF_BUFFER: + return HAILO_DMA_DMABUF_BUFFER; } assert(false); - // On release build Return value that will make ioctls to fail. - return HAILO_DMA_NONE; + return HAILO_DMA_BUFFER_MAX_ENUM; } static enum hailo_cpu_id translate_cpu_id(hailo_cpu_id_t cpu_id) @@ -102,12 +103,31 @@ static hailo_transfer_memory_type translate_memory_type(HailoRTDriver::MemoryTyp return HAILO_TRANSFER_MEMORY_DMA_ENGINE1; case MemoryType::DMA_ENGINE2: return HAILO_TRANSFER_MEMORY_DMA_ENGINE2; + case MemoryType::PCIE_EP_CONFIG: + return HAILO_TRANSFER_MEMORY_PCIE_EP_CONFIG; + case MemoryType::PCIE_EP_BRIDGE: + return HAILO_TRANSFER_MEMORY_PCIE_EP_BRIDGE; } assert(false); return HAILO_TRANSFER_MEMORY_MAX_ENUM; } +static hailo_dma_data_direction direction_to_dma_data_direction(HailoRTDriver::DmaDirection direction) { + switch (direction) { + case HailoRTDriver::DmaDirection::H2D: + return HAILO_DMA_TO_DEVICE; + case HailoRTDriver::DmaDirection::D2H: + return HAILO_DMA_FROM_DEVICE; + case HailoRTDriver::DmaDirection::BOTH: + return HAILO_DMA_BIDIRECTIONAL; + } + + assert(false); + // On release build Return value that will make ioctls to fail. + return HAILO_DMA_NONE; +} + // TODO: validate wraparounds for buffer/mapping handles in the driver (HRT-9509) const uintptr_t HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE = INVALID_DRIVER_HANDLE_VALUE; const size_t HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE = INVALID_DRIVER_HANDLE_VALUE; @@ -121,18 +141,61 @@ const vdma_mapped_buffer_driver_identifier HailoRTDriver::INVALID_MAPPED_BUFFER_ #error "unsupported platform!" #endif -Expected> HailoRTDriver::create(const DeviceInfo &device_info) +Expected> HailoRTDriver::create(const std::string &device_id, const std::string &dev_path) { - TRY(auto fd, open_device_file(device_info.dev_path)); + TRY(auto fd, open_device_file(dev_path)); hailo_status status = HAILO_UNINITIALIZED; - std::unique_ptr driver(new (std::nothrow) HailoRTDriver(device_info, std::move(fd), status)); + std::unique_ptr driver(new (std::nothrow) HailoRTDriver(device_id, std::move(fd), status)); CHECK_NOT_NULL_AS_EXPECTED(driver, HAILO_OUT_OF_HOST_MEMORY); CHECK_SUCCESS_AS_EXPECTED(status); return driver; } +Expected> HailoRTDriver::create_pcie(const std::string &device_id) +{ + TRY(const auto scan_results, scan_devices()); + + auto device_found = std::find_if(scan_results.cbegin(), scan_results.cend(), + [&device_id](const auto &compared_scan_result) { + return (device_id == compared_scan_result.device_id); + }); + CHECK(device_found != scan_results.cend(), HAILO_INVALID_ARGUMENT, "Requested device not found"); + + return create(device_found->device_id, device_found->dev_path); +} + +Expected> HailoRTDriver::create_integrated_nnc() +{ + return create(INTEGRATED_NNC_DEVICE_ID, INTEGRATED_NNC_DRIVER_PATH); +} + +bool HailoRTDriver::is_integrated_nnc_loaded() +{ +#if defined(_MSC_VER) + // windows is not supported for integrated_nnc driver + return false; +#else + return (access(INTEGRATED_NNC_DRIVER_PATH.c_str(), F_OK) == 0); +#endif // defined(_MSC_VER) +} + +Expected> HailoRTDriver::create_pcie_ep() +{ + return create(PCIE_EP_DEVICE_ID, PCIE_EP_DRIVER_PATH); +} + +bool HailoRTDriver::is_pcie_ep_loaded() +{ +#if defined(_MSC_VER) + // windows is not supported for pcie_ep driver + return false; +#else + return (access(PCIE_EP_DRIVER_PATH.c_str(), F_OK) == 0); +#endif // defined(_MSC_VER) +} + static hailo_status validate_driver_version(const hailo_driver_info &driver_info) { hailo_version_t library_version{}; @@ -147,9 +210,9 @@ static hailo_status validate_driver_version(const hailo_driver_info &driver_info return HAILO_SUCCESS; } -HailoRTDriver::HailoRTDriver(const DeviceInfo &device_info, FileDescriptor &&fd, hailo_status &status) : +HailoRTDriver::HailoRTDriver(const std::string &device_id, FileDescriptor &&fd, hailo_status &status) : m_fd(std::move(fd)), - m_device_info(device_info), + m_device_id(device_id), m_allocate_driver_buffer(false) { hailo_driver_info driver_info{}; @@ -184,6 +247,9 @@ HailoRTDriver::HailoRTDriver(const DeviceInfo &device_info, FileDescriptor &&fd, case HAILO_DMA_TYPE_DRAM: m_dma_type = DmaType::DRAM; break; + case HAILO_DMA_TYPE_PCI_EP: + m_dma_type = DmaType::PCIE_EP; + break; default: LOGGER__ERROR("Invalid dma type returned from ioctl {}", device_properties.dma_type); status = HAILO_DRIVER_FAIL; @@ -209,17 +275,38 @@ HailoRTDriver::~HailoRTDriver() } } +Expected> scan_all_devices() +{ + std::vector devices_info; + + TRY(auto nnc_devices, scan_nnc_devices()); + if (!nnc_devices.empty()) { + devices_info.insert(devices_info.end(), nnc_devices.begin(), nnc_devices.end()); + } + + TRY(auto soc_devices, scan_soc_devices()); + if (!soc_devices.empty()) { + devices_info.insert(devices_info.end(), soc_devices.begin(), soc_devices.end()); + } + + return devices_info; +} + Expected> HailoRTDriver::scan_devices() { - auto device_names = list_devices(); - CHECK_EXPECTED(device_names, "Failed listing pcie devices"); + return scan_all_devices(); +} +Expected> HailoRTDriver::scan_devices(AcceleratorType acc_type) +{ std::vector devices_info; - for (const auto &device_name : device_names.value()) { - auto device_info = query_device_info(device_name); - CHECK_EXPECTED(device_info, "failed parsing device info for {}", device_name); - devices_info.push_back(device_info.release()); + + if (AcceleratorType::SOC_ACCELERATOR == acc_type) { + TRY(devices_info, scan_soc_devices()); + } else if (AcceleratorType::NNC_ACCELERATOR == acc_type) { + TRY(devices_info, scan_nnc_devices()); } + return devices_info; } @@ -273,24 +360,24 @@ hailo_status HailoRTDriver::write_memory(MemoryType memory_type, uint64_t addres return HAILO_SUCCESS; } -hailo_status HailoRTDriver::vdma_interrupts_enable(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure) +hailo_status HailoRTDriver::vdma_enable_channels(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure) { CHECK(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given"); - hailo_vdma_interrupts_enable_params params{}; + hailo_vdma_enable_channels_params params{}; std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine); params.enable_timestamps_measure = enable_timestamps_measure; - CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_INTERRUPTS_ENABLE, ¶ms), "Failed to enabled vdma interrupts"); + CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_ENABLE_CHANNELS, ¶ms), "Failed to enable vdma channels"); return HAILO_SUCCESS; } -hailo_status HailoRTDriver::vdma_interrupts_disable(const ChannelsBitmap &channels_bitmap) +hailo_status HailoRTDriver::vdma_disable_channels(const ChannelsBitmap &channels_bitmap) { CHECK(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given"); - hailo_vdma_interrupts_disable_params params{}; + hailo_vdma_disable_channels_params params{}; std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine); - CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_INTERRUPTS_DISABLE, ¶ms), "Failed to disable vdma interrupts"); + CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_DISABLE_CHANNELS, ¶ms), "Failed to disable vdma channels"); return HAILO_SUCCESS; } @@ -329,7 +416,7 @@ static Expected to_irq_data(const hailo_vdma_interrupts_wait_params& pa irq.channels_irq_data[i].channel_id.engine_index = engine_index; irq.channels_irq_data[i].channel_id.channel_index = channel_index; irq.channels_irq_data[i].is_active = params.irq_data[i].is_active; - irq.channels_irq_data[i].desc_num_processed = params.irq_data[i].host_num_processed; + irq.channels_irq_data[i].transfers_completed = params.irq_data[i].transfers_completed; irq.channels_irq_data[i].host_error = params.irq_data[i].host_error; irq.channels_irq_data[i].device_error = params.irq_data[i].device_error; irq.channels_irq_data[i].validation_success = params.irq_data[i].validation_success; @@ -443,8 +530,19 @@ hailo_status HailoRTDriver::reset_nn_core() return HAILO_SUCCESS; } -Expected HailoRTDriver::vdma_buffer_map(void *user_address, size_t required_size, - DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle) { +Expected HailoRTDriver::vdma_buffer_map_dmabuf(int dmabuf_fd, size_t required_size, DmaDirection data_direction, + DmaBufferType buffer_type) +{ + CHECK_AS_EXPECTED (DmaBufferType::DMABUF_BUFFER == buffer_type, HAILO_INVALID_ARGUMENT, + "Error, Invalid buffer type given, buffer type {}", buffer_type); + + return vdma_buffer_map(dmabuf_fd, required_size, data_direction, INVALID_MAPPED_BUFFER_DRIVER_IDENTIFIER, + buffer_type); +} + +Expected HailoRTDriver::vdma_buffer_map(uintptr_t user_address, size_t required_size, + DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle, + DmaBufferType buffer_type) { std::unique_lock mapping_lock(m_mapped_buffer_lock); auto mapped_buffer = std::find_if(m_mapped_buffer.begin(), m_mapped_buffer.end(), @@ -464,7 +562,8 @@ Expected HailoRTDriver::vdma_buffer_map(void *u return Expected(mapped_buffer->handle); } else { // Buffer not mapped, map it now - auto handle = vdma_buffer_map_ioctl(user_address, required_size, data_direction, driver_buff_handle); + auto handle = vdma_buffer_map_ioctl(user_address, required_size, data_direction, + driver_buff_handle, buffer_type); CHECK_EXPECTED(handle); const auto mapping_count = 1; @@ -498,7 +597,7 @@ hailo_status HailoRTDriver::vdma_buffer_unmap(VdmaBufferHandle handle) { return HAILO_SUCCESS; } -hailo_status HailoRTDriver::vdma_buffer_unmap(void *user_address, size_t size, DmaDirection data_direction) +hailo_status HailoRTDriver::vdma_buffer_unmap(uintptr_t user_address, size_t size, DmaDirection data_direction) { std::unique_lock mapping_lock(m_mapped_buffer_lock); auto mapped_buffer = std::find_if(m_mapped_buffer.begin(), m_mapped_buffer.end(), @@ -541,60 +640,28 @@ hailo_status HailoRTDriver::vdma_buffer_sync(VdmaBufferHandle handle, DmaSyncDir #endif } -Expected HailoRTDriver::descriptors_list_create(size_t desc_count, uint16_t desc_page_size, - bool is_circular) -{ - uintptr_t desc_handle = INVALID_DRIVER_HANDLE_VALUE; - uint64_t dma_address = 0; - TRY(std::tie(desc_handle, dma_address), - descriptors_list_create_ioctl(desc_count, desc_page_size, is_circular)); - - auto user_address = descriptors_list_create_mmap(desc_handle, desc_count); - if (!user_address) { - auto status = descriptors_list_release_ioctl(desc_handle); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed releasing descriptors list, status {}", status); - // continue - } - return make_unexpected(user_address.status()); - } - - return DescriptorsListInfo{desc_handle, dma_address, desc_count, user_address.release()}; -} - -hailo_status HailoRTDriver::descriptors_list_release(const DescriptorsListInfo &descriptors_list_info) +hailo_status HailoRTDriver::descriptors_list_program(uintptr_t desc_handle, VdmaBufferHandle buffer_handle, + size_t buffer_size, size_t buffer_offset, uint8_t channel_index, uint32_t starting_desc, bool should_bind, + InterruptsDomain last_desc_interrupts) { - hailo_status status = HAILO_SUCCESS; - - auto unmap_status = descriptors_list_create_munmap(descriptors_list_info.user_address, descriptors_list_info.desc_count); - if (HAILO_SUCCESS != unmap_status) { - LOGGER__ERROR("Descriptors list unmap failed with {}", unmap_status); - status = unmap_status; - // continue - } + hailo_desc_list_program_params params{}; + params.buffer_handle = buffer_handle; + params.buffer_size = buffer_size; + params.buffer_offset = buffer_offset; + params.desc_handle = desc_handle; + params.channel_index = channel_index; + params.starting_desc = starting_desc; - auto release_status = descriptors_list_release_ioctl(descriptors_list_info.handle); - if (HAILO_SUCCESS != release_status) { - LOGGER__ERROR("Descriptors list release status failed with {}", release_status); - status = release_status; - // continue - } + params.should_bind = should_bind; + params.last_interrupts_domain = (hailo_vdma_interrupts_domain)last_desc_interrupts; - return status; -} +#ifdef NDEBUG + params.is_debug = false; +#else + params.is_debug = true; +#endif -hailo_status HailoRTDriver::descriptors_list_bind_vdma_buffer(uintptr_t desc_handle, VdmaBufferHandle buffer_handle, - size_t buffer_size, size_t buffer_offset, uint8_t channel_index, uint32_t starting_desc) -{ - hailo_desc_list_bind_vdma_buffer_params config_info{}; - config_info.buffer_handle = buffer_handle; - config_info.buffer_size = buffer_size; - config_info.buffer_offset = buffer_offset; - config_info.desc_handle = desc_handle; - config_info.channel_index = channel_index; - config_info.starting_desc = starting_desc; - - CHECK_IOCTL_RESULT(run_ioctl(HAILO_DESC_LIST_BIND_VDMA_BUFFER, &config_info), "Failed bind buffer to desc list"); + CHECK_IOCTL_RESULT(run_ioctl(HAILO_DESC_LIST_PROGRAM, ¶ms), "Failed bind buffer to desc list"); return HAILO_SUCCESS; } @@ -627,7 +694,12 @@ Expected HailoRTDriver::launch_transfer(vdma::ChannelId channel_id, ui params.is_debug = true; #endif - CHECK_IOCTL_RESULT(run_ioctl(HAILO_VDMA_LAUNCH_TRANSFER, ¶ms), "Failed launch transfer"); + int err = run_ioctl(HAILO_VDMA_LAUNCH_TRANSFER, ¶ms); + if ((0 != err) && (-ECONNRESET == params.launch_transfer_status)) { + return make_unexpected(HAILO_STREAM_ABORT); + } + CHECK(0 == err, HAILO_DRIVER_FAIL, "Failed launch transfer errno: {}", err); + return Expected(params.descs_programed); } @@ -731,6 +803,50 @@ hailo_status HailoRTDriver::mark_as_used() return params.in_use ? HAILO_DEVICE_IN_USE : HAILO_SUCCESS; } +Expected> HailoRTDriver::soc_connect(uintptr_t input_buffer_desc_handle, + uintptr_t output_buffer_desc_handle) +{ + hailo_soc_connect_params params{}; + params.input_desc_handle = input_buffer_desc_handle; + params.output_desc_handle = output_buffer_desc_handle; + CHECK_IOCTL_RESULT(run_ioctl(HAILO_SOC_CONNECT, ¶ms), "Failed soc_connect"); + vdma::ChannelId input_channel{0, params.input_channel_index}; + vdma::ChannelId output_channel{0, params.output_channel_index}; + return std::make_pair(input_channel, output_channel); +} + +Expected> HailoRTDriver::pci_ep_accept(uintptr_t input_buffer_desc_handle, uintptr_t output_buffer_desc_handle) +{ + hailo_pci_ep_accept_params params{}; + params.input_desc_handle = input_buffer_desc_handle; + params.output_desc_handle = output_buffer_desc_handle; + CHECK_IOCTL_RESULT(run_ioctl(HAILO_PCI_EP_ACCEPT, ¶ms), "Failed pci_ep accept"); + vdma::ChannelId input_channel{0, params.input_channel_index}; + vdma::ChannelId output_channel{0, params.output_channel_index}; + return std::make_pair(input_channel, output_channel); +} + +hailo_status HailoRTDriver::close_connection(vdma::ChannelId input_channel, vdma::ChannelId output_channel, + PcieSessionType session_type) +{ + if (PcieSessionType::SERVER == session_type) { + hailo_pci_ep_close_params params{}; + params.input_channel_index = input_channel.channel_index; + params.output_channel_index = output_channel.channel_index; + CHECK_IOCTL_RESULT(run_ioctl(HAILO_PCI_EP_CLOSE, ¶ms), "Failed pci_ep_close"); + return HAILO_SUCCESS; + } else if (PcieSessionType::CLIENT == session_type) { + hailo_soc_close_params params{}; + params.input_channel_index = input_channel.channel_index; + params.output_channel_index = output_channel.channel_index; + CHECK_IOCTL_RESULT(run_ioctl(HAILO_SOC_CLOSE, ¶ms), "Failed soc_close"); + return HAILO_SUCCESS; + } else { + LOGGER__ERROR("close_connection not supported with session type {}", session_type); + return HAILO_NOT_SUPPORTED; + } +} + #if defined(__linux__) static bool is_blocking_ioctl(unsigned long request) { @@ -818,13 +934,15 @@ hailo_status HailoRTDriver::write_memory_ioctl(MemoryType memory_type, uint64_t } #if defined(__linux__) || defined(_WIN32) -Expected HailoRTDriver::vdma_buffer_map_ioctl(void *user_address, size_t required_size, - DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle) +Expected HailoRTDriver::vdma_buffer_map_ioctl(uintptr_t user_address, size_t required_size, + DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle, + DmaBufferType buffer_type) { hailo_vdma_buffer_map_params map_user_buffer_info{}; map_user_buffer_info.user_address = user_address; map_user_buffer_info.size = required_size; map_user_buffer_info.data_direction = direction_to_dma_data_direction(data_direction); + map_user_buffer_info.buffer_type = driver_dma_buffer_type_to_dma_buffer_type(buffer_type); map_user_buffer_info.allocated_buffer_handle = driver_buff_handle; map_user_buffer_info.mapped_handle = 0; @@ -833,8 +951,9 @@ Expected HailoRTDriver::vdma_buffer_map_ioctl(v return std::move(map_user_buffer_info.mapped_handle); } #elif defined(__QNX__) -Expected HailoRTDriver::vdma_buffer_map_ioctl(void *user_address, size_t required_size, - DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle) +Expected HailoRTDriver::vdma_buffer_map_ioctl(uintptr_t user_address, size_t required_size, + DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle, + DmaBufferType buffer_type) { // Mapping is done by the driver_buff_handle (shm file descriptor), and not by address. (void)user_address; @@ -854,6 +973,7 @@ Expected HailoRTDriver::vdma_buffer_map_ioctl(v .shared_memory_handle = shm_handle, .size = required_size, .data_direction = direction_to_dma_data_direction(data_direction), + .buffer_type = driver_dma_buffer_type_to_dma_buffer_type(buffer_type), .allocated_buffer_handle = INVALID_DRIVER_HANDLE_VALUE, .mapped_handle = 0 }; @@ -882,7 +1002,7 @@ hailo_status HailoRTDriver::vdma_buffer_unmap_ioctl(VdmaBufferHandle handle) return HAILO_SUCCESS; } -Expected> HailoRTDriver::descriptors_list_create_ioctl(size_t desc_count, +Expected HailoRTDriver::descriptors_list_create(size_t desc_count, uint16_t desc_page_size, bool is_circular) { CHECK(is_powerof2(desc_page_size), HAILO_INVALID_ARGUMENT, "Invalid desc page size {}", desc_page_size); @@ -894,92 +1014,17 @@ Expected> HailoRTDriver::descriptors_list_create_ CHECK_IOCTL_RESULT(run_ioctl(HAILO_DESC_LIST_CREATE, &create_desc_info), "Failed create desc list"); - return std::make_pair(create_desc_info.desc_handle, create_desc_info.dma_address); + return DescriptorsListInfo{create_desc_info.desc_handle, create_desc_info.dma_address}; } -hailo_status HailoRTDriver::descriptors_list_release_ioctl(uintptr_t desc_handle) +hailo_status HailoRTDriver::descriptors_list_release(const DescriptorsListInfo &desc_info) { struct hailo_desc_list_release_params params{}; - params.desc_handle = desc_handle; + params.desc_handle = desc_info.handle; CHECK_IOCTL_RESULT(run_ioctl(HAILO_DESC_LIST_RELEASE, ¶ms), "Failed release desc list"); return HAILO_SUCCESS; } -#if defined(__linux__) -Expected HailoRTDriver::descriptors_list_create_mmap(uintptr_t desc_handle, size_t desc_count) -{ - // We lock m_driver_lock before calling mmap. Read m_driver_lock doc in the header - std::unique_lock lock(m_driver_lock); - - const size_t buffer_size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR; - void *address = mmap(nullptr, buffer_size, PROT_WRITE | PROT_READ, MAP_SHARED, m_fd, (off_t)desc_handle); - if (MAP_FAILED == address) { - LOGGER__ERROR("Failed to map descriptors list buffer with errno: {}", errno); - return make_unexpected(HAILO_DRIVER_FAIL); - } - return address; -} - -hailo_status HailoRTDriver::descriptors_list_create_munmap(void *address, size_t desc_count) -{ - const size_t buffer_size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR; - if (0 != munmap(address, buffer_size)) { - LOGGER__ERROR("munmap of address {}, length: {} failed with errno: {}", address, buffer_size, errno); - return HAILO_DRIVER_FAIL; - } - return HAILO_SUCCESS; -} - -#elif defined(__QNX__) - -Expected HailoRTDriver::descriptors_list_create_mmap(uintptr_t desc_handle, size_t desc_count) -{ - const size_t buffer_size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR; - struct hailo_non_linux_desc_list_mmap_params map_vdma_list_params { - .desc_handle = desc_handle, - .size = buffer_size, - .user_address = nullptr, - }; - - CHECK_IOCTL_RESULT(run_ioctl(HAILO_NON_LINUX_DESC_LIST_MMAP, &map_vdma_list_params), "Failed mmap descriptors list"); - - void *address = mmap(nullptr, buffer_size, PROT_WRITE | PROT_READ | PROT_NOCACHE, MAP_SHARED | MAP_PHYS, NOFD, - (off_t)map_vdma_list_params.user_address); - CHECK_AS_EXPECTED(MAP_FAILED != address, HAILO_INTERNAL_FAILURE, "Failed to mmap buffer fd with errno:{}", errno); - - return address; -} - -hailo_status HailoRTDriver::descriptors_list_create_munmap(void *address, size_t desc_count) -{ - const size_t buffer_size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR; - if (0 != munmap(address, buffer_size)) { - LOGGER__ERROR("munmap of address {}, length: {} failed with errno: {}", address, buffer_size, errno); - return HAILO_DRIVER_FAIL; - } - return HAILO_SUCCESS; -} - -#elif defined(_WIN32) -Expected HailoRTDriver::descriptors_list_create_mmap(uintptr_t desc_handle, size_t desc_count) -{ - hailo_non_linux_desc_list_mmap_params params{}; - params.desc_handle = desc_handle; - params.size = desc_count * SIZE_OF_SINGLE_DESCRIPTOR; - CHECK_IOCTL_RESULT(run_ioctl(HAILO_NON_LINUX_DESC_LIST_MMAP, ¶ms), "Failed mmap desc list"); - void *user_address = params.user_address; - return user_address; -} - -hailo_status HailoRTDriver::descriptors_list_create_munmap(void *, size_t ) -{ - // On windows, the unmap is done on the release ioctl - return HAILO_SUCCESS; -} -#else -#error "unsupported platform!" -#endif - #if defined(__linux__) Expected> HailoRTDriver::continous_buffer_alloc_ioctl(size_t size) diff --git a/hailort/libhailort/src/vdma/driver/hailort_driver.hpp b/hailort/libhailort/src/vdma/driver/hailort_driver.hpp index b5f99f9..75d4746 100755 --- a/hailort/libhailort/src/vdma/driver/hailort_driver.hpp +++ b/hailort/libhailort/src/vdma/driver/hailort_driver.hpp @@ -58,8 +58,6 @@ constexpr uint8_t MAX_H2D_CHANNEL_INDEX = 15; constexpr uint8_t MIN_D2H_CHANNEL_INDEX = MAX_H2D_CHANNEL_INDEX + 1; constexpr uint8_t MAX_D2H_CHANNEL_INDEX = 31; -constexpr size_t SIZE_OF_SINGLE_DESCRIPTOR = 0x10; - // NOTE: don't change members from this struct without updating all code using it (platform specific) struct ChannelInterruptTimestamp { std::chrono::nanoseconds timestamp; @@ -74,7 +72,7 @@ struct ChannelInterruptTimestampList { struct ChannelIrqData { vdma::ChannelId channel_id; bool is_active; - uint16_t desc_num_processed; + uint8_t transfers_completed; uint8_t host_error; uint8_t device_error; bool validation_success; @@ -101,8 +99,6 @@ using vdma_mapped_buffer_driver_identifier = int; struct DescriptorsListInfo { uintptr_t handle; // Unique identifier for the driver. uint64_t dma_address; - size_t desc_count; - void *user_address; }; struct ContinousBufferInfo { @@ -135,9 +131,16 @@ class HailoRTDriver final { public: + enum class AcceleratorType { + NNC_ACCELERATOR, + SOC_ACCELERATOR, + ACC_TYPE_MAX_VALUE + }; + struct DeviceInfo { std::string dev_path; std::string device_id; + enum AcceleratorType accelerator_type; }; enum class DmaDirection { @@ -146,6 +149,11 @@ public: BOTH }; + enum class DmaBufferType { + USER_PTR_BUFFER = 0, + DMABUF_BUFFER + }; + enum class DmaSyncDirection { TO_HOST = 0, TO_DEVICE @@ -153,7 +161,8 @@ public: enum class DmaType { PCIE, - DRAM + DRAM, + PCIE_EP }; enum class MemoryType { @@ -173,21 +182,39 @@ public: DMA_ENGINE0, DMA_ENGINE1, DMA_ENGINE2, + + // PCIe EP driver memories + PCIE_EP_CONFIG, + PCIE_EP_BRIDGE, + }; + + enum class PcieSessionType { + CLIENT, // (soc) + SERVER // (A53 - pci_ep) }; using VdmaBufferHandle = size_t; - static Expected> create(const DeviceInfo &device_info); + static Expected> create(const std::string &device_id, const std::string &dev_path); - ~HailoRTDriver(); + static Expected> create_pcie(const std::string &device_id); + + static Expected> create_integrated_nnc(); + static bool is_integrated_nnc_loaded(); + + static Expected> create_pcie_ep(); + static bool is_pcie_ep_loaded(); static Expected> scan_devices(); + static Expected> scan_devices(AcceleratorType accelerator_type); + + ~HailoRTDriver(); hailo_status read_memory(MemoryType memory_type, uint64_t address, void *buf, size_t size); hailo_status write_memory(MemoryType memory_type, uint64_t address, const void *buf, size_t size); - hailo_status vdma_interrupts_enable(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure); - hailo_status vdma_interrupts_disable(const ChannelsBitmap &channel_id); + hailo_status vdma_enable_channels(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure); + hailo_status vdma_disable_channels(const ChannelsBitmap &channel_id); Expected vdma_interrupts_wait(const ChannelsBitmap &channels_bitmap); Expected vdma_interrupts_read_timestamps(vdma::ChannelId channel_id); @@ -211,6 +238,17 @@ public: hailo_status reset_nn_core(); + /** + * Maps a dmabuf to physical memory. + * + * @param[in] dmabuf_fd - File decsriptor to the dmabuf. + * @param[in] required_size - size of dmabug we are mapping. + * @param[in] data_direction - direction is used for optimization. + * @param[in] buffer_type - buffer type must be DMABUF + */ + Expected vdma_buffer_map_dmabuf(int dmabuf_fd, size_t required_size, DmaDirection data_direction, + DmaBufferType buffer_type); + /** * Pins a page aligned user buffer to physical memory, creates an IOMMU mapping (pci_mag_sg). * The buffer is used for streaming D2H or H2D using DMA. @@ -219,14 +257,14 @@ public: * @param[in] driver_buff_handle - handle to driver allocated buffer - INVALID_DRIVER_BUFFER_HANDLE_VALUE in case * of user allocated buffer */ - Expected vdma_buffer_map(void *user_address, size_t required_size, DmaDirection data_direction, - const vdma_mapped_buffer_driver_identifier &driver_buff_handle); + Expected vdma_buffer_map(uintptr_t user_address, size_t required_size, DmaDirection data_direction, + const vdma_mapped_buffer_driver_identifier &driver_buff_handle, DmaBufferType buffer_type); /** * Unmaps user buffer mapped using HailoRTDriver::map_buffer. */ hailo_status vdma_buffer_unmap(VdmaBufferHandle handle); - hailo_status vdma_buffer_unmap(void *user_address, size_t size, DmaDirection data_direction); + hailo_status vdma_buffer_unmap(uintptr_t user_address, size_t size, DmaDirection data_direction); hailo_status vdma_buffer_sync(VdmaBufferHandle buffer, DmaSyncDirection sync_direction, size_t offset, size_t count); @@ -247,11 +285,11 @@ public: hailo_status descriptors_list_release(const DescriptorsListInfo &descriptors_list_info); /** - * Configure vdma channel descriptors to point to the given user address. + * Program the given descriptors list to point to the given buffer. */ - hailo_status descriptors_list_bind_vdma_buffer(uintptr_t desc_handle, VdmaBufferHandle buffer_handle, + hailo_status descriptors_list_program(uintptr_t desc_handle, VdmaBufferHandle buffer_handle, size_t buffer_size, size_t buffer_offset, uint8_t channel_index, - uint32_t starting_desc); + uint32_t starting_desc, bool should_bind, InterruptsDomain last_desc_interrupts); struct TransferBuffer { VdmaBufferHandle buffer_handle; @@ -288,9 +326,18 @@ public: */ hailo_status mark_as_used(); + Expected> soc_connect(uintptr_t input_buffer_desc_handle, + uintptr_t output_buffer_desc_handle); + + Expected> pci_ep_accept(uintptr_t input_buffer_desc_handle, + uintptr_t output_buffer_desc_handle); + + hailo_status close_connection(vdma::ChannelId input_channel, vdma::ChannelId output_channel, + PcieSessionType session_type); + const std::string &device_id() const { - return m_device_info.device_id; + return m_device_id; } inline DmaType dma_type() const @@ -330,6 +377,9 @@ public: static const uint8_t INVALID_VDMA_CHANNEL_INDEX; static const vdma_mapped_buffer_driver_identifier INVALID_MAPPED_BUFFER_DRIVER_IDENTIFIER; + static constexpr const char *INTEGRATED_NNC_DEVICE_ID = "[integrated]"; + static constexpr const char *PCIE_EP_DEVICE_ID = "[pci_ep]"; + private: template int run_ioctl(uint32_t ioctl_code, PointerType param); @@ -337,22 +387,17 @@ private: hailo_status read_memory_ioctl(MemoryType memory_type, uint64_t address, void *buf, size_t size); hailo_status write_memory_ioctl(MemoryType memory_type, uint64_t address, const void *buf, size_t size); - Expected vdma_buffer_map_ioctl(void *user_address, size_t required_size, - DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle); + Expected vdma_buffer_map_ioctl(uintptr_t user_address, size_t required_size, + DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle, + DmaBufferType buffer_type); hailo_status vdma_buffer_unmap_ioctl(VdmaBufferHandle handle); - Expected> descriptors_list_create_ioctl(size_t desc_count, uint16_t desc_page_size, - bool is_circular); - hailo_status descriptors_list_release_ioctl(uintptr_t desc_handle); - Expected descriptors_list_create_mmap(uintptr_t desc_handle, size_t desc_count); - hailo_status descriptors_list_create_munmap(void *address, size_t desc_count); - Expected> continous_buffer_alloc_ioctl(size_t size); hailo_status continous_buffer_free_ioctl(uintptr_t desc_handle); Expected continous_buffer_mmap(uintptr_t desc_handle, size_t size); hailo_status continous_buffer_munmap(void *address, size_t size); - HailoRTDriver(const DeviceInfo &device_info, FileDescriptor &&fd, hailo_status &status); + HailoRTDriver(const std::string &device_id, FileDescriptor &&fd, hailo_status &status); bool is_valid_channel_id(const vdma::ChannelId &channel_id); bool is_valid_channels_bitmap(const ChannelsBitmap &bitmap) @@ -369,6 +414,7 @@ private: FileDescriptor m_fd; DeviceInfo m_device_info; + std::string m_device_id; uint16_t m_desc_max_page_size; DmaType m_dma_type; bool m_allocate_driver_buffer; @@ -390,7 +436,7 @@ private: // TODO HRT-11937: when ioctl is combined, move caching to driver struct MappedBufferInfo { VdmaBufferHandle handle; - void *address; + uintptr_t address; DmaDirection direction; size_t size; vdma_mapped_buffer_driver_identifier driver_buff_handle; diff --git a/hailort/libhailort/src/vdma/driver/os/driver_os_specific.hpp b/hailort/libhailort/src/vdma/driver/os/driver_os_specific.hpp index 01a40d6..610749c 100644 --- a/hailort/libhailort/src/vdma/driver/os/driver_os_specific.hpp +++ b/hailort/libhailort/src/vdma/driver/os/driver_os_specific.hpp @@ -22,8 +22,9 @@ namespace hailort { Expected open_device_file(const std::string &path); -Expected> list_devices(); Expected query_device_info(const std::string &device_name); +Expected> scan_nnc_devices(); +Expected> scan_soc_devices(); #ifndef _WIN32 diff --git a/hailort/libhailort/src/vdma/driver/os/posix/linux/driver_os_specific.cpp b/hailort/libhailort/src/vdma/driver/os/posix/linux/driver_os_specific.cpp index f66c6e2..4dbea85 100644 --- a/hailort/libhailort/src/vdma/driver/os/posix/linux/driver_os_specific.cpp +++ b/hailort/libhailort/src/vdma/driver/os/posix/linux/driver_os_specific.cpp @@ -24,6 +24,7 @@ namespace hailort #define HAILO_CLASS_PATH ("/sys/class/hailo_chardev") #define HAILO_BOARD_LOCATION_FILENAME ("board_location") +#define HAILO_BOARD_ACCELERATOR_TYPE_FILENAME ("accelerator_type") Expected open_device_file(const std::string &path) { @@ -62,20 +63,77 @@ Expected> list_devices() return devices; } +Expected> scan_devices() +{ + TRY_V(auto devices, list_devices(), "Failed listing pcie devices"); + + std::vector devices_info; + for (const auto &device_name : devices) { + TRY(auto device_info, query_device_info(device_name), "Failed parsing device info for {}", device_name); + devices_info.push_back(device_info); + } + + return devices_info; +} + +Expected> scan_soc_devices() +{ + TRY(auto all_devices_info, scan_devices()); + + // TODO- HRT-14078: change to be based on different classes + std::vector soc_devices_info; + for (const auto &device_info : all_devices_info) { + if (HailoRTDriver::AcceleratorType::SOC_ACCELERATOR == device_info.accelerator_type) { + soc_devices_info.push_back(device_info); + } + } + + return soc_devices_info; +} + +Expected> scan_nnc_devices() +{ + TRY(auto all_devices_info, scan_devices()); + + // TODO- HRT-14078: change to be based on different classes + std::vector nnc_devices_info; + for (const auto &device_info : all_devices_info) { + if (HailoRTDriver::AcceleratorType::NNC_ACCELERATOR == device_info.accelerator_type) { + nnc_devices_info.push_back(device_info); + } + } + + return nnc_devices_info; +} + +Expected get_line_from_file(const std::string &file_path) +{ + std::ifstream file(file_path); + CHECK_AS_EXPECTED(file.good(), HAILO_DRIVER_FAIL, "Failed open {}", file_path); + + std::string line; + std::getline(file, line); + CHECK_AS_EXPECTED(file.eof(), HAILO_DRIVER_FAIL, "Failed read {}", file_path); + + return line; +} + Expected query_device_info(const std::string &device_name) { const std::string device_id_path = std::string(HAILO_CLASS_PATH) + "/" + device_name + "/" + HAILO_BOARD_LOCATION_FILENAME; - std::ifstream device_id_file(device_id_path); - CHECK_AS_EXPECTED(device_id_file.good(), HAILO_DRIVER_FAIL, "Failed open {}", device_id_path); + TRY(auto device_id , get_line_from_file(device_id_path)); + + const std::string accelerator_type_path = std::string(HAILO_CLASS_PATH) + "/" + + device_name + "/" + HAILO_BOARD_ACCELERATOR_TYPE_FILENAME; + TRY(auto accelerator_type_s, get_line_from_file(accelerator_type_path)); - std::string device_id; - std::getline(device_id_file, device_id); - CHECK_AS_EXPECTED(device_id_file.eof(), HAILO_DRIVER_FAIL, "Failed read {}", device_id_path); + int accelerator_type = std::stoi(accelerator_type_s); HailoRTDriver::DeviceInfo device_info = {}; device_info.dev_path = std::string("/dev/") + device_name; device_info.device_id = device_id; + device_info.accelerator_type = static_cast(accelerator_type); return device_info; } diff --git a/hailort/libhailort/src/vdma/driver/os/windows/driver_os_specific.cpp b/hailort/libhailort/src/vdma/driver/os/windows/driver_os_specific.cpp index 04e8c23..cc0f30d 100644 --- a/hailort/libhailort/src/vdma/driver/os/windows/driver_os_specific.cpp +++ b/hailort/libhailort/src/vdma/driver/os/windows/driver_os_specific.cpp @@ -157,10 +157,9 @@ Expected open_device_file(const std::string &dev_path) return FileDescriptor(handle); } -Expected> list_devices() -{ - GUID guid = GUID_DEVINTERFACE_HailoKM; +Expected> list_devices(GUID guid) +{ ULONG len = 0; CONFIGRET cr = CM_Get_Device_Interface_List_SizeA( &len, @@ -189,6 +188,37 @@ Expected> list_devices() return names; } +Expected> scan_devices(GUID guid) +{ + TRY (auto names, list_devices(guid), "Failed listing pcie devices"); + + std::vector devices_info; + for (const auto &name : names) { + auto device_info = query_device_info(name); + CHECK_EXPECTED(device_info, "Failed parsing device info for {}", name); + if (GUID_DEVINTERFACE_HailoKM_NNC == guid) { + device_info->accelerator_type = HailoRTDriver::AcceleratorType::NNC_ACCELERATOR; + } else if (GUID_DEVINTERFACE_HailoKM_SOC == guid) { + device_info->accelerator_type = HailoRTDriver::AcceleratorType::SOC_ACCELERATOR; + } + devices_info.push_back(device_info.release()); + } + + return devices_info; +} + +Expected> scan_soc_devices() +{ + GUID guid = GUID_DEVINTERFACE_HailoKM_SOC; + return scan_devices(guid); +} + +Expected> scan_nnc_devices() +{ + GUID guid = GUID_DEVINTERFACE_HailoKM_NNC; + return scan_devices(guid); +} + static Expected parse_uint32_property(const std::wstring &dev_interface, const DEVPROPKEY* key) { @@ -241,8 +271,8 @@ Expected query_device_info(const std::string &device_ } COMPATIBLE_PARAM_CAST(hailo_memory_transfer_params, MemoryTransfer); -COMPATIBLE_PARAM_CAST(hailo_vdma_interrupts_enable_params, VdmaInterruptsEnable) -COMPATIBLE_PARAM_CAST(hailo_vdma_interrupts_disable_params, VdmaInterruptsDisable) +COMPATIBLE_PARAM_CAST(hailo_vdma_enable_channels_params, VdmaEnableChannels) +COMPATIBLE_PARAM_CAST(hailo_vdma_disable_channels_params, VdmaDisableChannels) COMPATIBLE_PARAM_CAST(hailo_vdma_interrupts_read_timestamp_params, VdmaInterruptsReadTimestamps) COMPATIBLE_PARAM_CAST(hailo_vdma_interrupts_wait_params, VdmaInterruptsWait) COMPATIBLE_PARAM_CAST(hailo_vdma_buffer_sync_params, VdmaBufferSync) @@ -251,14 +281,17 @@ COMPATIBLE_PARAM_CAST(hailo_vdma_buffer_map_params, VdmaBufferMap) COMPATIBLE_PARAM_CAST(hailo_vdma_buffer_unmap_params, VdmaBufferUnmap) COMPATIBLE_PARAM_CAST(hailo_desc_list_create_params, DescListCreate) COMPATIBLE_PARAM_CAST(hailo_desc_list_release_params, DescListReleaseParam) -COMPATIBLE_PARAM_CAST(hailo_desc_list_bind_vdma_buffer_params, DescListBind) +COMPATIBLE_PARAM_CAST(hailo_desc_list_program_params, DescListProgram) COMPATIBLE_PARAM_CAST(hailo_d2h_notification, D2HNotification) COMPATIBLE_PARAM_CAST(hailo_device_properties, DeviceProperties) COMPATIBLE_PARAM_CAST(hailo_driver_info, DriverInfo) -COMPATIBLE_PARAM_CAST(hailo_non_linux_desc_list_mmap_params, DescListMmap) COMPATIBLE_PARAM_CAST(hailo_read_log_params, ReadLog) COMPATIBLE_PARAM_CAST(hailo_mark_as_in_use_params, MarkAsInUse) COMPATIBLE_PARAM_CAST(hailo_vdma_launch_transfer_params, LaunchTransfer) +COMPATIBLE_PARAM_CAST(hailo_soc_connect_params, ConnectParams) +COMPATIBLE_PARAM_CAST(hailo_soc_close_params, SocCloseParams) +COMPATIBLE_PARAM_CAST(hailo_pci_ep_accept_params, AcceptParams) +COMPATIBLE_PARAM_CAST(hailo_pci_ep_close_params, PciEpCloseParams) // Special handle for nullptr_t. This case occurs when there is no parameters passed. tCompatibleHailoIoctlData WindowsIoctlParamCast::to_compatible(nullptr_t data) diff --git a/hailort/libhailort/src/vdma/integrated/integrated_device.cpp b/hailort/libhailort/src/vdma/integrated/integrated_device.cpp index 6b14d07..922f661 100644 --- a/hailort/libhailort/src/vdma/integrated/integrated_device.cpp +++ b/hailort/libhailort/src/vdma/integrated/integrated_device.cpp @@ -9,39 +9,68 @@ #include "md5.h" #include -static const std::string INTEGRATED_NNC_DRIVER_PATH = "/dev/hailo_integrated_nnc"; - namespace hailort { +// 16 MB +#define INTEGRATED_DEVICE_INFINITE_ACTION_LIST_POOL_SIZE (16777216) + bool IntegratedDevice::is_loaded() { -#if defined(_MSC_VER) - // windows is not supported for core driver - return false; -#else - return (access(INTEGRATED_NNC_DRIVER_PATH.c_str(), F_OK) == 0); -#endif // defined(_MSC_VER) + return HailoRTDriver::is_integrated_nnc_loaded(); +} + +Expected> IntegratedDevice::allocate_infinite_action_list_buffer(size_t size) +{ + CHECK_AS_EXPECTED(0 == (size % OsUtils::get_page_size()), HAILO_INVALID_ARGUMENT, + "Infinte action list buffer size must be a multiple of page size"); + CHECK_AS_EXPECTED(m_device_infinite_action_list_pool_allocation_offset + size <= m_device_infinite_action_list_pool.size(), + HAILO_INVALID_ARGUMENT, "Buffer pool size is too small for requested infinte action list buffer"); + + auto user_addres = static_cast(reinterpret_cast(m_device_infinite_action_list_pool.user_address()) + + m_device_infinite_action_list_pool_allocation_offset); + auto dma_address = m_device_infinite_action_list_pool.dma_address() + m_device_infinite_action_list_pool_allocation_offset; + + m_device_infinite_action_list_pool_allocation_offset += size; + + return std::make_pair(user_addres, dma_address); } Expected> IntegratedDevice::create() { hailo_status status = HAILO_UNINITIALIZED; - const HailoRTDriver::DeviceInfo device_info {INTEGRATED_NNC_DRIVER_PATH, DEVICE_ID}; - auto driver = HailoRTDriver::create(device_info); - CHECK_EXPECTED(driver, "Failed to initialize HailoRTDriver"); + TRY(auto driver, HailoRTDriver::create_integrated_nnc()); + + // Create pool of memory for infinite action list so can all be in the LUT memory area + // TODO: remove this when infinite action list allocates from its own pool of CMA memory + TRY(auto infinite_action_list_pool, vdma::ContinuousBuffer::create(INTEGRATED_DEVICE_INFINITE_ACTION_LIST_POOL_SIZE, + *driver)); + + // Verify pool is in mapped range + CHECK_AS_EXPECTED(DDRActionListBufferBuilder::verify_dma_addr(infinite_action_list_pool), HAILO_INTERNAL_FAILURE, + "Failed to allocate continous buffer pool M4 mapped memory region"); + - auto device = std::unique_ptr(new (std::nothrow) IntegratedDevice(driver.release(), status)); + auto device = std::unique_ptr(new (std::nothrow) IntegratedDevice(std::move(driver), + std::move(infinite_action_list_pool), status)); CHECK_AS_EXPECTED((nullptr != device), HAILO_OUT_OF_HOST_MEMORY); CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating IntegratedDevice"); return device; } -IntegratedDevice::IntegratedDevice(std::unique_ptr &&driver, hailo_status &status) : - VdmaDevice::VdmaDevice(std::move(driver), Device::Type::INTEGRATED) +IntegratedDevice::IntegratedDevice(std::unique_ptr &&driver, vdma::ContinuousBuffer &&pool, + hailo_status &status) : + VdmaDevice::VdmaDevice(std::move(driver), Device::Type::INTEGRATED, status), + m_device_infinite_action_list_pool(std::move(pool)), + m_device_infinite_action_list_pool_allocation_offset(0) { + if (status != HAILO_SUCCESS) { + LOGGER__ERROR("Failed to create VdmaDevice"); + return; + } + status = update_fw_state(); if (HAILO_SUCCESS != status) { LOGGER__ERROR("update_fw_state() failed with status {}", status); diff --git a/hailort/libhailort/src/vdma/integrated/integrated_device.hpp b/hailort/libhailort/src/vdma/integrated/integrated_device.hpp index eacae83..8588210 100644 --- a/hailort/libhailort/src/vdma/integrated/integrated_device.hpp +++ b/hailort/libhailort/src/vdma/integrated/integrated_device.hpp @@ -15,6 +15,7 @@ #include "hailo/hailort.h" #include "vdma/vdma_device.hpp" +#include "vdma/memory/continuous_buffer.hpp" #include @@ -46,13 +47,18 @@ public: } } - static constexpr const char *DEVICE_ID = "[integrated]"; + static constexpr const char *DEVICE_ID = HailoRTDriver::INTEGRATED_NNC_DEVICE_ID; + + Expected> allocate_infinite_action_list_buffer(size_t size); protected: virtual hailo_status reset_impl(CONTROL_PROTOCOL__reset_type_t reset_type) override; private: - IntegratedDevice(std::unique_ptr &&driver, hailo_status &status); + IntegratedDevice(std::unique_ptr &&driver, vdma::ContinuousBuffer &&pool, hailo_status &status); + + vdma::ContinuousBuffer m_device_infinite_action_list_pool; + size_t m_device_infinite_action_list_pool_allocation_offset; }; diff --git a/hailort/libhailort/src/vdma/memory/continuous_buffer.cpp b/hailort/libhailort/src/vdma/memory/continuous_buffer.cpp index c2443f0..d7e15f8 100644 --- a/hailort/libhailort/src/vdma/memory/continuous_buffer.cpp +++ b/hailort/libhailort/src/vdma/memory/continuous_buffer.cpp @@ -51,6 +51,11 @@ uint64_t ContinuousBuffer::dma_address() const return m_buffer_info.dma_address; } +void* ContinuousBuffer::user_address() const +{ + return m_buffer_info.user_address; +} + hailo_status ContinuousBuffer::read(void *buf_dst, size_t count, size_t offset) { CHECK((count + offset) <= m_buffer_info.size, HAILO_INSUFFICIENT_BUFFER, diff --git a/hailort/libhailort/src/vdma/memory/continuous_buffer.hpp b/hailort/libhailort/src/vdma/memory/continuous_buffer.hpp index 1ad50ac..62aa56a 100644 --- a/hailort/libhailort/src/vdma/memory/continuous_buffer.hpp +++ b/hailort/libhailort/src/vdma/memory/continuous_buffer.hpp @@ -49,6 +49,7 @@ public: virtual hailo_status read(void *buf_dst, size_t count, size_t offset) override; virtual hailo_status write(const void *buf_src, size_t count, size_t offset) override; + void *user_address() const; uint64_t dma_address() const; private: ContinuousBuffer(HailoRTDriver &driver, const ContinousBufferInfo &buffer_info); diff --git a/hailort/libhailort/src/vdma/memory/continuous_edge_layer.cpp b/hailort/libhailort/src/vdma/memory/continuous_edge_layer.cpp index d3feb00..7fb44a2 100644 --- a/hailort/libhailort/src/vdma/memory/continuous_edge_layer.cpp +++ b/hailort/libhailort/src/vdma/memory/continuous_edge_layer.cpp @@ -50,10 +50,12 @@ uint32_t ContinuousEdgeLayer::descs_count() const } Expected ContinuousEdgeLayer::program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, - size_t desc_offset) + size_t desc_offset, size_t buffer_offset, bool should_bind) { (void)last_desc_interrupts_domain; (void)desc_offset; + (void)buffer_offset; + (void)should_bind; // The descriptors in continuous mode are programmed by the hw, nothing to do here. return descriptors_in_buffer(transfer_size); diff --git a/hailort/libhailort/src/vdma/memory/continuous_edge_layer.hpp b/hailort/libhailort/src/vdma/memory/continuous_edge_layer.hpp index 515c4f6..3206030 100644 --- a/hailort/libhailort/src/vdma/memory/continuous_edge_layer.hpp +++ b/hailort/libhailort/src/vdma/memory/continuous_edge_layer.hpp @@ -41,7 +41,7 @@ public: virtual uint32_t descs_count() const override; virtual Expected program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, - size_t desc_offset) override; + size_t desc_offset, size_t buffer_offset = 0, bool should_bind = false) override; private: ContinuousEdgeLayer(std::shared_ptr &&buffer, size_t size, size_t offset, diff --git a/hailort/libhailort/src/vdma/memory/descriptor_list.cpp b/hailort/libhailort/src/vdma/memory/descriptor_list.cpp index 2452cb4..8f7a1e0 100644 --- a/hailort/libhailort/src/vdma/memory/descriptor_list.cpp +++ b/hailort/libhailort/src/vdma/memory/descriptor_list.cpp @@ -11,22 +11,6 @@ #include "utils.h" - -#define DESC_STATUS_REQ (1 << 0) -#define DESC_STATUS_REQ_ERR (1 << 1) -#define DESC_REQUREST_IRQ_PROCESSED (1 << 2) -#define DESC_REQUREST_IRQ_ERR (1 << 3) - -#define PCIE_DMA_HOST_INTERRUPTS_BITMASK (1 << 5) -#define PCIE_DMA_DEVICE_INTERRUPTS_BITMASK (1 << 4) - -#define DRAM_DMA_HOST_INTERRUPTS_BITMASK (1 << 4) -#define DRAM_DMA_DEVICE_INTERRUPTS_BITMASK (1 << 5) - -#define DESC_PAGE_SIZE_SHIFT (8) -#define DESC_PAGE_SIZE_MASK (0xFFFFFF00) -#define DESC_IRQ_MASK (0x0000003C) - namespace hailort { namespace vdma { @@ -53,6 +37,7 @@ Expected DescriptorList::create(uint32_t desc_count, uint16_t de DescriptorList::DescriptorList(uint32_t desc_count, uint16_t desc_page_size, bool is_circular, HailoRTDriver &driver, hailo_status &status) : m_desc_list_info(), + m_desc_count(desc_count), m_is_circular(is_circular), m_driver(driver), m_desc_page_size(desc_page_size) @@ -86,47 +71,26 @@ DescriptorList::~DescriptorList() DescriptorList::DescriptorList(DescriptorList &&other) noexcept : m_desc_list_info(), + m_desc_count(other.m_desc_count), m_is_circular(std::move(other.m_is_circular)), m_driver(other.m_driver), m_desc_page_size(other.m_desc_page_size) { m_desc_list_info.handle = std::exchange(other.m_desc_list_info.handle, 0); m_desc_list_info.dma_address = std::exchange(other.m_desc_list_info.dma_address, 0); - m_desc_list_info.desc_count = std::move(other.m_desc_list_info.desc_count); - m_desc_list_info.user_address = std::exchange(other.m_desc_list_info.user_address, nullptr); } -hailo_status DescriptorList::configure_to_use_buffer(MappedBuffer& buffer, size_t buffer_size, - size_t buffer_offset, ChannelId channel_id, uint32_t starting_desc) +hailo_status DescriptorList::program(MappedBuffer& buffer, size_t buffer_size, + size_t buffer_offset, ChannelId channel_id, uint32_t starting_desc, bool should_bind /* = true */, + InterruptsDomain last_desc_interrupts /* = InterruptsDomain::NONE */) { const auto desc_list_capacity = m_desc_page_size * count(); CHECK(buffer_size <= desc_list_capacity, HAILO_INVALID_ARGUMENT, "Can't bind a buffer larger than the descriptor list's capacity. Buffer size {}, descriptor list capacity {}", buffer_size, desc_list_capacity); - return m_driver.descriptors_list_bind_vdma_buffer(m_desc_list_info.handle, buffer.handle(), buffer_size, - buffer_offset, channel_id.channel_index, starting_desc); -} - -Expected DescriptorList::program_last_descriptor(size_t transfer_size, - InterruptsDomain last_desc_interrupts_domain, size_t desc_offset) -{ - assert(transfer_size > 0); - const auto required_descriptors = descriptors_in_buffer(transfer_size); - // Required_descriptors + desc_offset can't reach m_count. - if ((!m_is_circular) && ((required_descriptors + desc_offset) > count())){ - LOGGER__ERROR("Requested transfer size ({}) result in more descriptors than available ({})", transfer_size, count()); - return make_unexpected(HAILO_OUT_OF_DESCRIPTORS); - } - - // Program last descriptor of the transfer size - /* write residue page with the remaining buffer size*/ - auto resuide = transfer_size - (required_descriptors - 1) * m_desc_page_size; - assert(IS_FIT_IN_UINT16(resuide)); - size_t last_desc = (desc_offset + required_descriptors - 1) % count(); - program_single_descriptor(last_desc, static_cast(resuide), last_desc_interrupts_domain); - - return std::move(static_cast(required_descriptors)); + return m_driver.descriptors_list_program(m_desc_list_info.handle, buffer.handle(), buffer_size, + buffer_offset, channel_id.channel_index, starting_desc, should_bind, last_desc_interrupts); } uint32_t DescriptorList::descriptors_in_buffer(size_t buffer_size) const @@ -151,60 +115,5 @@ uint32_t DescriptorList::calculate_descriptors_count(uint32_t buffer_size, uint1 return get_nearest_powerof_2(descs_count, MIN_SG_DESCS_COUNT); } -uint32_t DescriptorList::get_interrupts_bitmask(InterruptsDomain interrupts_domain) -{ - uint32_t host_bitmask = 0; - uint32_t device_bitmask = 0; - - switch (m_driver.dma_type()) { - case HailoRTDriver::DmaType::PCIE: - host_bitmask = PCIE_DMA_HOST_INTERRUPTS_BITMASK; - device_bitmask = PCIE_DMA_DEVICE_INTERRUPTS_BITMASK; - break; - case HailoRTDriver::DmaType::DRAM: - host_bitmask = DRAM_DMA_HOST_INTERRUPTS_BITMASK; - device_bitmask = DRAM_DMA_DEVICE_INTERRUPTS_BITMASK; - break; - default: - assert(false); - } - - uint32_t bitmask = 0; - if (host_interuptes_enabled(interrupts_domain)) { - bitmask |= host_bitmask; - } - if (device_interuptes_enabled(interrupts_domain)) { - bitmask |= device_bitmask; - } - - return bitmask; -} - -void DescriptorList::program_single_descriptor(size_t desc_index, uint16_t page_size, - InterruptsDomain interrupts_domain) -{ - auto &descriptor = (*this)[desc_index]; - - // Update the descriptor's PAGE_SIZE field in the control register with the maximum size of the DMA page. - // Make all edits to the local variable local_pagesize_desc_ctrl that is on the stack to save read/writes to DDR - auto local_pagesize_desc_ctrl = static_cast(page_size << DESC_PAGE_SIZE_SHIFT) & DESC_PAGE_SIZE_MASK; - - if (InterruptsDomain::NONE != interrupts_domain) { - // Update the desc_control - local_pagesize_desc_ctrl |= (DESC_REQUREST_IRQ_PROCESSED | DESC_REQUREST_IRQ_ERR | - get_interrupts_bitmask(interrupts_domain)); -#ifndef NDEBUG - local_pagesize_desc_ctrl |= (DESC_STATUS_REQ | DESC_STATUS_REQ_ERR); -#endif - } - - descriptor.PageSize_DescControl = local_pagesize_desc_ctrl; - -#ifndef NDEBUG - // Clear status - descriptor.RemainingPageSize_Status = 0; -#endif -} - } /* namespace vdma */ } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/memory/descriptor_list.hpp b/hailort/libhailort/src/vdma/memory/descriptor_list.hpp index 7f8222a..8a14860 100644 --- a/hailort/libhailort/src/vdma/memory/descriptor_list.hpp +++ b/hailort/libhailort/src/vdma/memory/descriptor_list.hpp @@ -19,8 +19,6 @@ #include "vdma/memory/mapped_buffer.hpp" #include "vdma/driver/hailort_driver.hpp" -#include "os/mmap_buffer.hpp" - namespace hailort { namespace vdma { @@ -54,49 +52,6 @@ static_assert(is_powerof2(DEFAULT_SG_PAGE_SIZE), "DEFAULT_SG_PAGE_SIZE must be a static_assert(DEFAULT_SG_PAGE_SIZE > 0, "DEFAULT_SG_PAGE_SIZE must be larger then 0"); -static constexpr auto DESCRIPTOR_STATUS_MASK = 0xFF; -static constexpr auto DESCRIPTOR_STATUS_DONE_BIT = 0; -static constexpr auto DESCRIPTOR_STATUS_ERROR_BIT = 1; - -struct VdmaDescriptor -{ - // Struct layout is taken from PLDA spec for vDMA, and cannot be changed. - uint32_t PageSize_DescControl; - uint32_t AddrL_rsvd_DataID; - uint32_t AddrH; - uint32_t RemainingPageSize_Status; - -#ifndef NDEBUG - // Easy accessors (only on debug since we mark DESC_STATUS_REQ and DESC_STATUS_REQ_ERR are set only on debug). - uint8_t status() const - { - return RemainingPageSize_Status & DESCRIPTOR_STATUS_MASK; - } - - bool is_done() const - { - return is_bit_set(status(), DESCRIPTOR_STATUS_DONE_BIT); - } - - bool is_error() const - { - return is_bit_set(status(), DESCRIPTOR_STATUS_ERROR_BIT); - } -#endif /* NDEBUG */ -}; - -static_assert(SIZE_OF_SINGLE_DESCRIPTOR == sizeof(VdmaDescriptor), "Invalid size of descriptor"); - -inline bool host_interuptes_enabled(InterruptsDomain interrupts_domain) -{ - return 0 != (static_cast(interrupts_domain) & static_cast(InterruptsDomain::HOST)); -} - -inline bool device_interuptes_enabled(InterruptsDomain interrupts_domain) -{ - return 0 != (static_cast(interrupts_domain) & static_cast(InterruptsDomain::DEVICE)); -} - class DescriptorList { public: @@ -112,8 +67,7 @@ public: uint32_t count() const { - assert(m_desc_list_info.desc_count <= std::numeric_limits::max()); - return static_cast(m_desc_list_info.desc_count); + return m_desc_count; } uint64_t dma_address() const @@ -121,12 +75,6 @@ public: return m_desc_list_info.dma_address; } - VdmaDescriptor& operator[](size_t i) - { - assert(i < count()); - return desc_list()[i]; - } - uint16_t desc_page_size() const { return m_desc_page_size; @@ -137,22 +85,18 @@ public: return m_desc_list_info.handle; } - uint16_t max_transfers(uint32_t transfer_size) + uint16_t max_transfers(uint32_t transfer_size, bool include_bounce_buffer = false) const { + const auto descs_needed = descriptors_in_buffer(transfer_size) + (include_bounce_buffer ? 1 : 0); // We need to keep at least 1 free desc at all time. - return static_cast((count() - 1) / descriptors_in_buffer(transfer_size)); + return static_cast((count() - 1) / descs_needed); } // Map descriptors starting at offset to the start of buffer, wrapping around the descriptor list as needed // On hailo8, we allow configuring buffer without specific channel index (default is INVALID_VDMA_CHANNEL_INDEX). - hailo_status configure_to_use_buffer(MappedBuffer& buffer, size_t buffer_size, size_t buffer_offset, - ChannelId channel_id, uint32_t starting_desc = 0); - // All descritors are initialized to have size of m_desc_page_size - so all we do is set the last descritor for the - // Interrupt - and then after transfer has finished clear the previously used first and last decsriptors. - // This saves us write/ reads to the desscriptor list which is DMA memory. - Expected program_last_descriptor(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, - size_t desc_offset); - void program_single_descriptor(size_t desc_index, uint16_t page_size, InterruptsDomain interrupts_domain); + hailo_status program(MappedBuffer& buffer, size_t buffer_size, size_t buffer_offset, + ChannelId channel_id, uint32_t starting_desc = 0, bool should_bind = true, + InterruptsDomain last_desc_interrupts = InterruptsDomain::NONE); uint32_t descriptors_in_buffer(size_t buffer_size) const; static uint32_t descriptors_in_buffer(size_t buffer_size, uint16_t desc_page_size); @@ -162,12 +106,8 @@ private: DescriptorList(uint32_t desc_count, uint16_t desc_page_size, bool is_circular, HailoRTDriver &driver, hailo_status &status); - VdmaDescriptor *desc_list() { return reinterpret_cast(m_desc_list_info.user_address); } - - uint32_t get_interrupts_bitmask(InterruptsDomain interrupts_domain); - - DescriptorsListInfo m_desc_list_info; + const uint32_t m_desc_count; const bool m_is_circular; HailoRTDriver &m_driver; const uint16_t m_desc_page_size; diff --git a/hailort/libhailort/src/vdma/memory/dma_able_buffer.cpp b/hailort/libhailort/src/vdma/memory/dma_able_buffer.cpp index e76860f..5a9aaf6 100644 --- a/hailort/libhailort/src/vdma/memory/dma_able_buffer.cpp +++ b/hailort/libhailort/src/vdma/memory/dma_able_buffer.cpp @@ -14,6 +14,8 @@ #if defined(_MSC_VER) #include "os/windows/virtual_alloc_guard.hpp" +#else +#include #endif /* defined(_MSC_VER) */ @@ -272,5 +274,6 @@ Expected DmaAbleBuffer::create_by_allocation(size_t size, Hail #error "unsupported platform!" #endif + } /* namespace vdma */ } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/memory/mapped_buffer.cpp b/hailort/libhailort/src/vdma/memory/mapped_buffer.cpp index 99812ab..3fcf9cb 100644 --- a/hailort/libhailort/src/vdma/memory/mapped_buffer.cpp +++ b/hailort/libhailort/src/vdma/memory/mapped_buffer.cpp @@ -9,8 +9,6 @@ #include "mapped_buffer.hpp" -#include "vdma/vdma_device.hpp" - namespace hailort { namespace vdma { @@ -18,9 +16,10 @@ namespace vdma { Expected MappedBuffer::create_shared(DmaAbleBufferPtr buffer, HailoRTDriver &driver, HailoRTDriver::DmaDirection data_direction) { - auto status = HAILO_UNINITIALIZED; - auto result = make_shared_nothrow(driver, buffer, data_direction, status); - CHECK_SUCCESS_AS_EXPECTED(status); + TRY(auto buffer_handle, driver.vdma_buffer_map(reinterpret_cast(buffer->user_address()), buffer->size(), data_direction, + buffer->buffer_identifier(), HailoRTDriver::DmaBufferType::USER_PTR_BUFFER)); + + auto result = make_shared_nothrow(driver, buffer, data_direction, buffer_handle); CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); return result; @@ -35,25 +34,27 @@ Expected MappedBuffer::create_shared_by_allocation(size_t size, return create_shared(buffer.release(), driver, data_direction); } -MappedBuffer::MappedBuffer(HailoRTDriver &driver, DmaAbleBufferPtr buffer, - HailoRTDriver::DmaDirection data_direction, hailo_status &status) : - m_driver(driver), - m_buffer(buffer), - m_mapping_handle(HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE), - m_data_direction(data_direction) +Expected MappedBuffer::create_shared_from_dmabuf(int dmabuf_fd, size_t size, HailoRTDriver &driver, + HailoRTDriver::DmaDirection data_direction) { - auto expected_handle = driver.vdma_buffer_map(m_buffer->user_address(), m_buffer->size(), m_data_direction, - m_buffer->buffer_identifier()); - if (!expected_handle) { - LOGGER__ERROR("Mapping address {} to dma failed", m_buffer->user_address()); - status = expected_handle.status(); - return; - } + TRY(auto buffer_handle, driver.vdma_buffer_map_dmabuf(dmabuf_fd, size, data_direction, + HailoRTDriver::DmaBufferType::DMABUF_BUFFER)); - m_mapping_handle = expected_handle.release(); - status = HAILO_SUCCESS; + // TODO: if need user address for dmabuf use DmaBufDmaAbleBuffer + auto result = make_shared_nothrow(driver, nullptr, data_direction, buffer_handle); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + + return result; } +MappedBuffer::MappedBuffer(HailoRTDriver &driver, DmaAbleBufferPtr buffer, HailoRTDriver::DmaDirection data_direction, + HailoRTDriver::VdmaBufferHandle vdma_buffer_handle) : + m_driver(driver), + m_buffer(buffer), + m_mapping_handle(vdma_buffer_handle), + m_data_direction(data_direction) +{} + MappedBuffer::~MappedBuffer() { if (HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE != m_mapping_handle) { diff --git a/hailort/libhailort/src/vdma/memory/mapped_buffer.hpp b/hailort/libhailort/src/vdma/memory/mapped_buffer.hpp index 1c2e8fa..7578118 100644 --- a/hailort/libhailort/src/vdma/memory/mapped_buffer.hpp +++ b/hailort/libhailort/src/vdma/memory/mapped_buffer.hpp @@ -46,8 +46,12 @@ public: static Expected create_shared_by_allocation(size_t size, HailoRTDriver &driver, HailoRTDriver::DmaDirection data_direction); + // Receive an fd to a dmabuf object and map it in our driver and create MappedBuffer + static Expected create_shared_from_dmabuf(int dmabuf_fd, size_t size, HailoRTDriver &driver, + HailoRTDriver::DmaDirection data_direction); + MappedBuffer(HailoRTDriver &driver, DmaAbleBufferPtr buffer, HailoRTDriver::DmaDirection data_direction, - hailo_status &status); + HailoRTDriver::VdmaBufferHandle vdma_buffer_handle); MappedBuffer(MappedBuffer &&other) noexcept; MappedBuffer(const MappedBuffer &other) = delete; MappedBuffer &operator=(const MappedBuffer &other) = delete; diff --git a/hailort/libhailort/src/vdma/memory/sg_edge_layer.cpp b/hailort/libhailort/src/vdma/memory/sg_edge_layer.cpp index 371f52b..6c7345c 100644 --- a/hailort/libhailort/src/vdma/memory/sg_edge_layer.cpp +++ b/hailort/libhailort/src/vdma/memory/sg_edge_layer.cpp @@ -33,16 +33,17 @@ Expected SgEdgeLayer::create(std::shared_ptr &&buffer, si assert((desc_count * desc_page_size) <= std::numeric_limits::max()); - auto status = desc_list_exp->configure_to_use_buffer(*(buffer->get_mapped_buffer()), size , offset, channel_id); + auto status = desc_list_exp->program(*(buffer->get_mapped_buffer()), size , offset, channel_id); CHECK_SUCCESS_AS_EXPECTED(status); - return SgEdgeLayer(std::move(buffer), desc_list_exp.release(), size, offset); + return SgEdgeLayer(std::move(buffer), desc_list_exp.release(), size, offset, channel_id); } SgEdgeLayer::SgEdgeLayer(std::shared_ptr &&buffer, DescriptorList &&desc_list, - size_t size, size_t offset) : + size_t size, size_t offset, ChannelId channel_id) : VdmaEdgeLayer(std::move(buffer), size, offset), - m_desc_list(std::move(desc_list)) + m_desc_list(std::move(desc_list)), + m_channel_id(channel_id) {} uint64_t SgEdgeLayer::dma_address() const @@ -61,9 +62,11 @@ uint32_t SgEdgeLayer::descs_count() const } Expected SgEdgeLayer::program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, - size_t desc_offset) + size_t desc_offset, size_t buffer_offset, bool should_bind) { - return m_desc_list.program_last_descriptor(transfer_size, last_desc_interrupts_domain, desc_offset); + CHECK_SUCCESS(m_desc_list.program(*get_mapped_buffer(), transfer_size, buffer_offset, m_channel_id, + static_cast(desc_offset), should_bind, last_desc_interrupts_domain)); + return descriptors_in_buffer(transfer_size); } } diff --git a/hailort/libhailort/src/vdma/memory/sg_edge_layer.hpp b/hailort/libhailort/src/vdma/memory/sg_edge_layer.hpp index bd9716c..aae731f 100644 --- a/hailort/libhailort/src/vdma/memory/sg_edge_layer.hpp +++ b/hailort/libhailort/src/vdma/memory/sg_edge_layer.hpp @@ -47,15 +47,19 @@ public: virtual uint32_t descs_count() const override; virtual Expected program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, - size_t desc_offset) override; + size_t desc_offset, size_t buffer_offset = 0, bool should_bind = false) override; private: SgEdgeLayer(std::shared_ptr &&buffer, DescriptorList &&desc_list, - size_t size, size_t offset); + size_t size, size_t offset, ChannelId channel_id); + + vdma::MappedBufferPtr get_mapped_buffer() + { + return std::static_pointer_cast(m_buffer)->get_mapped_buffer(); + } - // Initialization Dependency: The descriptor list points into the mapped buffer so it must be freed before it - std::shared_ptr m_buffer; DescriptorList m_desc_list; + const ChannelId m_channel_id; }; } /* vdma */ diff --git a/hailort/libhailort/src/vdma/memory/vdma_edge_layer.hpp b/hailort/libhailort/src/vdma/memory/vdma_edge_layer.hpp index 8814f26..691f3ab 100644 --- a/hailort/libhailort/src/vdma/memory/vdma_edge_layer.hpp +++ b/hailort/libhailort/src/vdma/memory/vdma_edge_layer.hpp @@ -44,6 +44,11 @@ public: return m_size; } + size_t backing_buffer_size() const + { + return m_buffer->size(); + } + uint32_t descriptors_in_buffer(size_t buffer_size) const { assert(buffer_size < std::numeric_limits::max()); @@ -55,7 +60,7 @@ public: hailo_status write(const void *buf_src, size_t count, size_t offset); virtual Expected program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, - size_t desc_offset) = 0; + size_t desc_offset, size_t buffer_offset = 0, bool should_bind = false) = 0; CONTROL_PROTOCOL__host_buffer_info_t get_host_buffer_info(uint32_t transfer_size); static CONTROL_PROTOCOL__host_buffer_info_t get_host_buffer_info(Type type, uint64_t dma_address, diff --git a/hailort/libhailort/src/vdma/pcie/pcie_device.cpp b/hailort/libhailort/src/vdma/pcie/pcie_device.cpp index b8bb537..483c44f 100644 --- a/hailort/libhailort/src/vdma/pcie/pcie_device.cpp +++ b/hailort/libhailort/src/vdma/pcie/pcie_device.cpp @@ -64,7 +64,7 @@ Expected> PcieDevice::create(const hailo_pcie_device auto device_info = find_device_info(pcie_device_info); CHECK_EXPECTED(device_info); - auto driver = HailoRTDriver::create(*device_info); + auto driver = HailoRTDriver::create(device_info->device_id, device_info->dev_path); CHECK_EXPECTED(driver); hailo_status status = HAILO_UNINITIALIZED; @@ -137,8 +137,13 @@ bool PcieDevice::pcie_device_infos_equal(const hailo_pcie_device_info_t &first, } PcieDevice::PcieDevice(std::unique_ptr &&driver, hailo_status &status) : - VdmaDevice(std::move(driver), Device::Type::PCIE) + VdmaDevice(std::move(driver), Device::Type::PCIE, status) { + if (status != HAILO_SUCCESS) { + LOGGER__ERROR("Failed to create VdmaDevice"); + return; + } + if (m_driver->is_fw_loaded()) { status = update_fw_state(); if (HAILO_SUCCESS != status) { diff --git a/hailort/libhailort/src/vdma/pcie_session.cpp b/hailort/libhailort/src/vdma/pcie_session.cpp new file mode 100644 index 0000000..318ab98 --- /dev/null +++ b/hailort/libhailort/src/vdma/pcie_session.cpp @@ -0,0 +1,177 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file pcie_session.cpp + **/ + +#include "pcie_session.hpp" +#include "vdma/channel/channels_group.hpp" + +namespace hailort +{ + +static constexpr uint64_t MAX_ONGOING_TRANSFERS = 128; + +Expected PcieSession::connect(std::shared_ptr driver, pcie_connection_port_t port) +{ + TRY(auto input_desc_list, create_desc_list(*driver)); + TRY(auto output_desc_list, create_desc_list(*driver)); + + TRY(auto channel_pair, driver->soc_connect(input_desc_list.handle(), output_desc_list.handle())); + + (void)port; + + return PcieSession::create(driver, channel_pair.first, channel_pair.second, std::move(input_desc_list), + std::move(output_desc_list), PcieSessionType::CLIENT); +} + +Expected PcieSession::accept(std::shared_ptr driver, pcie_connection_port_t port) +{ + TRY(auto input_desc_list, create_desc_list(*driver)); + TRY(auto output_desc_list, create_desc_list(*driver)); + + TRY(auto channel_pair, driver->pci_ep_accept(input_desc_list.handle(), output_desc_list.handle())); + + (void)port; + + return PcieSession::create(driver, channel_pair.first, channel_pair.second, std::move(input_desc_list), + std::move(output_desc_list), PcieSessionType::SERVER); +} + +Expected PcieSession::create(std::shared_ptr driver, vdma::ChannelId input_channel_id, + vdma::ChannelId output_channel_id, vdma::DescriptorList &&input_desc_list, vdma::DescriptorList &&output_desc_list, + PcieSessionType session_type) +{ + // TODO: HRT-14038 - remove this to support multiple connections. Until then, mark as used to allow ctrl+c handle + CHECK_SUCCESS(driver->mark_as_used()); + + TRY(auto interrupts_dispatcher, vdma::InterruptsDispatcher::create(*driver)); + TRY(auto transfer_launcher, vdma::TransferLauncher::create()); + + auto create_channel = [&](vdma::ChannelId id, vdma::BoundaryChannel::Direction dir, vdma::DescriptorList &&desc_list) { + return vdma::BoundaryChannel::create(*driver, id, dir, std::move(desc_list), *transfer_launcher, + MAX_ONGOING_TRANSFERS); + }; + + TRY(auto input_channel, create_channel(input_channel_id, vdma::BoundaryChannel::Direction::H2D, std::move(input_desc_list))); + TRY(auto output_channel, create_channel(output_channel_id, vdma::BoundaryChannel::Direction::D2H, std::move(output_desc_list))); + + CHECK_SUCCESS(interrupts_dispatcher->start(vdma::ChannelsGroup{input_channel, output_channel})); + CHECK_SUCCESS(transfer_launcher->start()); + + CHECK_SUCCESS(input_channel->activate()); + CHECK_SUCCESS(output_channel->activate()); + + return PcieSession(std::move(driver), std::move(interrupts_dispatcher), std::move(transfer_launcher), + std::move(input_channel), std::move(output_channel), session_type); +} + +hailo_status PcieSession::write(const void *buffer, size_t size, std::chrono::milliseconds timeout) +{ + return launch_transfer_sync(*m_input, const_cast(buffer), size, timeout); +} + +hailo_status PcieSession::read(void *buffer, size_t size, std::chrono::milliseconds timeout) +{ + return launch_transfer_sync(*m_output, buffer, size, timeout); +} + +hailo_status PcieSession::write_async(const void *buffer, size_t size, std::function &&callback) +{ + return launch_transfer_async(*m_input, const_cast(buffer), size, std::move(callback)); +} + +hailo_status PcieSession::read_async(void *buffer, size_t size, std::function &&callback) +{ + return launch_transfer_async(*m_output, buffer, size, std::move(callback)); +} + +hailo_status PcieSession::close() +{ + hailo_status status = HAILO_SUCCESS; // Success orietnted + + // First, close all host resources, disallow new transfers + m_input->deactivate(); + m_output->deactivate(); + + auto stop_status = m_interrupts_dispatcher->stop(); + if (HAILO_SUCCESS != stop_status) { + LOGGER__ERROR("Failed to stop interrupts dispatcher with status {}", stop_status); + status = stop_status; + } + + // Then, close the connection to ABORT any vDMA channel. + stop_status = m_driver->close_connection(m_input->get_channel_id(), m_output->get_channel_id(), m_session_type); + if (HAILO_SUCCESS != stop_status) { + LOGGER__ERROR("Failed to close connection with status {}", stop_status); + status = stop_status; + } + + // Finally, cancel any pending transfer (must happen after the vDMA channel was aborted). + m_input->cancel_pending_transfers(); + m_output->cancel_pending_transfers(); + + stop_status = m_transfer_launcher->stop(); + if (HAILO_SUCCESS != stop_status) { + LOGGER__ERROR("Failed to stop transfer launcher with status {}", stop_status); + status = stop_status; + } + + return status; +} + +uint64_t PcieSession::max_transfer_size() +{ + // The max transfer size, is the size feet in MAX_SG_DESCS_COUNT -1. + // We don't use the last 8 descritpors to max sure max_transfer_size is aligened to 4K + return vdma::DEFAULT_SG_PAGE_SIZE * (MAX_SG_DESCS_COUNT - 8); +} + +hailo_status PcieSession::launch_transfer_sync(vdma::BoundaryChannel &channel, + void *buffer, size_t size, std::chrono::milliseconds timeout) +{ + std::mutex mutex; + std::condition_variable cv; + hailo_status transfer_status = HAILO_UNINITIALIZED; + auto callback = [&](hailo_status status) mutable { + { + std::unique_lock lock(mutex); + assert(status != HAILO_UNINITIALIZED); + transfer_status = status; + } + cv.notify_one(); + }; + + auto status = launch_transfer_async(channel, buffer, size, std::move(callback)); + if (HAILO_STREAM_ABORT == status) { + return status; + } + CHECK_SUCCESS(status); + + std::unique_lock lock(mutex); + CHECK(cv.wait_for(lock, timeout, [&] { return transfer_status != HAILO_UNINITIALIZED; }), + HAILO_TIMEOUT, "Timeout waiting for transfer completion"); + return transfer_status; +} + +hailo_status PcieSession::launch_transfer_async(vdma::BoundaryChannel &channel, + void *buffer, size_t size, std::function &&callback) +{ + TransferRequest request{ + {TransferBuffer(MemoryView(buffer, size))}, + std::move(callback) + }; + + return channel.launch_transfer(std::move(request)); +} + +Expected PcieSession::create_desc_list(HailoRTDriver &driver) +{ + const bool circular = true; + TRY(auto desc_list, vdma::DescriptorList::create(MAX_SG_DESCS_COUNT, vdma::DEFAULT_SG_PAGE_SIZE, circular, driver)); + return desc_list; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/pcie_session.hpp b/hailort/libhailort/src/vdma/pcie_session.hpp new file mode 100644 index 0000000..3b089c9 --- /dev/null +++ b/hailort/libhailort/src/vdma/pcie_session.hpp @@ -0,0 +1,115 @@ +/** + * Copyright (c) 2024 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file pcie_session.hpp + * @brief Wrapper for 2 BoundaryChannels, one for input and one for output, used as a the transport layer of a + * session over PCIe. + **/ + +#ifndef _HAILO_PCIE_SESSION_HPP_ +#define _HAILO_PCIE_SESSION_HPP_ + +#include "hailo/hailort.h" +#include "vdma/channel/boundary_channel.hpp" +#include "vdma/channel/interrupts_dispatcher.hpp" +#include "vdma/channel/transfer_launcher.hpp" + +namespace hailort +{ + +// A special magic number used to match each accept() with the corresponding connect(). +// By using this magic, multiple servers can be implemented and run simultaneously on the same device. +using pcie_connection_port_t = uint32_t; +using PcieSessionType = HailoRTDriver::PcieSessionType; + +/** + * a PcieSession object need to be constructed both at the device side (via accept) or the host side (via connect). + * After the session is created on both sides, the session can be used to send and receive data (based on the desired + * protocol). + * + * This session object is a low-level object offering fast zero-copy data transfer over PCIe with negligible overhead. + * To achieve this, the object have the following limitations: + * 1. Buffers Alignment - + * a. The input/output buffers must be page aligned. + * b. The output buffer should own the full cache line size, otherwise invalidating the cache can cause memory + * corruption. Simple solution is to allocate the buffer using mmap. + * 2. Buffer Sizes + * a. The buffer must be a multiple of 8 bytes. + * b. The max size of each buffer is (desc_page_size * (max_desc_count - 1)) = 32 MB. + * 3. Read/Write synchronization - The size pattern of the writes in one edge must be the same as the size pattern + * of reads in the other edge. + * For example, if the host writes this pattern: + * WRITE 8 bytes + * WRITE 32 bytes + * WRITE 16 bytes + * The device must read the same pattern: + * READ 8 bytes + * READ 32 bytes + * READ 16 bytes + * + * The protocol must ensure this behavior (for example by sending a const size header before each write). + */ +class PcieSession final { +public: + + static Expected connect(std::shared_ptr driver, pcie_connection_port_t port); + static Expected accept(std::shared_ptr driver, pcie_connection_port_t port); + + hailo_status write(const void *buffer, size_t size, std::chrono::milliseconds timeout); + hailo_status read(void *buffer, size_t size, std::chrono::milliseconds timeout); + + hailo_status write_async(const void *buffer, size_t size, std::function &&callback); + hailo_status read_async(void *buffer, size_t size, std::function &&callback); + + hailo_status close(); + + inline PcieSessionType session_type() const + { + return m_session_type; + } + + static uint64_t max_transfer_size(); + +private: + + using ChannelIdsPair = std::pair; + using DescriptorsListPair = std::pair, std::reference_wrapper>; + + static Expected create(std::shared_ptr driver, vdma::ChannelId input_channel, + vdma::ChannelId output_channel, vdma::DescriptorList &&input_desc_list, vdma::DescriptorList &&output_desc_list, + PcieSessionType session_type); + + PcieSession(std::shared_ptr &&driver, + std::unique_ptr &&interrupts_dispatcher, + std::unique_ptr &&transfer_launcher, + vdma::BoundaryChannelPtr &&input, vdma::BoundaryChannelPtr &&output, PcieSessionType session_type) : + m_driver(std::move(driver)), + m_interrupts_dispatcher(std::move(interrupts_dispatcher)), + m_transfer_launcher(std::move(transfer_launcher)), + m_input(std::move(input)), + m_output(std::move(output)), + m_session_type(session_type) + {} + + static hailo_status launch_transfer_sync(vdma::BoundaryChannel &channel, + void *buffer, size_t size, std::chrono::milliseconds timeout); + static hailo_status launch_transfer_async(vdma::BoundaryChannel &channel, + void *buffer, size_t size, std::function &&callback); + static Expected create_desc_list(HailoRTDriver &driver); + + std::shared_ptr m_driver; + + std::unique_ptr m_interrupts_dispatcher; + std::unique_ptr m_transfer_launcher; + + vdma::BoundaryChannelPtr m_input; + vdma::BoundaryChannelPtr m_output; + + PcieSessionType m_session_type; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_PCIE_SESSION_HPP_ */ diff --git a/hailort/libhailort/src/vdma/vdma_config_core_op.cpp b/hailort/libhailort/src/vdma/vdma_config_core_op.cpp index 4afe078..bd22024 100644 --- a/hailort/libhailort/src/vdma/vdma_config_core_op.cpp +++ b/hailort/libhailort/src/vdma/vdma_config_core_op.cpp @@ -2,7 +2,7 @@ #include "vdma/vdma_config_core_op.hpp" #include "network_group/network_group_internal.hpp" #include "net_flow/pipeline/vstream_internal.hpp" - +#include "device_common/control.hpp" namespace hailort { @@ -10,23 +10,37 @@ namespace hailort Expected VdmaConfigCoreOp::create(ActiveCoreOpHolder &active_core_op_holder, const ConfigureNetworkParams &config_params, std::shared_ptr resources_manager, + std::shared_ptr cache_manager, std::shared_ptr metadata) { auto status = HAILO_UNINITIALIZED; - VdmaConfigCoreOp object(active_core_op_holder, config_params, - std::move(resources_manager), metadata, status); + VdmaConfigCoreOp core_op(active_core_op_holder, config_params, std::move(resources_manager), cache_manager, + metadata, status); CHECK_SUCCESS_AS_EXPECTED(status); - return object; + return core_op; } -VdmaConfigCoreOp::VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder, - const ConfigureNetworkParams &config_params, - std::shared_ptr &&resources_manager, - std::shared_ptr metadata, hailo_status &status) : - CoreOp(config_params, metadata, active_core_op_holder, status), - m_resources_manager(std::move(resources_manager)) +Expected> VdmaConfigCoreOp::create_shared(ActiveCoreOpHolder &active_core_op_holder, + const ConfigureNetworkParams &config_params, + std::shared_ptr resources_manager, + std::shared_ptr cache_manager, + std::shared_ptr metadata) +{ + TRY(auto core_op, create(active_core_op_holder, config_params, resources_manager, cache_manager, metadata)); + auto core_op_ptr = make_shared_nothrow(std::move(core_op)); + CHECK_NOT_NULL_AS_EXPECTED(core_op_ptr, HAILO_OUT_OF_HOST_MEMORY); + + return core_op_ptr; +} + +VdmaConfigCoreOp::VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder, const ConfigureNetworkParams &config_params, + std::shared_ptr &&resources_manager, std::shared_ptr cache_manager, + std::shared_ptr metadata, hailo_status &status) : + CoreOp(config_params, metadata, active_core_op_holder, status), + m_resources_manager(std::move(resources_manager)), + m_cache_manager(cache_manager) {} @@ -55,9 +69,10 @@ hailo_status VdmaConfigCoreOp::cancel_pending_transfers() hailo_status VdmaConfigCoreOp::activate_impl(uint16_t dynamic_batch_size) { - auto status = HAILO_UNINITIALIZED; - auto start_time = std::chrono::steady_clock::now(); + auto status = register_cache_update_callback(); + CHECK_SUCCESS(status, "Failed to register cache update callback"); + auto start_time = std::chrono::steady_clock::now(); if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE != dynamic_batch_size) { CHECK(dynamic_batch_size <= get_smallest_configured_batch_size(get_config_params()), HAILO_INVALID_ARGUMENT, "Batch size given is {} although max is {}", dynamic_batch_size, @@ -72,7 +87,7 @@ hailo_status VdmaConfigCoreOp::activate_impl(uint16_t dynamic_batch_size) //TODO: HRT-13019 - Unite with the calculation in core_op.cpp const auto elapsed_time_ms = std::chrono::duration( std::chrono::steady_clock::now() - start_time).count(); - TRACE(ActivateCoreOpTrace, std::string(m_resources_manager->get_dev_id()), vdevice_core_op_handle(), elapsed_time_ms); + TRACE(ActivateCoreOpTrace, std::string(m_resources_manager->get_dev_id()), vdevice_core_op_handle(), elapsed_time_ms, dynamic_batch_size); return HAILO_SUCCESS; } @@ -97,6 +112,55 @@ hailo_status VdmaConfigCoreOp::deactivate_impl() std::chrono::steady_clock::now() - start_time).count(); TRACE(DeactivateCoreOpTrace, std::string(m_resources_manager->get_dev_id()), vdevice_core_op_handle(), elapsed_time_ms); + status = unregister_cache_update_callback(); + CHECK_SUCCESS(status, "Failed to unregister cache update callback"); + + return HAILO_SUCCESS; +} + +hailo_status VdmaConfigCoreOp::register_cache_update_callback() +{ + const auto cache_offset_env_var = get_env_variable(HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR); + if (cache_offset_env_var.has_value() && has_caches()) { + std::string policy; + int32_t offset_delta = 0; + TRY(const auto cache_write_size, get_cache_write_size()); + if (cache_offset_env_var.value() == HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DEFAULT) { + offset_delta = cache_write_size; + policy = "cache write size (default)"; + } else if (cache_offset_env_var.value() == HAILORT_AUTO_UPDATE_CACHE_OFFSET_ENV_VAR_DISABLED) { + LOGGER__INFO("Skipping cache offset updates"); + return HAILO_SUCCESS; + } else { + offset_delta = std::stoi(cache_offset_env_var.value()); + policy = "environment variable"; + CHECK(offset_delta <= static_cast(cache_write_size), HAILO_INVALID_ARGUMENT, "Invalid cache offset delta"); + } + + auto &vdma_device = m_resources_manager->get_device(); + vdma_device.set_notification_callback([this, offset_delta](Device &, const hailo_notification_t ¬ification, void *) { + if (HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET != notification.id) { + LOGGER__ERROR("Notification id passed to callback is invalid"); + return; + } + + const auto status = this->update_cache_offset(static_cast(offset_delta)); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to update cache offset"); + } + }, HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET, nullptr); + + LOGGER__INFO("Cache offsets will automatically be updated by {} [{}]", offset_delta, policy); + } + + return HAILO_SUCCESS; +} + +hailo_status VdmaConfigCoreOp::unregister_cache_update_callback() +{ + auto &vdma_device = m_resources_manager->get_device(); + auto status = vdma_device.remove_notification_callback(HAILO_NOTIFICATION_ID_START_UPDATE_CACHE_OFFSET); + CHECK_SUCCESS(status); return HAILO_SUCCESS; } @@ -190,4 +254,88 @@ Expected VdmaConfigCoreOp::get_intermediate_buffer(const IntermediateBuf return m_resources_manager->read_intermediate_buffer(key); } +Expected VdmaConfigCoreOp::get_cache_buffer(uint32_t cache_id) +{ + return m_resources_manager->read_cache_buffer(cache_id); +} + +Expected> VdmaConfigCoreOp::get_cache_buffers() +{ + return m_resources_manager->read_cache_buffers(); +} + +bool VdmaConfigCoreOp::has_caches() const +{ + return m_resources_manager->get_cache_buffers().size() > 0; +} + +Expected VdmaConfigCoreOp::get_cache_read_size() const +{ + // Input to the core == cache read + size_t input_size = 0; + for (auto &cache_buffer : m_resources_manager->get_cache_buffers()) { + const auto curr_input_size = cache_buffer.second.input_size(); + if (input_size == 0) { + input_size = curr_input_size; + } else { + CHECK(input_size == curr_input_size, HAILO_INVALID_ARGUMENT, "Cache buffers have different input sizes"); + } + } + + return static_cast(input_size); +} + +Expected VdmaConfigCoreOp::get_cache_write_size() const +{ + // Output from the core == cache write + size_t output_size = 0; + for (auto &cache_buffer : m_resources_manager->get_cache_buffers()) { + const auto curr_output_size = cache_buffer.second.output_size(); + if (output_size == 0) { + output_size = curr_output_size; + } else { + CHECK(output_size == curr_output_size, HAILO_INVALID_ARGUMENT, "Cache buffers have different output sizes"); + } + } + + return static_cast(output_size); +} + + +hailo_status VdmaConfigCoreOp::init_cache(uint32_t read_offset, int32_t write_offset_delta) +{ + CHECK(has_caches(), HAILO_INVALID_OPERATION, "No caches in core-op"); + return m_cache_manager->init_caches(read_offset, write_offset_delta); +} + +Expected VdmaConfigCoreOp::get_cache_info() const +{ + CHECK(has_caches(), HAILO_INVALID_OPERATION, "No caches in core-op"); + + return hailo_cache_info_t{ + m_cache_manager->get_cache_size(), + m_cache_manager->get_read_offset_bytes(), + m_cache_manager->get_write_offset_bytes_delta() + }; +} + +hailo_status VdmaConfigCoreOp::update_cache_offset(int32_t offset_delta_bytes) +{ + CHECK(has_caches(), HAILO_INVALID_OPERATION, "No caches in core-op"); + + // TODO: figure out how to do this s.t. it'll work with the sched (HRT-14287) + // auto status = wait_for_activation(std::chrono::milliseconds(0)); + // CHECK_SUCCESS(status, "Core op must be activated before updating cache offset"); + + // Update the offsets in the cache manager + auto status = m_cache_manager->update_cache_offset(offset_delta_bytes); + CHECK_SUCCESS(status); + + // Signal to the fw that the cache offset has been updated + status = Control::context_switch_signal_cache_updated(m_resources_manager->get_device()); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_config_core_op.hpp b/hailort/libhailort/src/vdma/vdma_config_core_op.hpp index 42f60f9..5a86248 100644 --- a/hailort/libhailort/src/vdma/vdma_config_core_op.hpp +++ b/hailort/libhailort/src/vdma/vdma_config_core_op.hpp @@ -18,6 +18,7 @@ #include "vdma/channel/boundary_channel.hpp" #include "core_op/resource_manager/resource_manager.hpp" +#include "core_op/resource_manager/cache_manager.hpp" #include "core_op/active_core_op_holder.hpp" #include "control_protocol.h" @@ -37,6 +38,13 @@ public: static Expected create(ActiveCoreOpHolder &active_core_op_holder, const ConfigureNetworkParams &config_params, std::shared_ptr resources_managers, + std::shared_ptr cache_manager, + std::shared_ptr metadata); + + static Expected> create_shared(ActiveCoreOpHolder &active_core_op_holder, + const ConfigureNetworkParams &config_params, + std::shared_ptr resources_manager, + std::shared_ptr cache_manager, std::shared_ptr metadata); std::shared_ptr &get_resources_manager() @@ -58,6 +66,9 @@ public: hailo_status cancel_pending_transfers(); + hailo_status register_cache_update_callback(); + hailo_status unregister_cache_update_callback(); + virtual Expected get_default_streams_interface() override; virtual Expected> get_latency_meters() override; @@ -70,22 +81,33 @@ public: virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name) override; virtual Expected run_hw_infer_estimator() override; virtual Expected get_intermediate_buffer(const IntermediateBufferKey &) override; + virtual Expected get_cache_buffer(uint32_t cache_id) override; + virtual Expected> get_cache_buffers() override; + virtual bool has_caches() const override; + virtual Expected get_cache_read_size() const override; + virtual Expected get_cache_write_size() const override; + virtual hailo_status init_cache(uint32_t read_offset, int32_t write_offset_delta) override; + virtual Expected get_cache_info() const; + virtual hailo_status update_cache_offset(int32_t offset_delta_bytes) override; virtual ~VdmaConfigCoreOp() = default; VdmaConfigCoreOp(const VdmaConfigCoreOp &other) = delete; VdmaConfigCoreOp &operator=(const VdmaConfigCoreOp &other) = delete; VdmaConfigCoreOp &operator=(VdmaConfigCoreOp &&other) = delete; VdmaConfigCoreOp(VdmaConfigCoreOp &&other) noexcept : CoreOp(std::move(other)), - m_resources_manager(std::move(other.m_resources_manager)) + m_resources_manager(std::move(other.m_resources_manager)), + m_cache_manager(std::move(other.m_cache_manager)) {} private: -VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder, + VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder, const ConfigureNetworkParams &config_params, std::shared_ptr &&resources_manager, + std::shared_ptr cache_manager, std::shared_ptr metadata, hailo_status &status); std::shared_ptr m_resources_manager; + std::shared_ptr m_cache_manager; }; } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_config_manager.cpp b/hailort/libhailort/src/vdma/vdma_config_manager.cpp index 753d34f..71fbcb3 100644 --- a/hailort/libhailort/src/vdma/vdma_config_manager.cpp +++ b/hailort/libhailort/src/vdma/vdma_config_manager.cpp @@ -31,7 +31,7 @@ hailo_status VdmaConfigManager::set_core_op(const std::string &device_id, std::s const auto core_op_handle = next ? next->vdevice_core_op_handle() : INVALID_CORE_OP_HANDLE; const auto elapsed_time_ms = std::chrono::duration( std::chrono::steady_clock::now() - start_time).count(); - TRACE(SwitchCoreOpTrace, device_id, core_op_handle, elapsed_time_ms); + TRACE(SwitchCoreOpTrace, device_id, core_op_handle, elapsed_time_ms, batch_size); return HAILO_SUCCESS; } @@ -50,6 +50,7 @@ hailo_status VdmaConfigManager::set_state_machine(std::shared_ptrregister_cache_update_callback(), "Failed to register cache update callback"); CHECK_SUCCESS(next->get_resources_manager()->enable_state_machine(batch_size), "Failed to enable state machine"); // In the case of switch NG, we call FW switch to next NG without marking the current NG as deactivated. // Added setter to mark the current NG as deactivated. @@ -87,6 +88,7 @@ hailo_status VdmaConfigManager::switch_core_op(std::shared_ptr if (current != nullptr) { CHECK_SUCCESS(current->cancel_pending_transfers(), "Failed canceling pending transfers from previous core-op"); + CHECK_SUCCESS(current->unregister_cache_update_callback(), "Failed unregistering cache updates from previous core-op"); } return HAILO_SUCCESS; diff --git a/hailort/libhailort/src/vdma/vdma_device.cpp b/hailort/libhailort/src/vdma/vdma_device.cpp index 026a3e9..93d4353 100644 --- a/hailort/libhailort/src/vdma/vdma_device.cpp +++ b/hailort/libhailort/src/vdma/vdma_device.cpp @@ -35,12 +35,14 @@ static constexpr std::chrono::milliseconds DEFAULT_TIMEOUT(1000); static constexpr std::chrono::milliseconds DEFAULT_TIMEOUT(50000); #endif /* ifndef HAILO_EMULATOR */ -VdmaDevice::VdmaDevice(std::unique_ptr &&driver, Device::Type type) : +VdmaDevice::VdmaDevice(std::unique_ptr &&driver, Device::Type type, hailo_status &status) : DeviceBase::DeviceBase(type), m_driver(std::move(driver)), m_is_configured(false) { activate_notifications(get_dev_id()); + + status = HAILO_SUCCESS; } Expected> VdmaDevice::create(const std::string &device_id) @@ -145,8 +147,11 @@ Expected VdmaDevice::add_hef(Hef &hef, const Netwo status = clear_configured_apps(); CHECK_SUCCESS_AS_EXPECTED(status); + assert(nullptr == m_cache_manager); + TRY(m_cache_manager, CacheManager::create_shared(get_driver())); + assert(nullptr == m_vdma_interrupts_dispatcher); - TRY(m_vdma_interrupts_dispatcher, vdma::InterruptsDispatcher::create(std::ref(*m_driver))); + TRY(m_vdma_interrupts_dispatcher, vdma::InterruptsDispatcher::create(get_driver())); assert(nullptr == m_vdma_transfer_launcher); TRY(m_vdma_transfer_launcher, vdma::TransferLauncher::create()); @@ -174,26 +179,23 @@ Expected> VdmaDevice::create_configured_ assert(core_ops_metadata.size() == 1); auto core_op_metadata = core_ops_metadata[0]; - /* build HEF supported features */ - auto resource_manager = ResourcesManagerBuilder::build(current_core_op_index, - *this, get_driver(), config_params, core_op_metadata, static_cast(hef.pimpl->get_device_arch()), - hef.pimpl->get_shef_file_handle()); - CHECK_EXPECTED(resource_manager); - + auto status = m_cache_manager->create_caches_from_core_op(core_op_metadata); + CHECK_SUCCESS(status); - auto core_op = VdmaConfigCoreOp::create(m_active_core_op_holder, config_params, - resource_manager.release(), core_op_metadata); + TRY(auto resource_manager, ResourcesManagerBuilder::build(current_core_op_index, + *this, get_driver(), m_cache_manager, config_params, core_op_metadata, + static_cast(hef.pimpl->get_device_arch()))); - auto core_op_ptr = make_shared_nothrow(core_op.release()); - CHECK_AS_EXPECTED(nullptr != core_op_ptr, HAILO_OUT_OF_HOST_MEMORY); + TRY(auto core_op_ptr, VdmaConfigCoreOp::create_shared(m_active_core_op_holder, config_params, + resource_manager, m_cache_manager, core_op_metadata)); // TODO: move this func into VdmaConfigCoreOp c'tor - auto status = core_op_ptr->create_streams_from_config_params(*this); - CHECK_SUCCESS_AS_EXPECTED(status); + status = core_op_ptr->create_streams_from_config_params(*this); + CHECK_SUCCESS(status); - // Check that all boundary streams were created + // Check that all boundary streams were created status = hef.pimpl->validate_boundary_streams_were_created(core_op_metadata->core_op_name(), core_op_ptr); - CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_SUCCESS(status); core_ops.emplace_back(core_op_ptr); m_core_ops.emplace_back(core_op_ptr); @@ -298,7 +300,8 @@ hailo_status VdmaDevice::dma_map(void *address, size_t size, hailo_dma_buffer_di buffer_identifier = buffer->buffer_identifier(); } - CHECK_EXPECTED(m_driver->vdma_buffer_map(address, size, to_hailo_driver_direction(data_direction), buffer_identifier)); + CHECK_EXPECTED(m_driver->vdma_buffer_map(reinterpret_cast(address), size, to_hailo_driver_direction(data_direction), buffer_identifier, + HailoRTDriver::DmaBufferType::USER_PTR_BUFFER)); return HAILO_SUCCESS; } @@ -313,7 +316,21 @@ hailo_status VdmaDevice::dma_unmap(void *address, size_t size, hailo_dma_buffer_ return HAILO_SUCCESS; } - return m_driver->vdma_buffer_unmap(address, size, to_hailo_driver_direction(data_direction)); + return m_driver->vdma_buffer_unmap(reinterpret_cast(address), size, to_hailo_driver_direction(data_direction)); +} + +hailo_status VdmaDevice::dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t data_direction) +{ + // Dont use BufferStorageResourceManager for dmabuf seeing as dmabufs are not allocated from hailoRT + auto buffer_identifier = HailoRTDriver::INVALID_MAPPED_BUFFER_DRIVER_IDENTIFIER; + CHECK_EXPECTED(m_driver->vdma_buffer_map(dmabuf_fd, size, to_hailo_driver_direction(data_direction), buffer_identifier, + HailoRTDriver::DmaBufferType::DMABUF_BUFFER)); + return HAILO_SUCCESS; +} + +hailo_status VdmaDevice::dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t data_direction) +{ + return m_driver->vdma_buffer_unmap(dmabuf_fd, size, to_hailo_driver_direction(data_direction)); } Expected VdmaDevice::create_networks_group_vector(Hef &hef, const NetworkGroupsParamsMap &configure_params) diff --git a/hailort/libhailort/src/vdma/vdma_device.hpp b/hailort/libhailort/src/vdma/vdma_device.hpp index 105e6d8..f7bcc90 100644 --- a/hailort/libhailort/src/vdma/vdma_device.hpp +++ b/hailort/libhailort/src/vdma/vdma_device.hpp @@ -19,7 +19,7 @@ #include "vdma/channel/interrupts_dispatcher.hpp" #include "vdma/channel/transfer_launcher.hpp" #include "vdma/driver/hailort_driver.hpp" - +#include "core_op/resource_manager/cache_manager.hpp" namespace hailort { @@ -53,9 +53,11 @@ public: virtual hailo_status dma_map(void *address, size_t size, hailo_dma_buffer_direction_t direction) override; virtual hailo_status dma_unmap(void *address, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_map_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override; + virtual hailo_status dma_unmap_dmabuf(int dmabuf_fd, size_t size, hailo_dma_buffer_direction_t direction) override; protected: - VdmaDevice(std::unique_ptr &&driver, Type type); + VdmaDevice(std::unique_ptr &&driver, Type type, hailo_status &status); virtual Expected read_notification() override; virtual hailo_status disable_notifications() override; @@ -64,6 +66,7 @@ protected: virtual Expected add_hef(Hef &hef, const NetworkGroupsParamsMap &configure_params) override; std::unique_ptr m_driver; + CacheManagerPtr m_cache_manager; // TODO - HRT-13234, move to DeviceBase std::vector> m_core_ops; std::vector> m_network_groups; // TODO: HRT-9547 - Remove when ConfiguredNetworkGroup will be kept in global context diff --git a/hailort/libhailort/src/vdma/vdma_stream.cpp b/hailort/libhailort/src/vdma/vdma_stream.cpp index 8c324ad..51f856f 100644 --- a/hailort/libhailort/src/vdma/vdma_stream.cpp +++ b/hailort/libhailort/src/vdma/vdma_stream.cpp @@ -12,7 +12,6 @@ #include "vdma/circular_stream_buffer_pool.hpp" #include "utils/profiler/tracer_macros.hpp" #include "utils/buffer_storage.hpp" -#include "common/os_utils.hpp" namespace hailort @@ -86,12 +85,7 @@ VdmaInputStream::VdmaInputStream(VdmaDevice &device, vdma::BoundaryChannelPtr ch VdmaInputStream::~VdmaInputStream() { - // We want to stop the vdma channel before closing the stream in the firmware - // because sending data to a closed stream may terminate the dma engine - const auto status = m_channel->deactivate(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to deactivate stream with error status {}", status); - } + m_channel->deactivate(); } hailo_stream_interface_t VdmaInputStream::get_interface() const @@ -106,16 +100,29 @@ void VdmaInputStream::set_vdevice_core_op_handle(vdevice_core_op_handle_t core_o Expected> VdmaInputStream::allocate_buffer_pool() { - TRY(auto circular_pool, CircularStreamBufferPool::create(m_device, HAILO_DMA_BUFFER_DIRECTION_H2D, - m_channel->get_desc_list()->desc_page_size(), m_channel->get_desc_list()->count(), get_frame_size())); + const auto frame_size = get_frame_size(); + const auto max_transfers_in_desc_list = m_channel->get_max_aligned_transfers_in_desc_list(frame_size); + const auto max_ongoing_transfers = m_channel->get_max_ongoing_transfers(frame_size); + if (max_transfers_in_desc_list < max_ongoing_transfers) { + // In this case we don't bind, since the descriptor list isn't big enough to hold all the buffers. + // (otherwise pending_transfers_queue_in_use would be false) + TRY(auto stream_buffer_pool, QueuedStreamBufferPool::create(max_ongoing_transfers, frame_size, + BufferStorageParams::create_dma())); + + return std::unique_ptr(std::move(stream_buffer_pool)); + } else { + // We can fit all the buffers in the descriptor list, so we can bind them statically. + TRY(auto circular_pool, CircularStreamBufferPool::create(m_device, HAILO_DMA_BUFFER_DIRECTION_H2D, + m_channel->get_desc_list().desc_page_size(), m_channel->get_desc_list().count(), frame_size)); - // Bind the buffer to the channel to avoid the need to do it on every transfer. - TRY(auto pool_dma_able_buffer, circular_pool->get_base_buffer().storage().get_dma_able_buffer()); - TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared(pool_dma_able_buffer, m_device.get_driver(), - HailoRTDriver::DmaDirection::H2D)); - CHECK_SUCCESS(m_channel->bind_buffer(mapped_buffer)); + // Bind the buffer to the channel to avoid the need to do it on every transfer. + TRY(auto pool_dma_able_buffer, circular_pool->get_base_buffer().storage().get_dma_able_buffer()); + TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared(pool_dma_able_buffer, m_device.get_driver(), + HailoRTDriver::DmaDirection::H2D)); + CHECK_SUCCESS(m_channel->bind_buffer(mapped_buffer)); - return std::unique_ptr(std::move(circular_pool)); + return std::unique_ptr(std::move(circular_pool)); + } } size_t VdmaInputStream::get_max_ongoing_transfers() const @@ -127,7 +134,8 @@ Expected VdmaInputStream::align_transfer_request(TransferReques { const auto dma_alignment = OsUtils::get_dma_able_alignment(); std::vector transfer_buffers; - const auto buffer_address = transfer_request.transfer_buffers[0].base_buffer().data(); + TRY(auto base_buffer, transfer_request.transfer_buffers[0].base_buffer()); + const auto buffer_address = base_buffer.data(); const auto buffer_size = transfer_request.transfer_buffers[0].size(); TRY(const auto dma_able_bounce_buffer, m_bounce_buffers_pool->dequeue()); @@ -167,13 +175,17 @@ hailo_status VdmaInputStream::write_async_impl(TransferRequest &&transfer_reques { TRACE(FrameDequeueH2DTrace, m_device.get_dev_id(), m_core_op_handle, name()); - const auto dma_able_alignment = OsUtils::get_dma_able_alignment(); - if (reinterpret_cast(transfer_request.transfer_buffers[0].base_buffer().data()) % dma_able_alignment == 0) { + if (transfer_request.transfer_buffers[0].type() == TransferBufferType::DMABUF) { return m_channel->launch_transfer(std::move(transfer_request)); } else { - auto unaligned_transfer_request = align_transfer_request(std::move(transfer_request)); - CHECK_EXPECTED_AS_STATUS(unaligned_transfer_request); - return m_channel->launch_transfer(unaligned_transfer_request.release()); + TRY(auto is_request_aligned, transfer_request.is_request_aligned()); + if (is_request_aligned) { + return m_channel->launch_transfer(std::move(transfer_request)); + } else { + auto realigned_transfer_request = align_transfer_request(std::move(transfer_request)); + CHECK_EXPECTED_AS_STATUS(realigned_transfer_request); + return m_channel->launch_transfer(realigned_transfer_request.release()); + } } } @@ -184,7 +196,8 @@ hailo_status VdmaInputStream::activate_stream_impl() hailo_status VdmaInputStream::deactivate_stream_impl() { - return m_channel->deactivate(); + m_channel->deactivate(); + return HAILO_SUCCESS; } hailo_status VdmaInputStream::cancel_pending_transfers() @@ -219,7 +232,8 @@ VdmaOutputStream::VdmaOutputStream(VdmaDevice &device, vdma::BoundaryChannelPtr m_channel(std::move(channel)), m_interface(interface), m_transfer_size(get_transfer_size(m_stream_info, get_layer_info())), - m_core_op_handle(INVALID_CORE_OP_HANDLE) + m_core_op_handle(INVALID_CORE_OP_HANDLE), + m_d2h_callback(default_d2h_callback) { // Check status for base class c'tor if (HAILO_SUCCESS != status) { @@ -231,12 +245,7 @@ VdmaOutputStream::VdmaOutputStream(VdmaDevice &device, vdma::BoundaryChannelPtr VdmaOutputStream::~VdmaOutputStream() { - // We want to stop the vdma channel before closing the stream in the firmware - // because sending data to a closed stream may terminate the dma engine - const auto status = m_channel->deactivate(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to deactivate stream with error status {}", status); - } + m_channel->deactivate(); } hailo_stream_interface_t VdmaOutputStream::get_interface() const @@ -246,16 +255,28 @@ hailo_stream_interface_t VdmaOutputStream::get_interface() const Expected> VdmaOutputStream::allocate_buffer_pool() { - TRY(auto circular_pool, CircularStreamBufferPool::create(m_device, HAILO_DMA_BUFFER_DIRECTION_D2H, - m_channel->get_desc_list()->desc_page_size(), m_channel->get_desc_list()->count(), m_transfer_size)); - - // Bind the buffer to the channel to avoid the need to do it on every transfer. - TRY(auto pool_dma_able_buffer, circular_pool->get_base_buffer().storage().get_dma_able_buffer()); - TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared(pool_dma_able_buffer, m_device.get_driver(), - HailoRTDriver::DmaDirection::D2H)); - CHECK_SUCCESS(m_channel->bind_buffer(mapped_buffer)); - - return std::unique_ptr(std::move(circular_pool)); + const auto max_transfers_in_desc_list = m_channel->get_max_aligned_transfers_in_desc_list(m_transfer_size); + const auto max_ongoing_transfers = m_channel->get_max_ongoing_transfers(m_transfer_size); + if (max_transfers_in_desc_list < max_ongoing_transfers) { + // In this case we don't bind, since the descriptor list isn't big enough to hold all the buffers. + // (otherwise pending_transfers_queue_in_use would be false) + TRY(auto stream_buffer_pool, QueuedStreamBufferPool::create(max_ongoing_transfers, m_transfer_size, + BufferStorageParams::create_dma())); + + return std::unique_ptr(std::move(stream_buffer_pool)); + } else { + // We can fit all the buffers in the descriptor list, so we can bind them statically. + TRY(auto circular_pool, CircularStreamBufferPool::create(m_device, HAILO_DMA_BUFFER_DIRECTION_D2H, + m_channel->get_desc_list().desc_page_size(), m_channel->get_desc_list().count(), m_transfer_size)); + + // Bind the buffer to the channel to avoid the need to do it on every transfer. + TRY(auto pool_dma_able_buffer, circular_pool->get_base_buffer().storage().get_dma_able_buffer()); + TRY(auto mapped_buffer, vdma::MappedBuffer::create_shared(pool_dma_able_buffer, m_device.get_driver(), + HailoRTDriver::DmaDirection::D2H)); + CHECK_SUCCESS(m_channel->bind_buffer(mapped_buffer)); + + return std::unique_ptr(std::move(circular_pool)); + } } size_t VdmaOutputStream::get_max_ongoing_transfers() const @@ -270,7 +291,8 @@ Expected VdmaOutputStream::align_transfer_request(TransferReque CHECK_EXPECTED(bounce_buffer_exp); auto bounce_buffer = bounce_buffer_exp.release(); - auto wrapped_callback = [unaligned_user_buffer = transfer_request.transfer_buffers[0].base_buffer(), + TRY(auto base_buffer, transfer_request.transfer_buffers[0].base_buffer()); + auto wrapped_callback = [unaligned_user_buffer = base_buffer, bounce_buffer=bounce_buffer, user_callback=transfer_request.callback](hailo_status callback_status) { memcpy(const_cast(unaligned_user_buffer.data()), bounce_buffer->data(), unaligned_user_buffer.size()); user_callback(callback_status); @@ -281,26 +303,32 @@ Expected VdmaOutputStream::align_transfer_request(TransferReque hailo_status VdmaOutputStream::read_async_impl(TransferRequest &&transfer_request) { - if ((INVALID_CORE_OP_HANDLE != m_core_op_handle) && (HAILO_FORMAT_ORDER_HAILO_NMS != m_stream_info.format.order)) { + if (HAILO_FORMAT_ORDER_HAILO_NMS != m_stream_info.format.order) { // On NMS stream we trace EnqueueD2H inside nms_stream - transfer_request.callback = [original_callback=transfer_request.callback, this](hailo_status status) { - if (HAILO_SUCCESS == status) { + transfer_request.callback = [original_callback=transfer_request.callback, d2h_callback=m_d2h_callback, this](hailo_status status) { + if ((HAILO_SUCCESS == status) && (INVALID_CORE_OP_HANDLE != m_core_op_handle)) { TRACE(FrameEnqueueD2HTrace, m_device.get_dev_id(), m_core_op_handle, name()); } + d2h_callback(status); original_callback(status); }; } - const auto dma_able_alignment = OsUtils::get_dma_able_alignment(); - if (reinterpret_cast(transfer_request.transfer_buffers[0].base_buffer().data()) % dma_able_alignment == 0) { + if (transfer_request.transfer_buffers[0].type() == TransferBufferType::DMABUF) { return m_channel->launch_transfer(std::move(transfer_request)); } else { - // In case of read unaligned - currently doesnt support using users buffer - so allocate complete new buffer size of user's buffer - LOGGER__WARNING("read_async() was provided an unaligned buffer (address=0x{:x}), which causes performance degradation. Use buffers algined to {} bytes for optimal performance", - reinterpret_cast(transfer_request.transfer_buffers[0].base_buffer().data()), dma_able_alignment); - - auto realigned_transfer_request = align_transfer_request(std::move(transfer_request)); - CHECK_EXPECTED_AS_STATUS(realigned_transfer_request); - return m_channel->launch_transfer(realigned_transfer_request.release()); + TRY(auto is_request_aligned, transfer_request.is_request_aligned()); + if (is_request_aligned) { + return m_channel->launch_transfer(std::move(transfer_request)); + } else { + // In case of read unaligned - don't support using users buffer - so well allocate complete new buffer size of user's buffer + TRY(auto base_buffer, transfer_request.transfer_buffers[0].base_buffer()); + LOGGER__WARNING("read_async() was provided an unaligned buffer (address=0x{:x}), which causes performance degradation. Use buffers algined to {} bytes for optimal performance", + reinterpret_cast(base_buffer.data()), OsUtils::get_dma_able_alignment()); + + auto realigned_transfer_request = align_transfer_request(std::move(transfer_request)); + CHECK_EXPECTED_AS_STATUS(realigned_transfer_request); + return m_channel->launch_transfer(realigned_transfer_request.release()); + } } } @@ -311,7 +339,8 @@ hailo_status VdmaOutputStream::activate_stream_impl() hailo_status VdmaOutputStream::deactivate_stream_impl() { - return m_channel->deactivate(); + m_channel->deactivate(); + return HAILO_SUCCESS; } void VdmaOutputStream::set_vdevice_core_op_handle(vdevice_core_op_handle_t core_op_handle) @@ -331,4 +360,9 @@ hailo_status VdmaOutputStream::cancel_pending_transfers() return HAILO_SUCCESS; } +void VdmaOutputStream::set_d2h_callback(std::function callback) +{ + m_d2h_callback = callback; +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_stream.hpp b/hailort/libhailort/src/vdma/vdma_stream.hpp index b396212..836472a 100644 --- a/hailort/libhailort/src/vdma/vdma_stream.hpp +++ b/hailort/libhailort/src/vdma/vdma_stream.hpp @@ -90,8 +90,10 @@ public: // TODO - HRT-11739 - remove vdevice related members/functions (get/set_vdevice_core_op_handle) virtual inline vdevice_core_op_handle_t get_vdevice_core_op_handle() override { return m_core_op_handle; }; virtual hailo_status cancel_pending_transfers() override; + void set_d2h_callback(std::function callback); private: + static void default_d2h_callback(hailo_status) {}; static uint32_t get_transfer_size(const hailo_stream_info_t &stream_info, const LayerInfo &layer_info); Expected align_transfer_request(TransferRequest &&transfer_request); @@ -100,6 +102,7 @@ private: const hailo_stream_interface_t m_interface; const uint32_t m_transfer_size; vdevice_core_op_handle_t m_core_op_handle; + std::function m_d2h_callback; }; diff --git a/hailort/libhailort/tracer_profiler.proto b/hailort/libhailort/tracer_profiler.proto index d4d27c2..f01cd05 100644 --- a/hailort/libhailort/tracer_profiler.proto +++ b/hailort/libhailort/tracer_profiler.proto @@ -97,6 +97,7 @@ message ProtoProfilerActivateCoreOpTrace { int32 new_core_op_handle = 2; string device_id = 3; double duration = 4; //millisec + int32 dynamic_batch_size = 5; } message ProtoProfilerDeactivateCoreOpTrace { diff --git a/hailort/rpc/hailort_rpc.proto b/hailort/rpc/hailort_rpc.proto index a95d88c..f53b4f0 100644 --- a/hailort/rpc/hailort_rpc.proto +++ b/hailort/rpc/hailort_rpc.proto @@ -420,6 +420,7 @@ message ProtoNmsPostProcessConfig { bool background_removal = 5; uint32 background_removal_index = 6; bool cross_classes = 7; + bool bbox_only = 8; } message ProtoYolov8MatchingLayersNames { diff --git a/hailort/scripts/download_firmware_eth.cmd b/hailort/scripts/download_firmware_eth.cmd index 4d0a2dc..e2502e9 100644 --- a/hailort/scripts/download_firmware_eth.cmd +++ b/hailort/scripts/download_firmware_eth.cmd @@ -2,7 +2,7 @@ @ECHO OFF set BASE_URI=https://hailo-hailort.s3.eu-west-2.amazonaws.com -set HRT_VERSION=4.17.1 +set HRT_VERSION=4.18.0 set FW_DIR=Hailo8/%HRT_VERSION%/FW set FW=hailo8_fw.%HRT_VERSION%_eth.bin diff --git a/hailort/scripts/download_firmware_eth.sh b/hailort/scripts/download_firmware_eth.sh index 19a6591..72da7a1 100755 --- a/hailort/scripts/download_firmware_eth.sh +++ b/hailort/scripts/download_firmware_eth.sh @@ -2,7 +2,7 @@ set -e readonly BASE_URI="https://hailo-hailort.s3.eu-west-2.amazonaws.com" -readonly HRT_VERSION=4.17.1 +readonly HRT_VERSION=4.18.0 readonly FW_AWS_DIR="Hailo8/${HRT_VERSION}/FW" readonly FW="hailo8_fw.${HRT_VERSION}_eth.bin" diff --git a/hailort/scripts/download_hefs.cmd b/hailort/scripts/download_hefs.cmd index aac3980..ac442c5 100644 --- a/hailort/scripts/download_hefs.cmd +++ b/hailort/scripts/download_hefs.cmd @@ -1,7 +1,7 @@ :: cmd @ECHO OFF set BASE_URI=https://hailo-hailort.s3.eu-west-2.amazonaws.com -set HRT_VERSION=4.17.1 +set HRT_VERSION=4.18.0 set REMOTE_HEF_DIR=Hailo8/%HRT_VERSION%/HEFS set LOCAL_EXAMPLES_HEF_DIR=..\libhailort\examples\hefs set LOCAL_TUTORIALS_HEF_DIR=..\libhailort\bindings\python\platform\hailo_tutorials\hefs diff --git a/hailort/scripts/download_hefs.sh b/hailort/scripts/download_hefs.sh index 9b99725..a6baf2c 100755 --- a/hailort/scripts/download_hefs.sh +++ b/hailort/scripts/download_hefs.sh @@ -2,7 +2,7 @@ set -e readonly BASE_URI="https://hailo-hailort.s3.eu-west-2.amazonaws.com" -readonly HRT_VERSION=4.17.1 +readonly HRT_VERSION=4.18.0 readonly REMOTE_HEF_DIR="Hailo8/${HRT_VERSION}/HEFS" readonly LOCAL_EXAMPLES_HEF_DIR="../libhailort/examples/hefs" readonly LOCAL_TUTORIALS_HEF_DIR="../libhailort/bindings/python/platform/hailo_tutorials/hefs" -- 2.34.1