From 86bb9c4968c5c20c311e230ae24be25a02847b20 Mon Sep 17 00:00:00 2001 From: HailoRT-Automation <98901220+HailoRT-Automation@users.noreply.github.com> Date: Thu, 30 Mar 2023 14:42:56 +0300 Subject: [PATCH] v4.13.0 (#8) Co-authored-by: HailoRT-Automation --- README.md | 17 +- common/include/control_protocol.h | 63 +- common/include/firmware_header.h | 6 +- common/include/firmware_status.h | 11 + common/src/firmware_header_utils.c | 4 +- hailort/CMakeLists.txt | 10 +- hailort/common/CMakeLists.txt | 3 + hailort/common/async_thread.hpp | 21 +- hailort/common/circular_buffer.hpp | 85 +- hailort/common/device_measurements.cpp | 133 ++ hailort/common/device_measurements.hpp | 94 ++ hailort/common/ethernet_utils.hpp | 2 +- hailort/common/latency_meter.hpp | 6 +- hailort/common/os/posix/os_utils.cpp | 35 + hailort/common/os/windows/os_utils.cpp | 50 + hailort/common/os_utils.hpp | 80 + .../common/runtime_statistics_internal.hpp | 12 +- hailort/common/utils.hpp | 19 +- hailort/drivers/common/hailo_ioctl_common.h | 123 +- hailort/drivers/win/include/Public.h | 10 +- hailort/hailort_service/CMakeLists.txt | 57 +- .../hailort_service/hailort_rpc_service.cpp | 193 ++- .../hailort_service/hailort_rpc_service.hpp | 28 +- hailort/hailort_service/hailort_service | 5 +- .../service_resource_manager.hpp | 62 +- .../{ => unix}/hailort_service.cpp | 15 +- .../windows/hailort_service.cpp | 246 ++++ .../windows/hailort_service_env_vars.bat | 11 + hailort/hailortcli/CMakeLists.txt | 7 +- hailort/hailortcli/common.cpp | 20 +- hailort/hailortcli/common.hpp | 16 +- .../download_action_list_command.cpp | 8 +- hailort/hailortcli/fw_control_command.cpp | 6 +- hailort/hailortcli/fw_logger_command.cpp | 2 +- hailort/hailortcli/graph_printer.hpp | 4 +- hailort/hailortcli/hailortcli.hpp | 3 + hailort/hailortcli/infer_stats_printer.cpp | 27 +- hailort/hailortcli/inference_progress.cpp | 2 + hailort/hailortcli/inference_result.hpp | 8 +- hailort/hailortcli/mon_command.cpp | 187 ++- hailort/hailortcli/mon_command.hpp | 17 +- hailort/hailortcli/parse_hef_command.cpp | 99 +- hailort/hailortcli/run2/live_printer.cpp | 47 +- hailort/hailortcli/run2/live_printer.hpp | 19 +- .../run2/measurement_live_track.cpp | 141 ++ .../run2/measurement_live_track.hpp | 41 + .../hailortcli/run2/network_live_track.cpp | 46 +- .../hailortcli/run2/network_live_track.hpp | 17 +- hailort/hailortcli/run2/network_runner.cpp | 164 ++- hailort/hailortcli/run2/network_runner.hpp | 28 +- hailort/hailortcli/run2/run2_command.cpp | 191 ++- hailort/hailortcli/run2/timer_live_track.cpp | 15 +- hailort/hailortcli/run2/timer_live_track.hpp | 3 +- hailort/hailortcli/run_command.cpp | 42 +- hailort/hailortcli/run_command.hpp | 2 +- hailort/hailortcli/temp_measurement.cpp | 83 -- hailort/hailortcli/temp_measurement.hpp | 48 - hailort/libhailort/CMakeLists.txt | 5 +- .../bindings/gstreamer/CMakeLists.txt | 4 +- .../bindings/gstreamer/gst-hailo/common.hpp | 2 +- .../gst-hailo/gsthailodevicestats.hpp | 1 + .../gstreamer/gst-hailo/gsthailonet.cpp | 93 +- .../gstreamer/gst-hailo/gsthailonet.hpp | 8 +- .../gstreamer/gst-hailo/gsthailorecv.cpp | 2 +- .../gstreamer/gst-hailo/gsthailosend.cpp | 9 +- .../gst-hailo/network_group_handle.cpp | 18 +- .../gst-hailo/network_group_handle.hpp | 3 +- .../examples/hef_infer_pipeline_vstream.py | 34 - .../examples/hef_infer_virtual_stream.py | 69 - .../platform/hailo_platform/__init__.py | 4 +- .../pyhailort/control_object.py | 6 +- .../hailo_platform/pyhailort/hw_object.py | 55 +- .../hailo_platform/pyhailort/pyhailort.py | 280 +++- .../tools/hailocli/hailocli_commands.py | 32 +- .../hailo_platform/tools/hailocli/main.py | 25 +- .../tools/hailocli/version_action.py | 31 + .../hailo_platform/tools/udp_rate_limiter.py | 14 +- .../notebooks/HRT_0_Inference_Tutorial.ipynb | 4 +- .../bindings/python/platform/setup.py | 2 +- .../bindings/python/src/CMakeLists.txt | 5 + .../bindings/python/src/bindings_common.hpp | 7 +- .../bindings/python/src/device_api.cpp | 18 +- .../bindings/python/src/device_api.hpp | 17 +- .../bindings/python/src/hef_api.cpp | 54 +- .../python/src/internal/CMakeLists.txt | 7 +- .../python/src/internal/control_api.cpp | 2 +- .../python/src/internal/control_api.hpp | 2 +- .../src/internal/pyhailort_internal.cpp | 53 +- .../src/internal/pyhailort_internal.hpp | 9 +- .../bindings/python/src/net_flow_api.hpp | 72 +- .../bindings/python/src/pyhailort.cpp | 49 +- .../bindings/python/src/quantization_api.cpp | 2 + .../bindings/python/src/quantization_api.hpp | 4 +- .../libhailort/bindings/python/src/utils.hpp | 4 +- .../bindings/python/src/vdevice_api.hpp | 60 +- .../bindings/python/src/vstream_api.cpp | 126 ++ .../bindings/python/src/vstream_api.hpp | 6 + hailort/libhailort/examples/README.md | 3 +- hailort/libhailort/examples/c/CMakeLists.txt | 4 +- .../data_quantization_example/CMakeLists.txt | 2 +- .../c/infer_pipeline_example/CMakeLists.txt | 2 +- .../c/multi_device_example/CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../CMakeLists.txt | 20 + .../notification_callback_example.c | 77 + .../power_measurement_example/CMakeLists.txt | 2 +- .../c/raw_streams_example/CMakeLists.txt | 2 +- .../raw_streams_example/raw_streams_example.c | 13 +- .../CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- .../switch_network_groups_manually_example.c | 10 +- .../c/vstreams_example/CMakeLists.txt | 2 +- .../libhailort/examples/cpp/CMakeLists.txt | 6 +- .../cpp/infer_pipeline_example/CMakeLists.txt | 2 +- .../infer_pipeline_example.cpp | 3 +- .../cpp/multi_device_example/CMakeLists.txt | 2 +- .../multi_device_example.cpp | 4 +- .../CMakeLists.txt | 2 +- .../multi_network_vstream_example.cpp | 4 +- .../cpp/multi_process_example/CMakeLists.txt | 2 +- .../multi_process_example.cpp | 4 +- .../CMakeLists.txt | 16 + .../notification_callback_example.cpp | 68 + .../power_measurement_example/CMakeLists.txt | 2 +- .../power_measurement_example.cpp | 1 + .../raw_async_streams_example/CMakeLists.txt | 19 + .../raw_async_streams_example/buffer_pool.cpp | 76 + .../raw_async_streams_example/buffer_pool.hpp | 50 + .../raw_async_streams_example.cpp | 146 ++ .../cpp/raw_streams_example/CMakeLists.txt | 2 +- .../raw_streams_example.cpp | 26 +- .../CMakeLists.txt | 2 +- .../switch_network_groups_example.cpp | 2 + .../CMakeLists.txt | 2 +- ...switch_network_groups_manually_example.cpp | 2 + .../cpp/vstreams_example/CMakeLists.txt | 2 +- .../cpp/vstreams_example/vstreams_example.cpp | 4 +- hailort/libhailort/hef.proto | 51 +- hailort/libhailort/include/hailo/buffer.hpp | 3 +- hailort/libhailort/include/hailo/device.hpp | 22 +- .../include/hailo/dma_mapped_buffer.hpp | 76 + hailort/libhailort/include/hailo/event.hpp | 2 +- hailort/libhailort/include/hailo/expected.hpp | 8 + hailort/libhailort/include/hailo/hailort.h | 86 +- hailort/libhailort/include/hailo/hailort.hpp | 4 +- .../include/hailo/hailort_common.hpp | 36 +- .../include/hailo/hailort_defaults.hpp | 67 + hailort/libhailort/include/hailo/hef.hpp | 23 +- .../include/hailo/inference_pipeline.hpp | 9 +- .../include/hailo/network_group.hpp | 33 +- .../libhailort/include/hailo/quantization.hpp | 18 +- .../include/hailo/runtime_statistics.hpp | 4 +- hailort/libhailort/include/hailo/stream.hpp | 49 +- .../libhailort/include/hailo/transform.hpp | 1 + hailort/libhailort/include/hailo/vdevice.hpp | 26 +- hailort/libhailort/include/hailo/vstream.hpp | 16 +- hailort/libhailort/scheduler_mon.proto | 9 +- hailort/libhailort/src/CMakeLists.txt | 98 +- .../active_network_group_holder.hpp | 53 - .../hcp_config_activated_network_group.cpp | 100 -- .../hcp_config_network_group.cpp | 112 -- .../vdma_config_activated_network_group.hpp | 77 - .../multi_context/vdma_config_manager.hpp | 58 - .../vdma_config_network_group.hpp | 94 -- .../hcp_config_activated_network_group.hpp | 85 -- .../hcp_config_network_group.hpp | 65 - .../context_switch/vdevice_network_group.cpp | 368 ----- .../context_switch/vdevice_network_group.hpp | 182 --- .../vdma_config_activated_network_group.cpp | 136 -- .../vdma_config_network_group.cpp | 143 -- hailort/libhailort/src/core_device.cpp | 76 - hailort/libhailort/src/core_op/CMakeLists.txt | 15 + .../src/core_op/active_core_op_holder.hpp | 57 + .../network_group.cpp => core_op/core_op.cpp} | 497 +++---- hailort/libhailort/src/core_op/core_op.hpp | 211 +++ .../resource_manager}/channel_allocator.cpp | 13 +- .../resource_manager}/channel_allocator.hpp | 15 +- .../resource_manager}/config_buffer.cpp | 45 +- .../resource_manager}/config_buffer.hpp | 6 +- .../context_switch_buffer_builder.cpp | 2 - .../context_switch_buffer_builder.hpp | 8 +- .../resource_manager}/ddr_channels_pair.cpp | 37 +- .../resource_manager}/ddr_channels_pair.hpp | 6 +- .../inter_context_buffer.cpp | 48 +- .../inter_context_buffer.hpp | 10 +- .../resource_manager}/resource_manager.cpp | 476 ++++-- .../resource_manager}/resource_manager.hpp | 133 +- .../resource_manager_builder.cpp | 413 ++---- .../resource_manager_builder.hpp | 6 +- hailort/libhailort/src/core_stream.cpp | 76 - hailort/libhailort/src/core_stream.hpp | 66 - .../src/device_common/CMakeLists.txt | 11 + .../src/{ => device_common}/control.cpp | 419 +++++- .../src/{ => device_common}/control.hpp | 30 +- .../{ => device_common}/control_protocol.cpp | 67 +- .../{ => device_common}/control_protocol.hpp | 3 + .../{ => device_common}/d2h_event_queue.hpp | 4 +- .../{ => device_common}/d2h_events_parser.cpp | 0 .../src/{ => device_common}/device.cpp | 55 +- .../{ => device_common}/device_internal.cpp | 59 +- .../{ => device_common}/device_internal.hpp | 10 +- hailort/libhailort/src/eth/CMakeLists.txt | 12 + .../libhailort/src/{ => eth}/eth_device.cpp | 146 +- .../libhailort/src/{ => eth}/eth_device.hpp | 19 +- .../libhailort/src/{ => eth}/eth_stream.cpp | 71 +- .../libhailort/src/{ => eth}/eth_stream.hpp | 57 +- .../src/eth/hcp_config_activated_core_op.cpp | 106 ++ .../src/eth/hcp_config_activated_core_op.hpp | 87 ++ .../libhailort/src/eth/hcp_config_core_op.cpp | 124 ++ .../libhailort/src/eth/hcp_config_core_op.hpp | 70 + .../src/{ => eth}/network_rate_calculator.cpp | 8 +- .../libhailort/src/{ => eth}/token_bucket.hpp | 0 hailort/libhailort/src/{ => eth}/udp.cpp | 14 +- hailort/libhailort/src/{ => eth}/udp.hpp | 5 +- hailort/libhailort/src/hailort.cpp | 44 +- hailort/libhailort/src/hailort_defaults.cpp | 385 +++++ hailort/libhailort/src/hailort_defaults.hpp | 375 ----- hailort/libhailort/src/hef/CMakeLists.txt | 9 + .../context_switch_actions.cpp | 209 +-- .../context_switch_actions.hpp | 64 +- .../core_op_metadata.cpp} | 155 +- .../core_op_metadata.hpp} | 80 +- hailort/libhailort/src/{ => hef}/hef.cpp | 814 ++++++++--- .../libhailort/src/{ => hef}/hef_internal.hpp | 91 +- .../libhailort/src/{ => hef}/layer_info.hpp | 18 +- hailort/libhailort/src/hw_consts.hpp | 1 - hailort/libhailort/src/mipi/CMakeLists.txt | 7 + .../libhailort/src/{ => mipi}/mipi_stream.cpp | 19 +- .../libhailort/src/{ => mipi}/mipi_stream.hpp | 18 +- .../libhailort/src/net_flow/CMakeLists.txt | 17 +- .../src/net_flow/ops/nms_post_process.cpp | 133 ++ .../src/net_flow/ops/nms_post_process.hpp | 166 +++ hailort/libhailort/src/net_flow/ops/op.hpp | 86 ++ .../src/net_flow/ops/ssd_post_process.cpp | 202 +++ .../src/net_flow/ops/ssd_post_process.hpp | 200 +++ .../src/net_flow/ops/yolo_post_process.cpp | 117 ++ .../src/net_flow/ops/yolo_post_process.hpp | 198 +++ .../src/net_flow/ops/yolo_post_processing.hpp | 386 ----- .../pipeline}/inference_pipeline.cpp | 24 +- .../src/{ => net_flow/pipeline}/pipeline.cpp | 11 +- .../src/{ => net_flow/pipeline}/pipeline.hpp | 56 +- .../src/{ => net_flow/pipeline}/vstream.cpp | 369 ++--- .../pipeline}/vstream_internal.hpp | 71 +- .../src/network_group/CMakeLists.txt | 7 + .../src/network_group/network_group.cpp | 408 ++++++ .../network_group_internal.hpp | 220 +-- .../src/network_group_scheduler.cpp | 924 ------------ .../src/network_group_scheduler.hpp | 165 --- hailort/libhailort/src/os/hailort_driver.hpp | 62 +- hailort/libhailort/src/os/mmap_buffer.hpp | 31 +- .../src/os/posix/hailort_driver.cpp | 263 ++-- .../libhailort/src/os/posix/mmap_buffer.cpp | 21 +- .../src/os/posix/qnx/driver_scan.cpp | 2 +- hailort/libhailort/src/os/posix/qnx/event.cpp | 8 +- .../src/os/posix/unix/driver_scan.cpp | 6 +- .../libhailort/src/os/posix/unix/event.cpp | 8 +- .../libhailort/src/os/windows/driver_scan.cpp | 2 +- hailort/libhailort/src/os/windows/event.cpp | 8 +- .../src/os/windows/hailort_driver.cpp | 211 ++- hailort/libhailort/src/pcie_stream.cpp | 70 - hailort/libhailort/src/pcie_stream.hpp | 65 - .../libhailort/src/pipeline_multiplexer.cpp | 417 ------ hailort/libhailort/src/rpc_client_utils.hpp | 82 -- hailort/libhailort/src/scheduled_stream.hpp | 115 -- hailort/libhailort/src/scheduler_oracle.cpp | 74 - hailort/libhailort/src/scheduler_oracle.hpp | 34 - hailort/libhailort/src/service/CMakeLists.txt | 8 + .../src/{ => service}/hailort_rpc_client.cpp | 224 ++- .../src/{ => service}/hailort_rpc_client.hpp | 15 +- .../{ => service}/network_group_client.cpp | 71 +- .../src/service/rpc_client_utils.hpp | 152 ++ .../src/stream_common/CMakeLists.txt | 8 + .../src/{ => stream_common}/stream.cpp | 38 +- .../{ => stream_common}/stream_internal.cpp | 23 +- .../{ => stream_common}/stream_internal.hpp | 53 +- hailort/libhailort/src/tracer.hpp | 240 --- .../libhailort/src/transform/CMakeLists.txt | 7 + .../src/{ => transform}/transform.cpp | 148 +- .../{ => transform}/transform_internal.hpp | 6 +- hailort/libhailort/src/utils/CMakeLists.txt | 15 + hailort/libhailort/src/{ => utils}/buffer.cpp | 0 .../src/{ => utils}/event_internal.hpp | 0 .../src/{ => utils}/hailort_common.cpp | 0 .../src/{ => utils}/hailort_logger.cpp | 38 +- .../src/{ => utils}/hailort_logger.hpp | 0 .../src/utils/profiler/CMakeLists.txt | 7 + .../src/{ => utils/profiler}/tracer.cpp | 47 +- .../libhailort/src/utils/profiler/tracer.hpp | 242 ++++ .../{ => utils/profiler}/tracer_macros.hpp | 0 .../src/{ => utils}/sensor_config_utils.cpp | 4 +- .../src/{ => utils}/sensor_config_utils.hpp | 0 .../{ => utils}/shared_resource_manager.hpp | 0 .../src/{ => utils}/thread_safe_map.hpp | 0 .../src/{ => utils}/thread_safe_queue.hpp | 7 +- hailort/libhailort/src/vdevice/CMakeLists.txt | 17 + .../src/vdevice/pipeline_multiplexer.cpp | 454 ++++++ .../{ => vdevice}/pipeline_multiplexer.hpp | 63 +- .../multi_device_scheduled_stream.cpp | 59 +- .../multi_device_scheduled_stream.hpp | 69 +- .../scheduler/network_group_scheduler.cpp | 1006 +++++++++++++ .../scheduler/network_group_scheduler.hpp | 144 ++ .../scheduler/scheduled_core_op_cv.hpp | 82 ++ .../scheduler/scheduled_core_op_state.cpp} | 234 +-- .../scheduler/scheduled_core_op_state.hpp} | 74 +- .../vdevice/scheduler/scheduled_stream.hpp | 105 ++ .../src/vdevice/scheduler/scheduler_base.hpp | 121 ++ .../{ => vdevice/scheduler}/scheduler_mon.hpp | 6 +- .../vdevice/scheduler/scheduler_oracle.cpp | 109 ++ .../vdevice/scheduler/scheduler_oracle.hpp | 39 + .../libhailort/src/{ => vdevice}/vdevice.cpp | 342 +++-- .../src/vdevice/vdevice_core_op.cpp | 410 ++++++ .../src/vdevice/vdevice_core_op.hpp | 174 +++ .../src/{ => vdevice}/vdevice_internal.hpp | 46 +- .../{ => vdevice}/vdevice_native_stream.hpp | 26 +- .../src/{ => vdevice}/vdevice_stream.cpp | 147 +- .../src/{ => vdevice}/vdevice_stream.hpp | 59 +- .../vdevice_stream_multiplexer_wrapper.cpp | 121 +- .../vdevice_stream_multiplexer_wrapper.hpp | 60 +- hailort/libhailort/src/vdma/CMakeLists.txt | 31 + .../src/vdma/channel/async_channel.cpp | 209 +++ .../src/vdma/channel/async_channel.hpp | 77 + .../src/vdma/channel/boundary_channel.cpp | 361 +++++ .../src/vdma/channel/boundary_channel.hpp | 141 ++ .../src/vdma/channel/buffered_channel.cpp | 573 ++++++++ .../src/vdma/channel/buffered_channel.hpp | 90 ++ .../src/vdma/channel/channel_base.cpp | 219 +++ .../src/vdma/channel/channel_base.hpp | 103 ++ .../src/vdma/{ => channel}/channel_id.hpp | 0 .../src/vdma/channel/channel_state.cpp | 240 +++ .../src/vdma/channel/channel_state.hpp | 178 +++ .../vdma/channel/interrupts_dispatcher.cpp | 106 ++ .../vdma/channel/interrupts_dispatcher.hpp | 54 + .../{ => vdma/channel}/vdma_channel_regs.hpp | 41 +- .../src/vdma/integrated/integrated_device.cpp | 78 + .../integrated/integrated_device.hpp} | 24 +- hailort/libhailort/src/vdma/mapped_buffer.cpp | 145 -- hailort/libhailort/src/vdma/mapped_buffer.hpp | 116 -- .../vdma/{ => memory}/continuous_buffer.cpp | 15 +- .../vdma/{ => memory}/continuous_buffer.hpp | 9 +- .../memory/descriptor_list.cpp} | 165 ++- .../memory/descriptor_list.hpp} | 78 +- .../src/vdma/memory/dma_mapped_buffer.cpp | 96 ++ .../src/vdma/memory/mapped_buffer_factory.cpp | 30 + .../src/vdma/memory/mapped_buffer_factory.hpp | 33 + .../src/vdma/memory/mapped_buffer_impl.cpp | 279 ++++ .../src/vdma/memory/mapped_buffer_impl.hpp | 98 ++ .../libhailort/src/vdma/memory/sg_buffer.cpp | 137 ++ .../src/vdma/{ => memory}/sg_buffer.hpp | 43 +- .../src/vdma/{ => memory}/vdma_buffer.cpp | 15 +- .../src/vdma/{ => memory}/vdma_buffer.hpp | 13 +- .../src/{ => vdma/pcie}/pcie_device.cpp | 46 +- .../src/{ => vdma/pcie}/pcie_device.hpp | 10 +- hailort/libhailort/src/vdma/sg_buffer.cpp | 109 -- .../libhailort/src/vdma/vdma_async_stream.cpp | 90 ++ .../libhailort/src/vdma/vdma_async_stream.hpp | 62 + .../vdma/vdma_config_activated_core_op.cpp | 140 ++ .../vdma/vdma_config_activated_core_op.hpp | 80 + .../src/vdma/vdma_config_core_op.cpp | 168 +++ .../src/vdma/vdma_config_core_op.hpp | 98 ++ .../src/vdma/vdma_config_manager.cpp | 55 + .../src/vdma/vdma_config_manager.hpp | 35 + .../libhailort/src/{ => vdma}/vdma_device.cpp | 269 ++-- .../libhailort/src/{ => vdma}/vdma_device.hpp | 24 +- .../src/vdma/vdma_mapped_buffer_impl.cpp | 110 -- .../src/vdma/vdma_mapped_buffer_impl.hpp | 116 -- hailort/libhailort/src/vdma/vdma_stream.cpp | 134 ++ hailort/libhailort/src/vdma/vdma_stream.hpp | 74 + .../libhailort/src/vdma/vdma_stream_base.cpp | 443 ++++++ .../libhailort/src/vdma/vdma_stream_base.hpp | 111 ++ hailort/libhailort/src/vdma_channel.cpp | 1283 ----------------- hailort/libhailort/src/vdma_channel.hpp | 233 --- hailort/libhailort/src/vdma_stream.cpp | 498 ------- hailort/libhailort/src/vdma_stream.hpp | 132 -- hailort/rpc/hailort_rpc.proto | 38 +- hailort/rpc/rpc_definitions.hpp | 8 +- hailort/scripts/download_firmware_eth.cmd | 2 +- hailort/scripts/download_firmware_eth.sh | 2 +- hailort/scripts/download_hefs.cmd | 2 +- hailort/scripts/download_hefs.sh | 2 +- hailort/tools/hw_debug/CMakeLists.txt | 2 +- hailort/tools/hw_debug/driver_memory.cpp | 40 +- ...{mercury_fields.cpp => hailo15_fields.cpp} | 8 +- ...{mercury_fields.hpp => hailo15_fields.hpp} | 10 +- .../dram_dma_engine_config_macros.h | 0 .../dram_dma_engine_config_regs.h | 0 .../dram_dma_package_macros.h | 0 386 files changed, 19226 insertions(+), 12046 deletions(-) create mode 100644 hailort/common/device_measurements.cpp create mode 100644 hailort/common/device_measurements.hpp create mode 100644 hailort/common/os/posix/os_utils.cpp create mode 100644 hailort/common/os/windows/os_utils.cpp create mode 100644 hailort/common/os_utils.hpp rename hailort/hailort_service/{ => unix}/hailort_service.cpp (78%) create mode 100644 hailort/hailort_service/windows/hailort_service.cpp create mode 100644 hailort/hailort_service/windows/hailort_service_env_vars.bat create mode 100644 hailort/hailortcli/run2/measurement_live_track.cpp create mode 100644 hailort/hailortcli/run2/measurement_live_track.hpp delete mode 100644 hailort/hailortcli/temp_measurement.cpp delete mode 100644 hailort/hailortcli/temp_measurement.hpp delete mode 100644 hailort/libhailort/bindings/python/examples/hef_infer_pipeline_vstream.py delete mode 100644 hailort/libhailort/bindings/python/examples/hef_infer_virtual_stream.py create mode 100644 hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/version_action.py create mode 100644 hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt create mode 100644 hailort/libhailort/examples/c/notification_callback_example/notification_callback_example.c create mode 100644 hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt create mode 100644 hailort/libhailort/examples/cpp/notification_callback_example/notification_callback_example.cpp create mode 100644 hailort/libhailort/examples/cpp/raw_async_streams_example/CMakeLists.txt create mode 100644 hailort/libhailort/examples/cpp/raw_async_streams_example/buffer_pool.cpp create mode 100644 hailort/libhailort/examples/cpp/raw_async_streams_example/buffer_pool.hpp create mode 100644 hailort/libhailort/examples/cpp/raw_async_streams_example/raw_async_streams_example.cpp create mode 100644 hailort/libhailort/include/hailo/dma_mapped_buffer.hpp create mode 100644 hailort/libhailort/include/hailo/hailort_defaults.hpp delete mode 100644 hailort/libhailort/src/context_switch/active_network_group_holder.hpp delete mode 100644 hailort/libhailort/src/context_switch/hcp_config_activated_network_group.cpp delete mode 100644 hailort/libhailort/src/context_switch/hcp_config_network_group.cpp delete mode 100644 hailort/libhailort/src/context_switch/multi_context/vdma_config_activated_network_group.hpp delete mode 100644 hailort/libhailort/src/context_switch/multi_context/vdma_config_manager.hpp delete mode 100644 hailort/libhailort/src/context_switch/multi_context/vdma_config_network_group.hpp delete mode 100644 hailort/libhailort/src/context_switch/single_context/hcp_config_activated_network_group.hpp delete mode 100644 hailort/libhailort/src/context_switch/single_context/hcp_config_network_group.hpp delete mode 100644 hailort/libhailort/src/context_switch/vdevice_network_group.cpp delete mode 100644 hailort/libhailort/src/context_switch/vdevice_network_group.hpp delete mode 100644 hailort/libhailort/src/context_switch/vdma_config_activated_network_group.cpp delete mode 100644 hailort/libhailort/src/context_switch/vdma_config_network_group.cpp delete mode 100644 hailort/libhailort/src/core_device.cpp create mode 100644 hailort/libhailort/src/core_op/CMakeLists.txt create mode 100644 hailort/libhailort/src/core_op/active_core_op_holder.hpp rename hailort/libhailort/src/{context_switch/network_group.cpp => core_op/core_op.cpp} (50%) create mode 100644 hailort/libhailort/src/core_op/core_op.hpp rename hailort/libhailort/src/{ => core_op/resource_manager}/channel_allocator.cpp (88%) rename hailort/libhailort/src/{ => core_op/resource_manager}/channel_allocator.hpp (84%) rename hailort/libhailort/src/{ => core_op/resource_manager}/config_buffer.cpp (76%) rename hailort/libhailort/src/{ => core_op/resource_manager}/config_buffer.hpp (93%) rename hailort/libhailort/src/{context_switch => core_op/resource_manager}/context_switch_buffer_builder.cpp (97%) rename hailort/libhailort/src/{context_switch => core_op/resource_manager}/context_switch_buffer_builder.hpp (92%) rename hailort/libhailort/src/{ => core_op/resource_manager}/ddr_channels_pair.cpp (75%) rename hailort/libhailort/src/{ => core_op/resource_manager}/ddr_channels_pair.hpp (91%) rename hailort/libhailort/src/{ => core_op/resource_manager}/inter_context_buffer.cpp (76%) rename hailort/libhailort/src/{ => core_op/resource_manager}/inter_context_buffer.hpp (85%) rename hailort/libhailort/src/{context_switch => core_op/resource_manager}/resource_manager.cpp (51%) rename hailort/libhailort/src/{context_switch/multi_context => core_op/resource_manager}/resource_manager.hpp (58%) rename hailort/libhailort/src/{context_switch => core_op/resource_manager}/resource_manager_builder.cpp (72%) rename hailort/libhailort/src/{context_switch => core_op/resource_manager}/resource_manager_builder.hpp (81%) delete mode 100644 hailort/libhailort/src/core_stream.cpp delete mode 100644 hailort/libhailort/src/core_stream.hpp create mode 100644 hailort/libhailort/src/device_common/CMakeLists.txt rename hailort/libhailort/src/{ => device_common}/control.cpp (92%) rename hailort/libhailort/src/{ => device_common}/control.hpp (95%) rename hailort/libhailort/src/{ => device_common}/control_protocol.cpp (97%) rename hailort/libhailort/src/{ => device_common}/control_protocol.hpp (98%) rename hailort/libhailort/src/{ => device_common}/d2h_event_queue.hpp (93%) rename hailort/libhailort/src/{ => device_common}/d2h_events_parser.cpp (100%) rename hailort/libhailort/src/{ => device_common}/device.cpp (93%) rename hailort/libhailort/src/{ => device_common}/device_internal.cpp (95%) rename hailort/libhailort/src/{ => device_common}/device_internal.hpp (97%) create mode 100644 hailort/libhailort/src/eth/CMakeLists.txt rename hailort/libhailort/src/{ => eth}/eth_device.cpp (81%) rename hailort/libhailort/src/{ => eth}/eth_device.hpp (78%) rename hailort/libhailort/src/{ => eth}/eth_stream.cpp (93%) rename hailort/libhailort/src/{ => eth}/eth_stream.hpp (77%) create mode 100644 hailort/libhailort/src/eth/hcp_config_activated_core_op.cpp create mode 100644 hailort/libhailort/src/eth/hcp_config_activated_core_op.hpp create mode 100644 hailort/libhailort/src/eth/hcp_config_core_op.cpp create mode 100644 hailort/libhailort/src/eth/hcp_config_core_op.hpp rename hailort/libhailort/src/{ => eth}/network_rate_calculator.cpp (99%) rename hailort/libhailort/src/{ => eth}/token_bucket.hpp (100%) rename hailort/libhailort/src/{ => eth}/udp.cpp (98%) rename hailort/libhailort/src/{ => eth}/udp.hpp (98%) create mode 100644 hailort/libhailort/src/hailort_defaults.cpp delete mode 100644 hailort/libhailort/src/hailort_defaults.hpp create mode 100644 hailort/libhailort/src/hef/CMakeLists.txt rename hailort/libhailort/src/{context_switch => hef}/context_switch_actions.cpp (87%) rename hailort/libhailort/src/{context_switch => hef}/context_switch_actions.hpp (94%) rename hailort/libhailort/src/{network_group_metadata.cpp => hef/core_op_metadata.cpp} (67%) rename hailort/libhailort/src/{network_group_metadata.hpp => hef/core_op_metadata.hpp} (72%) rename hailort/libhailort/src/{ => hef}/hef.cpp (78%) rename hailort/libhailort/src/{ => hef}/hef_internal.hpp (88%) rename hailort/libhailort/src/{ => hef}/layer_info.hpp (91%) create mode 100644 hailort/libhailort/src/mipi/CMakeLists.txt rename hailort/libhailort/src/{ => mipi}/mipi_stream.cpp (94%) rename hailort/libhailort/src/{ => mipi}/mipi_stream.hpp (80%) create mode 100644 hailort/libhailort/src/net_flow/ops/nms_post_process.cpp create mode 100644 hailort/libhailort/src/net_flow/ops/nms_post_process.hpp create mode 100644 hailort/libhailort/src/net_flow/ops/op.hpp create mode 100644 hailort/libhailort/src/net_flow/ops/ssd_post_process.cpp create mode 100644 hailort/libhailort/src/net_flow/ops/ssd_post_process.hpp create mode 100644 hailort/libhailort/src/net_flow/ops/yolo_post_process.cpp create mode 100644 hailort/libhailort/src/net_flow/ops/yolo_post_process.hpp delete mode 100644 hailort/libhailort/src/net_flow/ops/yolo_post_processing.hpp rename hailort/libhailort/src/{ => net_flow/pipeline}/inference_pipeline.cpp (95%) rename hailort/libhailort/src/{ => net_flow/pipeline}/pipeline.cpp (99%) rename hailort/libhailort/src/{ => net_flow/pipeline}/pipeline.hpp (94%) rename hailort/libhailort/src/{ => net_flow/pipeline}/vstream.cpp (89%) rename hailort/libhailort/src/{ => net_flow/pipeline}/vstream_internal.hpp (91%) create mode 100644 hailort/libhailort/src/network_group/CMakeLists.txt create mode 100644 hailort/libhailort/src/network_group/network_group.cpp rename hailort/libhailort/src/{context_switch => network_group}/network_group_internal.hpp (67%) delete mode 100644 hailort/libhailort/src/network_group_scheduler.cpp delete mode 100644 hailort/libhailort/src/network_group_scheduler.hpp delete mode 100644 hailort/libhailort/src/pcie_stream.cpp delete mode 100644 hailort/libhailort/src/pcie_stream.hpp delete mode 100644 hailort/libhailort/src/pipeline_multiplexer.cpp delete mode 100644 hailort/libhailort/src/rpc_client_utils.hpp delete mode 100644 hailort/libhailort/src/scheduled_stream.hpp delete mode 100644 hailort/libhailort/src/scheduler_oracle.cpp delete mode 100644 hailort/libhailort/src/scheduler_oracle.hpp create mode 100644 hailort/libhailort/src/service/CMakeLists.txt rename hailort/libhailort/src/{ => service}/hailort_rpc_client.cpp (84%) rename hailort/libhailort/src/{ => service}/hailort_rpc_client.hpp (89%) rename hailort/libhailort/src/{ => service}/network_group_client.cpp (83%) create mode 100644 hailort/libhailort/src/service/rpc_client_utils.hpp create mode 100644 hailort/libhailort/src/stream_common/CMakeLists.txt rename hailort/libhailort/src/{ => stream_common}/stream.cpp (79%) rename hailort/libhailort/src/{ => stream_common}/stream_internal.cpp (66%) rename hailort/libhailort/src/{ => stream_common}/stream_internal.hpp (76%) delete mode 100644 hailort/libhailort/src/tracer.hpp create mode 100644 hailort/libhailort/src/transform/CMakeLists.txt rename hailort/libhailort/src/{ => transform}/transform.cpp (94%) rename hailort/libhailort/src/{ => transform}/transform_internal.hpp (97%) create mode 100644 hailort/libhailort/src/utils/CMakeLists.txt rename hailort/libhailort/src/{ => utils}/buffer.cpp (100%) rename hailort/libhailort/src/{ => utils}/event_internal.hpp (100%) rename hailort/libhailort/src/{ => utils}/hailort_common.cpp (100%) rename hailort/libhailort/src/{ => utils}/hailort_logger.cpp (78%) rename hailort/libhailort/src/{ => utils}/hailort_logger.hpp (100%) create mode 100644 hailort/libhailort/src/utils/profiler/CMakeLists.txt rename hailort/libhailort/src/{ => utils/profiler}/tracer.cpp (80%) create mode 100644 hailort/libhailort/src/utils/profiler/tracer.hpp rename hailort/libhailort/src/{ => utils/profiler}/tracer_macros.hpp (100%) rename hailort/libhailort/src/{ => utils}/sensor_config_utils.cpp (99%) rename hailort/libhailort/src/{ => utils}/sensor_config_utils.hpp (100%) rename hailort/libhailort/src/{ => utils}/shared_resource_manager.hpp (100%) rename hailort/libhailort/src/{ => utils}/thread_safe_map.hpp (100%) rename hailort/libhailort/src/{ => utils}/thread_safe_queue.hpp (99%) create mode 100644 hailort/libhailort/src/vdevice/CMakeLists.txt create mode 100644 hailort/libhailort/src/vdevice/pipeline_multiplexer.cpp rename hailort/libhailort/src/{ => vdevice}/pipeline_multiplexer.hpp (52%) rename hailort/libhailort/src/{ => vdevice/scheduler}/multi_device_scheduled_stream.cpp (60%) rename hailort/libhailort/src/{ => vdevice/scheduler}/multi_device_scheduled_stream.hpp (75%) create mode 100644 hailort/libhailort/src/vdevice/scheduler/network_group_scheduler.cpp create mode 100644 hailort/libhailort/src/vdevice/scheduler/network_group_scheduler.hpp create mode 100644 hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_cv.hpp rename hailort/libhailort/src/{scheduled_network_group.cpp => vdevice/scheduler/scheduled_core_op_state.cpp} (54%) rename hailort/libhailort/src/{scheduled_network_group.hpp => vdevice/scheduler/scheduled_core_op_state.hpp} (73%) create mode 100644 hailort/libhailort/src/vdevice/scheduler/scheduled_stream.hpp create mode 100644 hailort/libhailort/src/vdevice/scheduler/scheduler_base.hpp rename hailort/libhailort/src/{ => vdevice/scheduler}/scheduler_mon.hpp (93%) create mode 100644 hailort/libhailort/src/vdevice/scheduler/scheduler_oracle.cpp create mode 100644 hailort/libhailort/src/vdevice/scheduler/scheduler_oracle.hpp rename hailort/libhailort/src/{ => vdevice}/vdevice.cpp (58%) create mode 100644 hailort/libhailort/src/vdevice/vdevice_core_op.cpp create mode 100644 hailort/libhailort/src/vdevice/vdevice_core_op.hpp rename hailort/libhailort/src/{ => vdevice}/vdevice_internal.hpp (74%) rename hailort/libhailort/src/{ => vdevice}/vdevice_native_stream.hpp (74%) rename hailort/libhailort/src/{ => vdevice}/vdevice_stream.cpp (77%) rename hailort/libhailort/src/{ => vdevice}/vdevice_stream.hpp (74%) rename hailort/libhailort/src/{ => vdevice}/vdevice_stream_multiplexer_wrapper.cpp (78%) rename hailort/libhailort/src/{ => vdevice}/vdevice_stream_multiplexer_wrapper.hpp (68%) create mode 100644 hailort/libhailort/src/vdma/CMakeLists.txt create mode 100644 hailort/libhailort/src/vdma/channel/async_channel.cpp create mode 100644 hailort/libhailort/src/vdma/channel/async_channel.hpp create mode 100644 hailort/libhailort/src/vdma/channel/boundary_channel.cpp create mode 100644 hailort/libhailort/src/vdma/channel/boundary_channel.hpp create mode 100644 hailort/libhailort/src/vdma/channel/buffered_channel.cpp create mode 100644 hailort/libhailort/src/vdma/channel/buffered_channel.hpp create mode 100644 hailort/libhailort/src/vdma/channel/channel_base.cpp create mode 100644 hailort/libhailort/src/vdma/channel/channel_base.hpp rename hailort/libhailort/src/vdma/{ => channel}/channel_id.hpp (100%) create mode 100644 hailort/libhailort/src/vdma/channel/channel_state.cpp create mode 100644 hailort/libhailort/src/vdma/channel/channel_state.hpp create mode 100644 hailort/libhailort/src/vdma/channel/interrupts_dispatcher.cpp create mode 100644 hailort/libhailort/src/vdma/channel/interrupts_dispatcher.hpp rename hailort/libhailort/src/{ => vdma/channel}/vdma_channel_regs.hpp (64%) create mode 100644 hailort/libhailort/src/vdma/integrated/integrated_device.cpp rename hailort/libhailort/src/{core_device.hpp => vdma/integrated/integrated_device.hpp} (68%) delete mode 100644 hailort/libhailort/src/vdma/mapped_buffer.cpp delete mode 100644 hailort/libhailort/src/vdma/mapped_buffer.hpp rename hailort/libhailort/src/vdma/{ => memory}/continuous_buffer.cpp (89%) rename hailort/libhailort/src/vdma/{ => memory}/continuous_buffer.hpp (90%) rename hailort/libhailort/src/{vdma_descriptor_list.cpp => vdma/memory/descriptor_list.cpp} (64%) rename hailort/libhailort/src/{vdma_descriptor_list.hpp => vdma/memory/descriptor_list.hpp} (66%) create mode 100644 hailort/libhailort/src/vdma/memory/dma_mapped_buffer.cpp create mode 100644 hailort/libhailort/src/vdma/memory/mapped_buffer_factory.cpp create mode 100644 hailort/libhailort/src/vdma/memory/mapped_buffer_factory.hpp create mode 100644 hailort/libhailort/src/vdma/memory/mapped_buffer_impl.cpp create mode 100644 hailort/libhailort/src/vdma/memory/mapped_buffer_impl.hpp create mode 100644 hailort/libhailort/src/vdma/memory/sg_buffer.cpp rename hailort/libhailort/src/vdma/{ => memory}/sg_buffer.hpp (53%) rename hailort/libhailort/src/vdma/{ => memory}/vdma_buffer.cpp (50%) rename hailort/libhailort/src/vdma/{ => memory}/vdma_buffer.hpp (78%) rename hailort/libhailort/src/{ => vdma/pcie}/pcie_device.cpp (85%) rename hailort/libhailort/src/{ => vdma/pcie}/pcie_device.hpp (94%) delete mode 100644 hailort/libhailort/src/vdma/sg_buffer.cpp create mode 100644 hailort/libhailort/src/vdma/vdma_async_stream.cpp create mode 100644 hailort/libhailort/src/vdma/vdma_async_stream.hpp create mode 100644 hailort/libhailort/src/vdma/vdma_config_activated_core_op.cpp create mode 100644 hailort/libhailort/src/vdma/vdma_config_activated_core_op.hpp create mode 100644 hailort/libhailort/src/vdma/vdma_config_core_op.cpp create mode 100644 hailort/libhailort/src/vdma/vdma_config_core_op.hpp create mode 100644 hailort/libhailort/src/vdma/vdma_config_manager.cpp create mode 100644 hailort/libhailort/src/vdma/vdma_config_manager.hpp rename hailort/libhailort/src/{ => vdma}/vdma_device.cpp (55%) rename hailort/libhailort/src/{ => vdma}/vdma_device.hpp (61%) delete mode 100644 hailort/libhailort/src/vdma/vdma_mapped_buffer_impl.cpp delete mode 100644 hailort/libhailort/src/vdma/vdma_mapped_buffer_impl.hpp create mode 100644 hailort/libhailort/src/vdma/vdma_stream.cpp create mode 100644 hailort/libhailort/src/vdma/vdma_stream.hpp create mode 100644 hailort/libhailort/src/vdma/vdma_stream_base.cpp create mode 100644 hailort/libhailort/src/vdma/vdma_stream_base.hpp delete mode 100644 hailort/libhailort/src/vdma_channel.cpp delete mode 100644 hailort/libhailort/src/vdma_channel.hpp delete mode 100644 hailort/libhailort/src/vdma_stream.cpp delete mode 100644 hailort/libhailort/src/vdma_stream.hpp rename hailort/tools/hw_debug/{mercury_fields.cpp => hailo15_fields.cpp} (96%) rename hailort/tools/hw_debug/{mercury_fields.hpp => hailo15_fields.hpp} (90%) rename hailort/tools/hw_debug/hw_consts/{mercury => hailo15}/dram_dma_engine_config_macros.h (100%) rename hailort/tools/hw_debug/hw_consts/{mercury => hailo15}/dram_dma_engine_config_regs.h (100%) rename hailort/tools/hw_debug/hw_consts/{mercury => hailo15}/dram_dma_package_macros.h (100%) diff --git a/README.md b/README.md index fff49e1..7e3ad5e 100644 --- a/README.md +++ b/README.md @@ -5,19 +5,18 @@ # HailoRT # -HailoRT is a light-weight and production-grade run-time library, which runs on the host processor, and -implements a robust user-space run-time library (HailoRT Library) responsible for operating a Hailo device, with intuitive APIs in C/C++ for optimized performance. +HailoRT is a lightweight, production-grade runtime library that runs on the host processor and provides a robust +user-space runtime library (the HailoRT Library) with intuitive APIs in C/C++ for optimized performance -HailoRT is comprised of the following main components: +HailoRT consists of the following main components: - HailoRT Library. -- HailoRT CLI - command line application used to control the Hailo device, run inference using the device, - collect inference statistics and device events, etc. -- [**HailoRT PCIe Driver**](https://github.com/hailo-ai/hailort-drivers) - the device driver used to manage the Hailo device, communicate with the device and transfer - data to/from the device. The PCIe driver includes the Hailo-8 firmware that runs on the Hailo device, manages the boot and control of the Hailo device. -- pyHailoRT - HailoRT Python API (wraps the run-time library) +- HailoRT CLI - a command line application used to control the Hailo device, run inferences, collect statistics and device events, etc. +- [**HailoRT PCIe Driver**](https://github.com/hailo-ai/hailort-drivers) - the device driver used to manage the Hailo device, communicate with the device, +and transfer data to/from the device; it includes the Hailo-8 firmware that runs on the Hailo device and manages its boot and control. +- pyHailoRT - HailoRT Python API, which wraps the runtime library. - HailoRT GStreamer element (HailoNet). -HailoRT supports Linux and Windows, and can be compiled from sources to be integrated with various x86 and ARM processors. +HailoRT supports Linux and Windows, and it can be compiled from sources to be integrated with various x86 and ARM processors. ## Usage diff --git a/common/include/control_protocol.h b/common/include/control_protocol.h index 48952a1..c5889ec 100644 --- a/common/include/control_protocol.h +++ b/common/include/control_protocol.h @@ -41,7 +41,10 @@ extern "C" { #define CONTROL_PROTOCOL__SOC_ID_LENGTH (32) #define CONTROL_PROTOCOL__MAX_CFG_CHANNELS (4) #define CONTROL_PROTOCOL__MAX_NETWORKS_PER_NETWORK_GROUP (8) +#define CONTROL_PROTOCOL__MAX_VDMA_CHANNELS_PER_ENGINE (32) #define CONTROL_PROTOCOL__MAX_VDMA_ENGINES_COUNT (3) +#define CONTROL_PROTOCOL__MAX_TOTAL_CHANNEL_COUNT \ + (CONTROL_PROTOCOL__MAX_VDMA_CHANNELS_PER_ENGINE * CONTROL_PROTOCOL__MAX_VDMA_ENGINES_COUNT) /* Tightly coupled with the sizeof PROCESS_MONITOR__detection_results_t and HAILO_SOC_PM_VALUES_BYTES_LENGTH */ #define PM_RESULTS_LENGTH (24) @@ -154,6 +157,7 @@ extern "C" { CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_CLEAR_CONFIGURED_APPS, false, CPU_ID_CORE_CPU)\ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_GET_HW_CONSTS, false, CPU_ID_CORE_CPU)\ CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_SET_SLEEP_STATE, false, CPU_ID_APP_CPU)\ + CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CHANGE_HW_INFER_STATUS, false, CPU_ID_CORE_CPU)\ typedef enum { #define CONTROL_PROTOCOL__OPCODE_X(name, is_critical, cpu_id) name, @@ -340,8 +344,7 @@ typedef enum { CONTROL_PROTOCOL__HAILO8_A0 = 0, CONTROL_PROTOCOL__HAILO8, CONTROL_PROTOCOL__HAILO8L, - CONTROL_PROTOCOL__MERCURY_CA, - CONTROL_PROTOCOL__MERCURY_VPU, + CONTROL_PROTOCOL__HAILO15, /* Must be last!! */ CONTROL_PROTOCOL__DEVICE_ARCHITECTURE_COUNT } CONTROL_PROTOCOL__device_architecture_t; @@ -868,7 +871,9 @@ typedef struct { CONTROL_PROTOCOL__INFER_FEATURE_LIST_t infer_features; CONTROL_PROTOCOL__VALIDATION_FEATURE_LIST_t validation_features; uint8_t networks_count; + uint16_t csm_buffer_size; uint16_t batch_size[CONTROL_PROTOCOL__MAX_NETWORKS_PER_NETWORK_GROUP]; + uint32_t boundary_channels_bitmap[CONTROL_PROTOCOL__MAX_VDMA_ENGINES_COUNT]; } CONTROL_PROTOCOL__application_header_t; typedef struct { @@ -933,7 +938,8 @@ typedef struct { uint8_t buffer_type; // CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t uint64_t dma_address; uint16_t desc_page_size; - uint32_t total_desc_count; + uint32_t total_desc_count; //HRT-9913 - Some descriptors may not be initialized (to save space), needs to + // change this param or add another one for validation. uint32_t bytes_in_pattern; } CONTROL_PROTOCOL__host_buffer_info_t; @@ -950,8 +956,6 @@ typedef struct { uint8_t is_last_control_per_context; uint32_t context_type_length; uint8_t context_type; // CONTROL_PROTOCOL__context_switch_context_type_t - uint32_t actions_count_length; - uint32_t actions_count; uint32_t context_network_data_length; uint8_t context_network_data[0]; } CONTROL_PROTOCOL__context_switch_set_context_info_request_t; @@ -959,11 +963,6 @@ typedef struct { #pragma warning(pop) #endif -typedef struct { - /* Must be first */ - uint8_t action_type; // CONTEXT_SWITCH_DEFS__ACTION_TYPE_t - bool is_repeated; -} CONTROL_PROTOCOL__ACTION_HEADER_t; typedef CONTROL_PROTOCOL__read_memory_request_t CONTROL_PROTOCOL__read_user_config_request_t; typedef CONTROL_PROTOCOL__read_memory_response_t CONTROL_PROTOCOL__read_user_config_response_t; @@ -1279,6 +1278,47 @@ typedef struct { CONTROL_PROTOCOL__hw_consts_t hw_consts; } CONTROL_PROTOCOL__get_hw_consts_response_t; +/* TODO HRT-9545 - Return and hw only parse results */ +typedef struct { + bool infer_done; + uint32_t infer_cycles; +} CONTROL_PROTOCOL__hw_only_infer_results_t; + +typedef struct { + uint32_t results_length; + CONTROL_PROTOCOL__hw_only_infer_results_t results; +} CONTROL_PROTOCOL__change_hw_infer_status_response_t; + +typedef struct { + uint8_t channel_index; + uint8_t engine_index; + uint16_t desc_programed; +} CONTROL_PROTOCOL__hw_infer_channel_info_t; + +typedef struct { + CONTROL_PROTOCOL__hw_infer_channel_info_t channel_info[CONTROL_PROTOCOL__MAX_TOTAL_CHANNEL_COUNT]; + uint8_t channel_count; +} CONTROL_PROTOCOL__hw_infer_channels_info_t; + +typedef enum { + CONTROL_PROTOCOL__HW_INFER_STATE_START, + CONTROL_PROTOCOL__HW_INFER_STATE_STOP, + + /* must be last*/ + CONTROL_PROTOCOL__HW_INFER_STATE_COUNT +} CONTROL_PROTOCOL__hw_infer_state_t; + +typedef struct { + uint32_t hw_infer_state_length; + uint8_t hw_infer_state; + uint32_t application_index_length; + uint8_t application_index; + uint32_t dynamic_batch_size_length; + uint16_t dynamic_batch_size; + uint32_t channels_info_length; + CONTROL_PROTOCOL__hw_infer_channels_info_t channels_info; +} CONTROL_PROTOCOL__change_hw_infer_status_request_t; + typedef union { CONTROL_PROTOCOL_identify_response_t identity_response; CONTROL_PROTOCOL__core_identify_response_t core_identity_response; @@ -1305,6 +1345,7 @@ typedef union { CONTROL_PROTOCOL__get_throttling_state_response_t get_throttling_state_response; CONTROL_PROTOCOL__get_overcurrent_state_response_t get_overcurrent_state_response; CONTROL_PROTOCOL__get_hw_consts_response_t get_hw_consts_response; + CONTROL_PROTOCOL__change_hw_infer_status_response_t change_hw_infer_status_response; // Note: This array is larger than any legal request: // * Functions in this module won't write more than CONTROL_PROTOCOL__MAX_CONTROL_LENGTH bytes @@ -1364,6 +1405,7 @@ typedef union { CONTROL_PROTOCOL__sensor_set_i2c_bus_index_t sensor_set_i2c_bus_index; CONTROL_PROTOCOL__set_overcurrent_state_request_t set_overcurrent_state_request; CONTROL_PROTOCOL__set_sleep_state_request_t set_sleep_state_request; + CONTROL_PROTOCOL__change_hw_infer_status_request_t change_hw_infer_status_request; // Note: This array is larger than any legal request: // * Functions in this module won't write more than CONTROL_PROTOCOL__MAX_CONTROL_LENGTH bytes // when recieving a pointer to CONTROL_PROTOCOL__request_parameters_t. @@ -1427,7 +1469,6 @@ typedef struct { bool is_first_control_per_context; bool is_last_control_per_context; uint8_t context_type; // CONTROL_PROTOCOL__context_switch_context_type_t - uint32_t actions_count; uint32_t context_network_data_length; uint8_t context_network_data[CONTROL_PROTOCOL__CONTEXT_NETWORK_DATA_SINGLE_CONTROL_MAX_SIZE]; } CONTROL_PROTOCOL__context_switch_context_info_single_control_t; diff --git a/common/include/firmware_header.h b/common/include/firmware_header.h index c7d5254..bd0d1c4 100644 --- a/common/include/firmware_header.h +++ b/common/include/firmware_header.h @@ -18,7 +18,7 @@ extern "C" { #include "utils.h" #define FIRMWARE_HEADER_MAGIC_HAILO8 (0x1DD89DE0) -#define FIRMWARE_HEADER_MAGIC_MERCURY (0xE905DAAB) +#define FIRMWARE_HEADER_MAGIC_HAILO15 (0xE905DAAB) typedef enum { FIRMWARE_HEADER_VERSION_INITIAL = 0, @@ -29,12 +29,12 @@ typedef enum { typedef enum { FIRMWARE_TYPE_HAILO8 = 0, - FIRMWARE_TYPE_MERCURY + FIRMWARE_TYPE_HAILO15 } firmware_type_t; #ifdef MERCURY -#define COMPILED_FIRMWARE_TYPE (FIRMWARE_TYPE_MERCURY) +#define COMPILED_FIRMWARE_TYPE (FIRMWARE_TYPE_HAILO15) #elif defined(HAILO8_B0) #define COMPILED_FIRMWARE_TYPE (FIRMWARE_TYPE_HAILO8) #endif /* MERCURY */ diff --git a/common/include/firmware_status.h b/common/include/firmware_status.h index 7eb20e2..193bfef 100644 --- a/common/include/firmware_status.h +++ b/common/include/firmware_status.h @@ -409,6 +409,8 @@ Updating rules: FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_SET_SLEEP_STATE_FAILED)\ FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_SLEEP_STATE_SIZE)\ FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_SLEEP_STATE)\ + FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_HW_INFER_STATE_LENGTH)\ + FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_CHANNELS_INFO_LENGTH)\ \ FIRMWARE_MODULE__X(FIRMWARE_MODULE__POWER_MEASUREMENT)\ FIRMWARE_STATUS__X(HAILO_POWER_MEASUREMENT_STATUS_POWER_INIT_ERROR)\ @@ -968,6 +970,8 @@ Updating rules: FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_CSM_BURST_COUNTER_IS_NOT_ZERO)\ FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_CSM_CREDIT_COUNTER_IS_NOT_ZERO)\ FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_CSM_FIFO_NOT_EMPTY)\ + FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_INVALID_HOST_PAGE_SIZE)\ + FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_INVALID_BUFFER_SIZE)\ \ FIRMWARE_MODULE__X(FIRMWARE_MODULE__PCIE_CONFIG_MANAGER)\ FIRMWARE_STATUS__X(PCIE_CONFIG_MANAGER_STATUS_NOT_IMPLEMENTED)\ @@ -1074,6 +1078,7 @@ Updating rules: FIRMWARE_STATUS__X(NMS_MANAGER_STATUS_NOT_SUPPORTED_OPERATION)\ FIRMWARE_STATUS__X(NMS_MANAGER_STATUS_INVALID_NETWORK_INDEX)\ FIRMWARE_STATUS__X(NMS_MANAGER_STATUS_INVALID_NMS_UNIT_INDEX)\ + FIRMWARE_STATUS__X(NMS_MANAGER_STATUS_INVALID_BATCH_SIZE)\ \ FIRMWARE_MODULE__X(FIRMWARE_MODULE__CLUSTER_MANAGER)\ FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_CLUSTER_INDEX)\ @@ -1082,6 +1087,12 @@ Updating rules: FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_LCU_INDEX)\ FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_KERNEL_DONE_ADDRESS)\ FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_RECEIVED_UNEXPECTED_INTERRUPT)\ + \ + FIRMWARE_MODULE__X(FIRMWARE_MODULE__HW_INFER_MANAGER)\ + FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_NETWORK_GROUP_NOT_CONFIGURED_BEFORE_INFER_START)\ + FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_NETWORK_GROUP_ALREADY_ACTIVATED)\ + FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_STATE_MACHINE_NOT_IN_RESET_STATE_BEFORE_DEACTIVATE)\ + FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_INVALID_STATE)\ typedef enum { diff --git a/common/src/firmware_header_utils.c b/common/src/firmware_header_utils.c index 51435ba..a3f162e 100644 --- a/common/src/firmware_header_utils.c +++ b/common/src/firmware_header_utils.c @@ -47,8 +47,8 @@ static HAILO_COMMON_STATUS_t firmware_header_utils__validate_fw_header(uintptr_t case FIRMWARE_TYPE_HAILO8: firmware_magic = FIRMWARE_HEADER_MAGIC_HAILO8; break; - case FIRMWARE_TYPE_MERCURY: - firmware_magic = FIRMWARE_HEADER_MAGIC_MERCURY; + case FIRMWARE_TYPE_HAILO15: + firmware_magic = FIRMWARE_HEADER_MAGIC_HAILO15; break; default: status = HAILO_STATUS__FIRMWARE_HEADER_UTILS__INVALID_FIRMWARE_TYPE; diff --git a/hailort/CMakeLists.txt b/hailort/CMakeLists.txt index 0ec5b00..7d90f9d 100644 --- a/hailort/CMakeLists.txt +++ b/hailort/CMakeLists.txt @@ -10,10 +10,6 @@ option(HAILO_OFFLINE_COMPILATION "Don't download external dependencies" OFF) option(HAILO_BUILD_SERVICE "Build hailort service" OFF) option(HAILO_BUILD_PROFILER "Build hailort profiler" ON) -if(WIN32 AND ${HAILO_BUILD_SERVICE}) - message(FATAL_ERROR "HailoRT service is not supported on Windows") -endif() - # Flag for emulator (FPGA/Veloce) if(HAILO_BUILD_EMULATOR) message(WARNING "HailoRT is building with Emulator flag on") @@ -22,8 +18,8 @@ endif() # Set firmware version add_definitions( -DFIRMWARE_VERSION_MAJOR=4 ) -add_definitions( -DFIRMWARE_VERSION_MINOR=12 ) -add_definitions( -DFIRMWARE_VERSION_REVISION=1 ) +add_definitions( -DFIRMWARE_VERSION_MINOR=13 ) +add_definitions( -DFIRMWARE_VERSION_REVISION=0 ) if(HAILO_BUILD_SERVICE) add_definitions( -DHAILO_SUPPORT_MULTI_PROCESS ) endif() @@ -86,7 +82,7 @@ if(HAILO_BUILD_PYBIND) if(NOT PYTHON_EXECUTABLE AND PYBIND11_PYTHON_VERSION) # PYBIND11_PYTHON_VERSION is prioritized (not virtual environment) if PYTHON_EXECUTABLE is not set. # See https://pybind11.readthedocs.io/en/stable/changelog.html#v2-6-0-oct-21-2020 - if(${CMAKE_VERSION} VERSION_LESS "3.22.0") + if((${CMAKE_VERSION} VERSION_LESS "3.22.0") AND (NOT WIN32)) find_package(PythonInterp ${PYBIND11_PYTHON_VERSION} REQUIRED) set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) else() diff --git a/hailort/common/CMakeLists.txt b/hailort/common/CMakeLists.txt index 34427de..1056647 100644 --- a/hailort/common/CMakeLists.txt +++ b/hailort/common/CMakeLists.txt @@ -14,10 +14,13 @@ set(SRC_FILES ${HAILORT_COMMON_OS_DIR}/filesystem.cpp ${HAILORT_COMMON_OS_DIR}/socket.cpp ${HAILORT_COMMON_OS_DIR}/process.cpp + ${HAILORT_COMMON_OS_DIR}/os_utils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/barrier.cpp ${CMAKE_CURRENT_SOURCE_DIR}/file_utils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/string_utils.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/device_measurements.cpp ) if(WIN32) diff --git a/hailort/common/async_thread.hpp b/hailort/common/async_thread.hpp index 6771852..049fd8a 100644 --- a/hailort/common/async_thread.hpp +++ b/hailort/common/async_thread.hpp @@ -13,6 +13,8 @@ #include #include +#include "common/os_utils.hpp" + namespace hailort { @@ -23,16 +25,29 @@ namespace hailort template class AsyncThread final { public: - explicit AsyncThread(std::function func) : + AsyncThread(const std::string &name, std::function func) : m_result(), - m_thread([this, func]() { + m_thread([this, name, func]() { + if (!name.empty()) { + OsUtils::set_current_thread_name(name); + } m_result = func(); }) {} + explicit AsyncThread(std::function func) : AsyncThread("", func) + {} + + ~AsyncThread() + { + // Join on the thread. this can be a blocking operation, so to avoid it the user must call .get() + // before the object gets destracted (same behavoiur as in std::future returned from std::async). + get(); + } + /** * NOTE! this object is not moveable by purpose, on creation we create a lambda that take `this`, if we - * move the object `this` will change and the callback will be wrong. Use exeternal storage like std::unique_ptr + * move the object `this` will change and the callback will be wrong. Use external storage like std::unique_ptr * to move the object (or to put it inside a container) */ AsyncThread(const AsyncThread &) = delete; diff --git a/hailort/common/circular_buffer.hpp b/hailort/common/circular_buffer.hpp index 24c8ca6..f8348f6 100644 --- a/hailort/common/circular_buffer.hpp +++ b/hailort/common/circular_buffer.hpp @@ -15,6 +15,7 @@ #include "hailo/platform.h" #include "common/utils.hpp" #include +#include namespace hailort { @@ -60,14 +61,43 @@ typedef struct { MIN(CB_PROG((circbuf), (head), (tail)), (circbuf).size - (tail)) +template +struct is_std_array : public std::false_type {}; + +template +struct is_std_array> : public std::true_type {}; + // TODO: implement more functionalities, better move semantic handle // TODO: support consts methods (front(), empty()), right now CB_* macros requires non const pointer to head+tail -template +template> class CircularArray final { public: - static_assert(std::is_pod::value, "CircularArray can be used only with POD type"); + static_assert(std::is_default_constructible::value, "CircularArray object must be default constructible"); + + // Based on https://en.cppreference.com/w/cpp/iterator/iterator + class iterator: public std::iterator // reference + { + public: + explicit iterator(int index, CircularArray &array) : m_array(array), m_index(index) {} + iterator& operator++() { m_index = ((m_index + 1) & m_array.m_circ.size_mask); return *this; } + iterator operator++(int) { iterator retval = *this; ++(*this); return retval; } + bool operator==(iterator other) const { return m_index == other.m_index; } + bool operator!=(iterator other) const { return !(*this == other); } + T &operator*() const { return m_array.m_array[m_index]; } + private: + CircularArray &m_array; + int m_index; + }; + + // Ctor for Container=std::vector + template >::value>> CircularArray(size_t storage_size) { // storage size must be a power of 2 @@ -76,23 +106,55 @@ public: m_array.resize(storage_size); } + // Ctor for Container=std::array + template ::value>> + CircularArray(size_t storage_size, int = 0) + { + // storage size must be a power of 2 + assert(is_powerof2(storage_size)); + assert(storage_size <= std::tuple_size::value); + CB_INIT(m_circ, storage_size); + } + + void push_back(T &&element) + { + assert(!full()); + m_array[CB_HEAD(m_circ)] = std::move(element); + CB_ENQUEUE(m_circ, 1); + } + void push_back(const T& element) { - assert(CB_AVAIL(m_circ, CB_HEAD(m_circ), CB_TAIL(m_circ))); + assert(!full()); m_array[CB_HEAD(m_circ)] = element; CB_ENQUEUE(m_circ, 1); } void pop_front() { + assert(!empty()); + // Clear previous front + m_array[CB_TAIL(m_circ)] = T(); CB_DEQUEUE(m_circ, 1); } T &front() { + assert(!empty()); return m_array[CB_TAIL(m_circ)]; } + void reset() + { + // pop all fronts to make sure all destructors are called. + // TODO: if T is std::is_trivial, we can just reset the counters + const auto original_size = size(); + for (size_t i = 0 ; i < original_size; i++) { + pop_front(); + } + } + bool empty() { return CB_HEAD(m_circ) == CB_TAIL(m_circ); @@ -108,9 +170,24 @@ public: return CB_PROG(m_circ, CB_HEAD(m_circ), CB_TAIL(m_circ)); } + size_t capacity() + { + return CB_SIZE(m_circ) - 1; + } + + iterator begin() + { + return iterator(CB_TAIL(m_circ), *this); + } + + iterator end() + { + return iterator(CB_HEAD(m_circ), *this); + } + private: circbuf_t m_circ; - std::vector m_array; + Container m_array; }; } /* namespace hailort */ diff --git a/hailort/common/device_measurements.cpp b/hailort/common/device_measurements.cpp new file mode 100644 index 0000000..89fd0d3 --- /dev/null +++ b/hailort/common/device_measurements.cpp @@ -0,0 +1,133 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file device_measurements.hpp + * @brief Measure temperature, power and current of Hailo chip + **/ + + +#include "common/device_measurements.hpp" +#include "common/utils.hpp" + +using namespace hailort; + +constexpr std::chrono::milliseconds DEFAULT_MEASUREMENTS_INTERVAL(100); + +BaseMeasurement::BaseMeasurement(Device &device, hailo_status &status) : + m_device(device), + m_is_thread_running(false), + m_acc(make_shared_nothrow>("BaseMeasurementAccumulator")) +{ + if (nullptr == m_acc) { + status = HAILO_OUT_OF_HOST_MEMORY; + return; + } + status = HAILO_SUCCESS; +} + +BaseMeasurement::~BaseMeasurement() +{ + stop_measurement(); +} + +void BaseMeasurement::stop_measurement() +{ + m_is_thread_running = false; + if (m_thread.joinable()) { + m_thread.join(); + } +} + +AccumulatorResults BaseMeasurement::get_data() +{ + std::unique_lock lock(m_mutex); + return m_acc->get(); +} + +Expected> TemperatureMeasurement::create_shared(Device &device) +{ + auto status = HAILO_UNINITIALIZED; + auto ptr = make_shared_nothrow(device, status); + CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + + +TemperatureMeasurement::TemperatureMeasurement(Device &device, hailo_status &status) : BaseMeasurement(device, status) +{} + +hailo_status TemperatureMeasurement::start_measurement() +{ + // Checking sensor before starting thread + auto temp_info = m_device.get_chip_temperature(); + CHECK_EXPECTED_AS_STATUS(temp_info); + + m_is_thread_running = true; + m_thread = std::thread([this] () { + while (m_is_thread_running.load()) { + auto temp_info = m_device.get_chip_temperature(); + if (HAILO_SUCCESS != temp_info.status()) { + LOGGER__ERROR("Failed to get chip's temperature, status = {}", temp_info.status()); + m_is_thread_running = false; + break; + } + + float32_t ts_avg = ((temp_info->ts0_temperature + temp_info->ts1_temperature) / 2); + { + std::unique_lock lock(m_mutex); + m_acc->add_data_point(ts_avg, temp_info->sample_count); + } + + std::this_thread::sleep_for(DEFAULT_MEASUREMENTS_INTERVAL); + } + }); + + return HAILO_SUCCESS; +} + +Expected> PowerMeasurement::create_shared(Device &device, + hailo_power_measurement_types_t measurement_type) +{ + auto status = HAILO_UNINITIALIZED; + auto ptr = make_shared_nothrow(device, measurement_type, status); + CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +PowerMeasurement::PowerMeasurement(Device &device, hailo_power_measurement_types_t measurement_type, hailo_status &status) + : BaseMeasurement(device, status), m_measurement_type(measurement_type) +{} + +hailo_status PowerMeasurement::start_measurement() +{ + // Checking sensor before starting thread + auto power_info = m_device.power_measurement(HAILO_DVM_OPTIONS_AUTO, m_measurement_type); + CHECK_EXPECTED_AS_STATUS(power_info); + + m_is_thread_running = true; + m_thread = std::thread([this] () { + while (m_is_thread_running.load()) { + auto power_info = m_device.power_measurement(HAILO_DVM_OPTIONS_AUTO, m_measurement_type); + if (HAILO_SUCCESS != power_info.status()) { + LOGGER__ERROR("Failed to get chip's power, status = {}", power_info.status()); + m_is_thread_running = false; + break; + } + + { + std::unique_lock lock(m_mutex); + m_acc->add_data_point(*power_info); + } + + std::this_thread::sleep_for(DEFAULT_MEASUREMENTS_INTERVAL); + } + }); + + return HAILO_SUCCESS; +} \ No newline at end of file diff --git a/hailort/common/device_measurements.hpp b/hailort/common/device_measurements.hpp new file mode 100644 index 0000000..6089be8 --- /dev/null +++ b/hailort/common/device_measurements.hpp @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file device_measurements.hpp + * @brief Measure temperature, power and current of Hailo chip + **/ + +#ifndef _HAILO_DEVICE_MEASUREMENTS_HPP_ +#define _HAILO_DEVICE_MEASUREMENTS_HPP_ + +#include "hailo/hailort.h" +#include "hailo/device.hpp" + +#include "common/runtime_statistics_internal.hpp" + +#include +#include +#include + + +class BaseMeasurement +{ +public: + BaseMeasurement(hailort::Device &device, hailo_status &status); + virtual ~BaseMeasurement(); + + virtual hailo_status start_measurement() = 0; + void stop_measurement(); + hailort::AccumulatorResults get_data(); + + virtual std::string measurement_unit() = 0; + +protected: + hailort::Device &m_device; + std::thread m_thread; + std::atomic_bool m_is_thread_running; + std::mutex m_mutex; + hailort::AccumulatorPtr m_acc; +}; + + +class TemperatureMeasurement : public BaseMeasurement +{ +public: + static hailort::Expected> create_shared(hailort::Device &device); + + virtual ~TemperatureMeasurement() = default; + + virtual hailo_status start_measurement() override; + + virtual std::string measurement_unit() override + { + return "C"; + } + + TemperatureMeasurement(hailort::Device &device, hailo_status &status); +}; + + +class PowerMeasurement : public BaseMeasurement +{ +public: + static hailort::Expected> create_shared(hailort::Device &device, + hailo_power_measurement_types_t measurement_type); + virtual ~PowerMeasurement() = default; + + virtual hailo_status start_measurement() override; + + virtual std::string measurement_unit() override + { + switch (m_measurement_type) { + case HAILO_POWER_MEASUREMENT_TYPES__SHUNT_VOLTAGE: + case HAILO_POWER_MEASUREMENT_TYPES__BUS_VOLTAGE: + return "mV"; + case HAILO_POWER_MEASUREMENT_TYPES__AUTO: + case HAILO_POWER_MEASUREMENT_TYPES__POWER: + return "W"; + case HAILO_POWER_MEASUREMENT_TYPES__CURRENT: + return "mA"; + default: + return "Nan"; + }; + } + + PowerMeasurement(hailort::Device &device, hailo_power_measurement_types_t measurement_type, + hailo_status &status); + +private: + hailo_power_measurement_types_t m_measurement_type; +}; + +#endif /* _HAILO_DEVICE_MEASUREMENTS_HPP_ */ diff --git a/hailort/common/ethernet_utils.hpp b/hailort/common/ethernet_utils.hpp index 05dfde7..108b8a7 100644 --- a/hailort/common/ethernet_utils.hpp +++ b/hailort/common/ethernet_utils.hpp @@ -12,7 +12,7 @@ #ifndef __OS_ETHERNET_UTILS_H__ #define __OS_ETHERNET_UTILS_H__ - +#include #include #include "hailo/expected.hpp" diff --git a/hailort/common/latency_meter.hpp b/hailort/common/latency_meter.hpp index 07ca56a..c4a141f 100644 --- a/hailort/common/latency_meter.hpp +++ b/hailort/common/latency_meter.hpp @@ -73,7 +73,7 @@ public: if (m_latency_count == 0) { return make_unexpected(HAILO_NOT_AVAILABLE); } - + duration latency = (m_latency_sum / m_latency_count); if (clear) { m_latency_sum = duration(); @@ -93,7 +93,6 @@ private: return; } - duration start = m_start_timestamps.front(); duration end(0); for (auto &end_timesatmps : m_end_timestamps_per_channel) { if (end_timesatmps.second.empty()) { @@ -104,6 +103,9 @@ private: end = std::max(end, end_timesatmps.second.front()); } + duration start = m_start_timestamps.front(); + assert(start <= end); + // calculate the latency m_latency_sum += (end - start); m_latency_count++; diff --git a/hailort/common/os/posix/os_utils.cpp b/hailort/common/os/posix/os_utils.cpp new file mode 100644 index 0000000..36f1819 --- /dev/null +++ b/hailort/common/os/posix/os_utils.cpp @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file os_utils.cpp + * @brief Utilities for Posix methods + **/ + +#include "hailo/hailort.h" + +#include "common/os_utils.hpp" + +#include "spdlog/sinks/syslog_sink.h" + + +namespace hailort +{ + +HailoRTOSLogger::HailoRTOSLogger() +{ + m_hailort_os_logger = spdlog::syslog_logger_mt("syslog", "hailort_service", LOG_PID); + m_hailort_os_logger->set_pattern("%v"); + m_hailort_os_logger->set_level(spdlog::level::debug); +} + +uint32_t OsUtils::get_curr_pid() +{ + return getpid(); +} + +CursorAdjustment::CursorAdjustment(){} +CursorAdjustment::~CursorAdjustment(){} + +} /* namespace hailort */ diff --git a/hailort/common/os/windows/os_utils.cpp b/hailort/common/os/windows/os_utils.cpp new file mode 100644 index 0000000..3146d31 --- /dev/null +++ b/hailort/common/os/windows/os_utils.cpp @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file os_utils.cpp + * @brief Utilities for Windows methods + **/ + +#include "common/os_utils.hpp" +#include "hailo/hailort.h" + +#include +#include "spdlog/sinks/win_eventlog_sink.h" + +namespace hailort +{ + +HailoRTOSLogger::HailoRTOSLogger() +{ + auto event_log_sink = std::make_shared("hailort_service"); + m_hailort_os_logger = std::make_shared("eventlog", event_log_sink); + event_log_sink->set_pattern("%v"); + m_hailort_os_logger->set_level(spdlog::level::debug); +} + +uint32_t OsUtils::get_curr_pid() +{ + return static_cast(GetCurrentProcessId()); +} + +CursorAdjustment::CursorAdjustment() +{ + // Enables Vitual Terminal Processing - enables ANSI Escape Sequences on Windows + // Source: https://stackoverflow.com/questions/52607960/how-can-i-enable-virtual-terminal-processing + HANDLE h_out = GetStdHandle(STD_OUTPUT_HANDLE); + DWORD dword_mode = 0; + GetConsoleMode(h_out, &dword_mode); + m_previous_output_buffer_mode = dword_mode; + dword_mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + SetConsoleMode(h_out, dword_mode); +} + +CursorAdjustment::~CursorAdjustment() +{ + // Return to the original state + HANDLE h_out = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleMode(h_out, m_previous_output_buffer_mode); // Return the output buffer mode to it's original mode +} +} /* namespace hailort */ diff --git a/hailort/common/os_utils.hpp b/hailort/common/os_utils.hpp new file mode 100644 index 0000000..30d0c2e --- /dev/null +++ b/hailort/common/os_utils.hpp @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file os_utils.hpp + * @brief Utilities for OS methods + **/ + +#ifndef _HAILO_OS_UTILS_HPP_ +#define _HAILO_OS_UTILS_HPP_ + +#include "hailo/hailort.h" + +#include "common/logger_macros.hpp" + + +namespace hailort +{ + +class HailoRTOSLogger final +{ +public: + static HailoRTOSLogger& get_instance() + { + static HailoRTOSLogger instance; + return instance; + } + + std::shared_ptr logger() + { + return m_hailort_os_logger; + } + +private: + HailoRTOSLogger(); + std::shared_ptr m_hailort_os_logger; +}; + +class CursorAdjustment final +{ +public: + CursorAdjustment(); + ~CursorAdjustment(); +private: +#if defined(_WIN32) + unsigned int m_previous_output_buffer_mode; +#endif /* _WIN32 */ +}; + +#define _HAILORT_OS_LOG(level, ...) SPDLOG_LOGGER_CALL(hailort::HailoRTOSLogger::get_instance().logger(), level, __VA_ARGS__) +#define HAILORT_OS_LOG_INFO(...) _HAILORT_OS_LOG(spdlog::level::info, __VA_ARGS__) +#define HAILORT_OS_LOG_WARNNING(...) _HAILORT_OS_LOG(spdlog::level::warn, __VA_ARGS__) +#define HAILORT_OS_LOG_ERROR(...) _HAILORT_OS_LOG(spdlog::level::err, __VA_ARGS__) + +class OsUtils final +{ +public: + OsUtils() = delete; + static uint32_t get_curr_pid(); + + static void set_current_thread_name(const std::string &name) + { + (void)name; +#ifndef NDEBUG +#ifndef _WIN32 + // pthread_setname_np name size is limited to 16 chars (including null terminator) + assert(name.size() < 16); + pthread_setname_np(pthread_self(), name.c_str()); +#else +// TODO: implement for windows +#endif /* _WIN32 */ +#endif /* NDEBUG */ + } + +}; + +} /* namespace hailort */ + +#endif /* _HAILO_OS_UTILS_HPP_ */ diff --git a/hailort/common/runtime_statistics_internal.hpp b/hailort/common/runtime_statistics_internal.hpp index cee5785..707a707 100644 --- a/hailort/common/runtime_statistics_internal.hpp +++ b/hailort/common/runtime_statistics_internal.hpp @@ -39,17 +39,17 @@ public: FullAccumulator &operator=(const FullAccumulator &) = delete; virtual ~FullAccumulator() = default; - virtual void add_data_point(T data) override + virtual void add_data_point(T data, uint32_t samples_count = 1) override { std::lock_guard lock_guard(m_lock); m_min = std::min(m_min, static_cast(data)); m_max = std::max(m_max, static_cast(data)); - m_count++; + m_count += samples_count; // mean, variance, sd and mean_sd are calculated using Welford's_online_algorithm. // See: https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm const auto delta = static_cast(data) - m_mean; - m_mean += delta / static_cast(m_count); + m_mean += ((delta * samples_count )/ static_cast(m_count)); m_M2 += delta * (static_cast(data) - m_mean); } @@ -182,7 +182,7 @@ public: // data is a duration of time. // However, the statistics collected will be in frames per seconds (i.e. time^-1). - virtual void add_data_point(T data) override + virtual void add_data_point(T data, uint32_t samples_count = 1) override { assert(0 != data); @@ -192,13 +192,13 @@ public: // Note: 'this' is needed to access protected members of a template base class this->m_min = std::min(this->m_min, data_inverse); this->m_max = std::max(this->m_max, data_inverse); - this->m_count++; + this->m_count += samples_count; // mean, variance, sd and mean_sd are calculated using Welford's_online_algorithm. // See: https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm const auto delta = data_inverse - this->m_mean; // We calculate the arithmatic mean - this->m_mean = static_cast(this->m_count) / static_cast(m_sum); + this->m_mean = static_cast(this->m_count * samples_count) / static_cast(m_sum); this->m_M2 += delta * (data_inverse - this->m_mean); } diff --git a/hailort/common/utils.hpp b/hailort/common/utils.hpp index 5bb6f30..3dd3bc6 100644 --- a/hailort/common/utils.hpp +++ b/hailort/common/utils.hpp @@ -203,16 +203,15 @@ _ISEMPTY( \ #define CHECK_SUCCESS_AS_EXPECTED(status, ...) _CHECK_SUCCESS_AS_EXPECTED(status, ISEMPTY(__VA_ARGS__), "" __VA_ARGS__) #ifdef HAILO_SUPPORT_MULTI_PROCESS -#define _CHECK_SUCCESS_AS_RPC_STATUS(status, reply, is_default, fmt, ...) \ - do { \ - const auto &__check_success_status = (status); \ - if (__check_success_status != HAILO_SUCCESS) { \ - reply->set_status(static_cast(__check_success_status)); \ - LOGGER__ERROR( \ - _CONSTRUCT_MSG(is_default, "CHECK_SUCCESS_AS_RPC_STATUS failed with status={}", fmt, __check_success_status, ##__VA_ARGS__) \ - ); \ - return grpc::Status::OK; \ - } \ +#define _CHECK_SUCCESS_AS_RPC_STATUS(status, reply, is_default, fmt, ...) \ + do { \ + const auto &__check_success_status = (status); \ + reply->set_status(static_cast(__check_success_status)); \ + _CHECK( \ + HAILO_SUCCESS == __check_success_status, \ + grpc::Status::OK, \ + _CONSTRUCT_MSG(is_default, "CHECK_SUCCESS_AS_RPC_STATUS failed with status={}", fmt, __check_success_status, ##__VA_ARGS__) \ + ); \ } while(0) #define CHECK_SUCCESS_AS_RPC_STATUS(status, reply, ...) _CHECK_SUCCESS_AS_RPC_STATUS(status, reply, ISEMPTY(__VA_ARGS__), "" __VA_ARGS__) diff --git a/hailort/drivers/common/hailo_ioctl_common.h b/hailort/drivers/common/hailo_ioctl_common.h index 594d88d..f11eae7 100644 --- a/hailort/drivers/common/hailo_ioctl_common.h +++ b/hailort/drivers/common/hailo_ioctl_common.h @@ -7,8 +7,6 @@ #define _HAILO_IOCTL_COMMON_H_ -#define DESCRIPTORS_IN_BUFFER(buffer_size, desc_page_size) (((buffer_size) + (desc_page_size) - 1) / (desc_page_size)) - // This value is not easily changeable. // For example: the channel interrupts ioctls assume we have up to 32 channels #define MAX_VDMA_CHANNELS_PER_ENGINE (32) @@ -19,7 +17,6 @@ #define CHANNEL_IRQ_TIMESTAMPS_SIZE (128 * 2) // Should be same as MAX_IRQ_TIMESTAMPS_SIZE (hailort_driver.hpp) #define CHANNEL_IRQ_TIMESTAMPS_SIZE_MASK (CHANNEL_IRQ_TIMESTAMPS_SIZE - 1) -#define INVALID_CHANNEL_HANDLE_VALUE ((uint64_t)-1) #define INVALID_DRIVER_HANDLE_VALUE ((uintptr_t)-1) // Used by windows and unix driver to raise the right CPU control handle to the FW. The same as in pcie_service FW @@ -153,54 +150,43 @@ struct hailo_desc_list_bind_vdma_buffer_params { uintptr_t desc_handle; // in uint16_t desc_page_size; // in uint8_t channel_index; // in - size_t offset; // in + uint32_t starting_desc; // in }; -/* structure used in ioctl HAILO_VDMA_CHANNEL_ENABLE */ -struct hailo_vdma_channel_enable_params { - uint8_t engine_index; // in - uint8_t channel_index; // in - enum hailo_dma_data_direction direction; // in - bool enable_timestamps_measure; // in - uint64_t channel_handle; // out +/* structure used in ioctl HAILO_VDMA_INTERRUPTS_ENABLE */ +struct hailo_vdma_interrupts_enable_params { + uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in + bool enable_timestamps_measure; // in }; -/* structure used in ioctl HAILO_VDMA_CHANNEL_DISABLE */ -struct hailo_vdma_channel_disable_params { - uint8_t engine_index; // in - uint8_t channel_index; // in - uint64_t channel_handle; // in +/* structure used in ioctl HAILO_VDMA_INTERRUPTS_DISABLE */ +struct hailo_vdma_interrupts_disable_params { + uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in }; -/* structure used in ioctl HAILO_VDMA_CHANNEL_WAIT_INT */ -struct hailo_vdma_channel_wait_params { - uint8_t engine_index; // in - uint8_t channel_index; // in - uint64_t channel_handle; // in - uint64_t timeout_ms; // in - uint32_t timestamps_count; // inout -// In linux send address to local buffer because there isnt room on stack for array -#if defined(__linux__) - struct hailo_channel_interrupt_timestamp *timestamps; // out -#elif defined(__QNX__) || defined(_MSC_VER) - struct hailo_channel_interrupt_timestamp timestamps[CHANNEL_IRQ_TIMESTAMPS_SIZE]; // out -#else -#error "unsupported platform!" -#endif // __linux__ +/* structure used in ioctl HAILO_VDMA_INTERRUPTS_WAIT */ +struct hailo_vdma_interrupts_channel_data { + uint8_t engine_index; + uint8_t channel_index; + bool is_active; // If not activate, num_processed is ignored. + uint16_t host_num_processed; + uint8_t host_error; // Channel errors bits on source side + uint8_t device_error; // Channel errors bits on dest side }; -/* structure used in ioctl HAILO_VDMA_CHANNEL_ABORT */ -struct hailo_vdma_channel_abort_params { - uint8_t engine_index; // in - uint8_t channel_index; // in - uint64_t channel_handle; // in +struct hailo_vdma_interrupts_wait_params { + uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in + uint8_t channels_count; // out + struct hailo_vdma_interrupts_channel_data + irq_data[MAX_VDMA_CHANNELS_PER_ENGINE * MAX_VDMA_ENGINES]; // out }; -/* structure used in ioctl HAILO_VDMA_CHANNEL_CLEAR_ABORT */ -struct hailo_vdma_channel_clear_abort_params { - uint8_t engine_index; // in - uint8_t channel_index; // in - uint64_t channel_handle; // in +/* structure used in ioctl HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS */ +struct hailo_vdma_interrupts_read_timestamp_params { + uint8_t engine_index; // in + uint8_t channel_index; // in + uint32_t timestamps_count; // out + struct hailo_channel_interrupt_timestamp timestamps[CHANNEL_IRQ_TIMESTAMPS_SIZE]; // out }; /* structure used in ioctl HAILO_FW_CONTROL */ @@ -299,10 +285,10 @@ enum hailo_vdma_buffer_sync_type { }; struct hailo_vdma_buffer_sync_params { - size_t handle; // in - enum hailo_vdma_buffer_sync_type sync_type; // in - void* buffer_address; // in - uint64_t buffer_size; // in + size_t handle; // in + enum hailo_vdma_buffer_sync_type sync_type; // in + size_t offset; // in + size_t count; // in }; /* structure used in ioctl HAILO_READ_NOTIFICATION */ @@ -315,7 +301,7 @@ struct hailo_d2h_notification { enum hailo_board_type { HAILO_BOARD_TYPE_HAILO8 = 0, - HAILO_BOARD_TYPE_MERCURY, + HAILO_BOARD_TYPE_HAILO15, HAILO_BOARD_TYPE_COUNT, /** Max enum value to maintain ABI Integrity */ @@ -399,11 +385,10 @@ enum hailo_general_ioctl_code { #define HAILO_RESET_NN_CORE _IO_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_RESET_NN_CORE_CODE) enum hailo_vdma_ioctl_code { - HAILO_VDMA_CHANNEL_ENABLE_CODE, - HAILO_VDMA_CHANNEL_DISABLE_CODE, - HAILO_VDMA_CHANNEL_WAIT_INT_CODE, - HAILO_VDMA_CHANNEL_ABORT_CODE, - HAILO_VDMA_CHANNEL_CLEAR_ABORT_CODE, + HAILO_VDMA_INTERRUPTS_ENABLE_CODE, + HAILO_VDMA_INTERRUPTS_DISABLE_CODE, + HAILO_VDMA_INTERRUPTS_WAIT_CODE, + HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE, HAILO_VDMA_CHANNEL_READ_REGISTER_CODE, HAILO_VDMA_CHANNEL_WRITE_REGISTER_CODE, HAILO_VDMA_BUFFER_MAP_CODE, @@ -422,29 +407,29 @@ enum hailo_vdma_ioctl_code { HAILO_VDMA_IOCTL_MAX_NR, }; -#define HAILO_VDMA_CHANNEL_ENABLE _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_ENABLE_CODE, struct hailo_vdma_channel_enable_params) -#define HAILO_VDMA_CHANNEL_DISABLE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_DISABLE_CODE, struct hailo_vdma_channel_disable_params) -#define HAILO_VDMA_CHANNEL_WAIT_INT _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_WAIT_INT_CODE, struct hailo_vdma_channel_wait_params) -#define HAILO_VDMA_CHANNEL_ABORT _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_ABORT_CODE, struct hailo_vdma_channel_abort_params) -#define HAILO_VDMA_CHANNEL_CLEAR_ABORT _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_CLEAR_ABORT_CODE, struct hailo_vdma_channel_clear_abort_params) -#define HAILO_VDMA_CHANNEL_READ_REGISTER _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_READ_REGISTER_CODE, struct hailo_vdma_channel_read_register_params) -#define HAILO_VDMA_CHANNEL_WRITE_REGISTER _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_WRITE_REGISTER_CODE, struct hailo_vdma_channel_write_register_params) +#define HAILO_VDMA_INTERRUPTS_ENABLE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_ENABLE_CODE, struct hailo_vdma_interrupts_enable_params) +#define HAILO_VDMA_INTERRUPTS_DISABLE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_DISABLE_CODE, struct hailo_vdma_interrupts_disable_params) +#define HAILO_VDMA_INTERRUPTS_WAIT _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_WAIT_CODE, struct hailo_vdma_interrupts_wait_params) +#define HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE, struct hailo_vdma_interrupts_read_timestamp_params) + +#define HAILO_VDMA_CHANNEL_READ_REGISTER _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_READ_REGISTER_CODE, struct hailo_vdma_channel_read_register_params) +#define HAILO_VDMA_CHANNEL_WRITE_REGISTER _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_WRITE_REGISTER_CODE, struct hailo_vdma_channel_write_register_params) -#define HAILO_VDMA_BUFFER_MAP _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_MAP_CODE, struct hailo_vdma_buffer_map_params) -#define HAILO_VDMA_BUFFER_UNMAP _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_UNMAP_CODE, struct hailo_vdma_buffer_unmap_params) -#define HAILO_VDMA_BUFFER_SYNC _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_SYNC_CODE, struct hailo_vdma_buffer_sync_params) +#define HAILO_VDMA_BUFFER_MAP _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_MAP_CODE, struct hailo_vdma_buffer_map_params) +#define HAILO_VDMA_BUFFER_UNMAP _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_UNMAP_CODE, struct hailo_vdma_buffer_unmap_params) +#define HAILO_VDMA_BUFFER_SYNC _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_SYNC_CODE, struct hailo_vdma_buffer_sync_params) -#define HAILO_DESC_LIST_CREATE _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_CREATE_CODE, struct hailo_desc_list_create_params) -#define HAILO_DESC_LIST_RELEASE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_RELEASE_CODE, uintptr_t) -#define HAILO_DESC_LIST_BIND_VDMA_BUFFER _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_BIND_VDMA_BUFFER_CODE, struct hailo_desc_list_bind_vdma_buffer_params) +#define HAILO_DESC_LIST_CREATE _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_CREATE_CODE, struct hailo_desc_list_create_params) +#define HAILO_DESC_LIST_RELEASE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_RELEASE_CODE, uintptr_t) +#define HAILO_DESC_LIST_BIND_VDMA_BUFFER _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_BIND_VDMA_BUFFER_CODE, struct hailo_desc_list_bind_vdma_buffer_params) -#define HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE, struct hailo_allocate_low_memory_buffer_params) -#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE _IO_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE) +#define HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE, struct hailo_allocate_low_memory_buffer_params) +#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE _IO_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE) -#define HAILO_MARK_AS_IN_USE _IOW_(HAILO_VDMA_IOCTL_MAGIC, HAILO_MARK_AS_IN_USE_CODE, struct hailo_mark_as_in_use_params) +#define HAILO_MARK_AS_IN_USE _IOW_(HAILO_VDMA_IOCTL_MAGIC, HAILO_MARK_AS_IN_USE_CODE, struct hailo_mark_as_in_use_params) -#define HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE, struct hailo_allocate_continuous_buffer_params) -#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE _IO_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE) +#define HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE, struct hailo_allocate_continuous_buffer_params) +#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE _IO_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE) enum hailo_non_linux_ioctl_code { diff --git a/hailort/drivers/win/include/Public.h b/hailort/drivers/win/include/Public.h index 7c0c9d9..8dfecf6 100644 --- a/hailort/drivers/win/include/Public.h +++ b/hailort/drivers/win/include/Public.h @@ -56,7 +56,6 @@ struct tCommonHailoIoctlParam #define HAILO_CMD_MAP_BUFFER 0x0051 #define HAILO_CMD_FREE_MEMORY 0x0060 #define HAILO_CMD_ALLOC_MEMORY 0x0061 -#define HAILO_CMD_ABORT_ALL 0x0070 #define HAILO_IOCTL_COMPATIBLE CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_ANY_ACCESS) struct tCompatibleHailoIoctlParam @@ -103,11 +102,10 @@ struct tCompatibleHailoIoctlData ULONG_PTR Value; union { hailo_memory_transfer_params MemoryTransfer; - hailo_vdma_channel_enable_params ChannelEnable; - hailo_vdma_channel_disable_params ChannelDisable; - hailo_vdma_channel_wait_params ChannelWait; - hailo_vdma_channel_abort_params ChannelAbort; - hailo_vdma_channel_clear_abort_params ChannelClearAbort; + hailo_vdma_interrupts_enable_params VdmaInterruptsEnable; + hailo_vdma_interrupts_disable_params VdmaInterruptsDisable; + hailo_vdma_interrupts_read_timestamp_params VdmaInterruptsReadTimestamps; + hailo_vdma_interrupts_wait_params VdmaInterruptsWait; hailo_vdma_buffer_sync_params VdmaBufferSync; hailo_fw_control FirmwareControl; hailo_vdma_buffer_map_params VdmaBufferMap; diff --git a/hailort/hailort_service/CMakeLists.txt b/hailort/hailort_service/CMakeLists.txt index 98f1278..2b9e3a5 100644 --- a/hailort/hailort_service/CMakeLists.txt +++ b/hailort/hailort_service/CMakeLists.txt @@ -1,9 +1,17 @@ cmake_minimum_required(VERSION 3.0.0) +if(WIN32) + set(HAILORT_SERVICE_OS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/windows") +elseif(UNIX) + set(HAILORT_SERVICE_OS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/unix") +else() + message(FATAL_ERROR "Unexpeced platform target, stopping build") +endif() + add_executable(hailort_service hailort_rpc_service.cpp - hailort_service.cpp service_resource_manager.hpp + ${HAILORT_SERVICE_OS_DIR}/hailort_service.cpp ${HAILORT_COMMON_CPP_SOURCES} ) target_compile_options(hailort_service PRIVATE ${HAILORT_COMPILE_OPTIONS}) @@ -12,9 +20,16 @@ target_link_libraries(hailort_service libhailort spdlog::spdlog grpc++_unsecure - hailort_rpc_grpc_proto) + hailort_rpc_grpc_proto +) +if(WIN32) + # Needed in order to compile eth utils (we compile here ${HAILORT_COMMON_CPP_SOURCES}, consider removing) + target_link_libraries(hailort_service Iphlpapi Shlwapi Kernel32 Advapi32) +endif() + target_include_directories(hailort_service PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} ${HAILORT_INC_DIR} ${HAILORT_COMMON_DIR} ${COMMON_INC_DIR} @@ -27,24 +42,26 @@ if(NOT CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(SYSTEMD_UNIT_DIR ${CMAKE_INSTALL_PREFIX}/${SYSTEMD_UNIT_DIR}) endif() -# Install service's environment variables file -set(ENV_VARS_FILE_DIR ${CMAKE_INSTALL_SYSCONFDIR}/default/) -set(HAILORT_SERVICE_ENV_VARS_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hailort_service) -install( - FILES "${HAILORT_SERVICE_ENV_VARS_FILE}" - DESTINATION "${ENV_VARS_FILE_DIR}" - CONFIGURATIONS Release - COMPONENT hailort_service -) +if (UNIX) + # Install service's environment variables file + set(ENV_VARS_FILE_DIR ${CMAKE_INSTALL_SYSCONFDIR}/default/) + set(HAILORT_SERVICE_ENV_VARS_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hailort_service) + install( + FILES "${HAILORT_SERVICE_ENV_VARS_FILE}" + DESTINATION "${ENV_VARS_FILE_DIR}" + CONFIGURATIONS Release + COMPONENT hailort_service + ) -# Install systemd unit file -set(HAILORT_SERVICE_UNIT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hailort.service) -install( - FILES "${HAILORT_SERVICE_UNIT_FILE}" - DESTINATION "${SYSTEMD_UNIT_DIR}" - CONFIGURATIONS Release - COMPONENT hailort_service -) + # Install systemd unit file + set(HAILORT_SERVICE_UNIT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hailort.service) + install( + FILES "${HAILORT_SERVICE_UNIT_FILE}" + DESTINATION "${SYSTEMD_UNIT_DIR}" + CONFIGURATIONS Release + COMPONENT hailort_service + ) +endif() install( TARGETS hailort_service @@ -63,4 +80,4 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) # Create empty directory for default PID file install(DIRECTORY DESTINATION ${DAEMON_PID_DIR}) -endif() \ No newline at end of file +endif() diff --git a/hailort/hailort_service/hailort_rpc_service.cpp b/hailort/hailort_service/hailort_rpc_service.cpp index a7222cb..09a0472 100644 --- a/hailort/hailort_service/hailort_rpc_service.cpp +++ b/hailort/hailort_service/hailort_rpc_service.cpp @@ -7,32 +7,63 @@ * @brief Implementation of the hailort rpc service **/ -#include "hailort_rpc_service.hpp" -#include "rpc/rpc_definitions.hpp" -#include "service_resource_manager.hpp" -#include "common/utils.hpp" #include "hailo/network_group.hpp" #include "hailo/vdevice.hpp" #include "hailo/vstream.hpp" #include "hailo/hailort_common.hpp" -#include + +#include "common/utils.hpp" +#include "common/os_utils.hpp" + +#include "hailort_rpc_service.hpp" +#include "rpc/rpc_definitions.hpp" +#include "service_resource_manager.hpp" + +#include namespace hailort { -grpc::Status HailoRtRpcService::client_keep_alive(grpc::ServerContext *ctx, const keepalive_Request *request, +HailoRtRpcService::HailoRtRpcService() + : ProtoHailoRtRpc::Service() +{ + m_keep_alive = make_unique_nothrow([this] () { + this->keep_alive(); + }); +} + +void HailoRtRpcService::keep_alive() +{ + while (true) { + std::this_thread::sleep_for(hailort::HAILO_KEEPALIVE_INTERVAL / 2); + auto now = std::chrono::high_resolution_clock::now(); + std::unique_lock lock(m_mutex); + std::set pids_to_remove; + for (auto pid_to_last_alive : m_clients_pids) { + auto duration = std::chrono::duration_cast(now - pid_to_last_alive.second); + if (duration > hailort::HAILO_KEEPALIVE_INTERVAL) { + auto client_id = pid_to_last_alive.first; + pids_to_remove.insert(client_id); + LOGGER__INFO("Client disconnected, pid: {}", client_id); + HAILORT_OS_LOG_INFO("Client disconnected, pid: {}", client_id); + ServiceResourceManager::get_instance().release_by_pid(client_id); + ServiceResourceManager::get_instance().release_by_pid(client_id); + ServiceResourceManager::get_instance().release_by_pid(client_id); + ServiceResourceManager::get_instance().release_by_pid(client_id); + } + } + for (auto &pid : pids_to_remove) { + m_clients_pids.erase(pid); + } + } +} + +grpc::Status HailoRtRpcService::client_keep_alive(grpc::ServerContext*, const keepalive_Request *request, empty*) { - auto client_id = request->process_id(); - while (!ctx->IsCancelled()) { - sleep(hailort::HAILO_KEEPALIVE_INTERVAL_SEC); - } - LOGGER__INFO("Client disconnected, pid: {}", client_id); - syslog(LOG_NOTICE, "Client disconnected, pid: %i", client_id); - ServiceResourceManager::get_instance().release_by_pid(client_id); - ServiceResourceManager::get_instance().release_by_pid(client_id); - ServiceResourceManager::get_instance().release_by_pid(client_id); - ServiceResourceManager::get_instance().release_by_pid(client_id); + auto client_id = request->pid(); + std::unique_lock lock(m_mutex); + m_clients_pids[client_id] = std::chrono::high_resolution_clock::now(); return grpc::Status::OK; } @@ -50,6 +81,15 @@ grpc::Status HailoRtRpcService::get_service_version(grpc::ServerContext*, const return grpc::Status::OK; } +grpc::Status HailoRtRpcService::VDevice_dup_handle(grpc::ServerContext*, const dup_handle_Request *request, + dup_handle_Reply* reply) +{ + auto &manager = ServiceResourceManager::get_instance(); + auto handle = manager.dup_handle(request->pid(), request->handle()); + reply->set_handle(handle); + return grpc::Status::OK; +} + grpc::Status HailoRtRpcService::VDevice_create(grpc::ServerContext *, const VDevice_create_Request *request, VDevice_create_Reply *reply) { @@ -64,11 +104,11 @@ grpc::Status HailoRtRpcService::VDevice_create(grpc::ServerContext *, const VDev } hailo_vdevice_params_t params = { - .device_count = params_proto.device_count(), - .device_ids = device_ids.data(), - .scheduling_algorithm = static_cast(params_proto.scheduling_algorithm()), - .group_id = params_proto.group_id().c_str(), - .multi_process_service = false + params_proto.device_count(), + device_ids.data(), + static_cast(params_proto.scheduling_algorithm()), + params_proto.group_id().c_str(), + false }; auto vdevice = VDevice::create(params); @@ -100,7 +140,7 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD NetworkGroupsParamsMap configure_params_map; for (auto &name_configure_params_pair : request->configure_params_map()) { - ConfigureNetworkParams network_configure_params; + ConfigureNetworkParams network_configure_params{}; auto proto_configure_params = name_configure_params_pair.params(); network_configure_params.batch_size = static_cast(proto_configure_params.batch_size()); network_configure_params.power_mode = static_cast(proto_configure_params.power_mode()); @@ -110,23 +150,14 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD for (auto &proto_name_streams_params_pair : proto_configure_params.stream_params_map()) { auto proto_streams_params = proto_name_streams_params_pair.params(); auto stream_direction = static_cast(proto_streams_params.direction()); - hailo_stream_parameters_t stream_params; + hailo_stream_parameters_t stream_params{}; + stream_params.stream_interface = static_cast(proto_streams_params.stream_interface()); + stream_params.direction = stream_direction; + stream_params.flags = static_cast(proto_streams_params.flags()); if (stream_direction == HAILO_H2D_STREAM) { - stream_params = { - .stream_interface = static_cast(proto_streams_params.stream_interface()), - .direction = stream_direction, - {.pcie_input_params = { - .reserved = 0 - }} - }; + stream_params.pcie_input_params = {0}; } else { - stream_params = { - .stream_interface = static_cast(proto_streams_params.stream_interface()), - .direction = stream_direction, - {.pcie_output_params = { - .reserved = 0 - }} - }; + stream_params.pcie_output_params = {0}; } network_configure_params.stream_params_by_name.insert({proto_name_streams_params_pair.name(), stream_params}); } @@ -135,7 +166,7 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD for (auto &proto_name_network_params_pair : proto_configure_params.network_params_map()) { auto proto_network_params = proto_name_network_params_pair.params(); hailo_network_parameters_t net_params { - .batch_size = static_cast(proto_network_params.batch_size()) + static_cast(proto_network_params.batch_size()) }; network_configure_params.network_params_by_name.insert({proto_name_network_params_pair.name(), net_params}); @@ -192,6 +223,15 @@ grpc::Status HailoRtRpcService::VDevice_get_default_streams_interface(grpc::Serv return grpc::Status::OK; } +grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_dup_handle(grpc::ServerContext*, const dup_handle_Request *request, + dup_handle_Reply* reply) +{ + auto &manager = ServiceResourceManager::get_instance(); + auto handle = manager.dup_handle(request->pid(), request->handle()); + reply->set_handle(handle); + return grpc::Status::OK; +} + grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_release(grpc::ServerContext*, const Release_Request *request, Release_Reply *reply) { @@ -410,6 +450,20 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_all_vstream_infos(grp return grpc::Status::OK; } +grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_is_scheduled(grpc::ServerContext*, + const ConfiguredNetworkGroup_is_scheduled_Request *request, + ConfiguredNetworkGroup_is_scheduled_Reply *reply) +{ + auto lambda = [](std::shared_ptr cng) { + return cng->is_scheduled(); + }; + auto &manager = ServiceResourceManager::get_instance(); + auto is_scheduled = manager.execute(request->handle(), lambda); + reply->set_is_scheduled(static_cast(is_scheduled)); + reply->set_status(static_cast(HAILO_SUCCESS)); + return grpc::Status::OK; +} + grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_set_scheduler_timeout(grpc::ServerContext*, const ConfiguredNetworkGroup_set_scheduler_timeout_Request *request, ConfiguredNetworkGroup_set_scheduler_timeout_Reply *reply) @@ -437,6 +491,20 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_set_scheduler_threshold(g return grpc::Status::OK; } +grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_set_scheduler_priority(grpc::ServerContext*, + const ConfiguredNetworkGroup_set_scheduler_priority_Request *request, + ConfiguredNetworkGroup_set_scheduler_priority_Reply *reply) +{ + auto lambda = [](std::shared_ptr cng, uint8_t priority, std::string network_name) { + return cng->set_scheduler_priority(priority, network_name); + }; + auto &net_group_manager = ServiceResourceManager::get_instance(); + auto status = net_group_manager.execute(request->handle(), lambda, static_cast(request->priority()), + request->network_name()); + reply->set_status(status); + return grpc::Status::OK; +} + grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_config_params(grpc::ServerContext*, const ConfiguredNetworkGroup_get_config_params_Request *request, ConfiguredNetworkGroup_get_config_params_Reply *reply) @@ -485,11 +553,11 @@ grpc::Status HailoRtRpcService::InputVStreams_create(grpc::ServerContext *, cons format.order = hailo_format_order_t(user_buffer_format_proto.order()); format.type = hailo_format_type_t(user_buffer_format_proto.type()); hailo_vstream_params_t params = { - .user_buffer_format = format, - .timeout_ms = vstream_params_proto.timeout_ms(), - .queue_size = vstream_params_proto.queue_size(), - .vstream_stats_flags = hailo_vstream_stats_flags_t(vstream_params_proto.vstream_stats_flags()), - .pipeline_elements_stats_flags = hailo_pipeline_elem_stats_flags_t(vstream_params_proto.pipeline_elements_stats_flags()) + format, + vstream_params_proto.timeout_ms(), + vstream_params_proto.queue_size(), + hailo_vstream_stats_flags_t(vstream_params_proto.vstream_stats_flags()), + hailo_pipeline_elem_stats_flags_t(vstream_params_proto.pipeline_elements_stats_flags()) }; inputs_params.emplace(param_proto.name(), std::move(params)); } @@ -533,11 +601,11 @@ grpc::Status HailoRtRpcService::OutputVStreams_create(grpc::ServerContext *, con format.order = hailo_format_order_t(user_buffer_format_proto.order()); format.type = hailo_format_type_t(user_buffer_format_proto.type()); hailo_vstream_params_t params = { - .user_buffer_format = format, - .timeout_ms = vstream_params_proto.timeout_ms(), - .queue_size = vstream_params_proto.queue_size(), - .vstream_stats_flags = hailo_vstream_stats_flags_t(vstream_params_proto.vstream_stats_flags()), - .pipeline_elements_stats_flags = hailo_pipeline_elem_stats_flags_t(vstream_params_proto.pipeline_elements_stats_flags()) + format, + vstream_params_proto.timeout_ms(), + vstream_params_proto.queue_size(), + hailo_vstream_stats_flags_t(vstream_params_proto.vstream_stats_flags()), + hailo_pipeline_elem_stats_flags_t(vstream_params_proto.pipeline_elements_stats_flags()) }; output_params.emplace(param_proto.name(), std::move(params)); } @@ -594,11 +662,35 @@ grpc::Status HailoRtRpcService::InputVStream_write(grpc::ServerContext*, const I }; auto &manager = ServiceResourceManager::get_instance(); auto status = manager.execute(request->handle(), lambda, MemoryView::create_const(data.data(), data.size())); + + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("User aborted VStream write."); + reply->set_status(static_cast(HAILO_STREAM_ABORTED_BY_USER)); + return grpc::Status::OK; + } CHECK_SUCCESS_AS_RPC_STATUS(status, reply, "VStream write failed"); reply->set_status(static_cast(HAILO_SUCCESS)); return grpc::Status::OK; } +grpc::Status HailoRtRpcService::InputVStream_dup_handle(grpc::ServerContext*, const dup_handle_Request *request, + dup_handle_Reply *reply) +{ + auto &manager = ServiceResourceManager::get_instance(); + auto handle = manager.dup_handle(request->pid(), request->handle()); + reply->set_handle(handle); + return grpc::Status::OK; +} + +grpc::Status HailoRtRpcService::OutputVStream_dup_handle(grpc::ServerContext*, const dup_handle_Request *request, + dup_handle_Reply *reply) +{ + auto &manager = ServiceResourceManager::get_instance(); + auto handle = manager.dup_handle(request->pid(), request->handle()); + reply->set_handle(handle); + return grpc::Status::OK; +} + grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_network_infos(grpc::ServerContext*, const ConfiguredNetworkGroup_get_network_infos_Request *request, ConfiguredNetworkGroup_get_network_infos_Reply *reply) @@ -626,6 +718,11 @@ grpc::Status HailoRtRpcService::OutputVStream_read(grpc::ServerContext*, const O }; auto &manager = ServiceResourceManager::get_instance(); auto status = manager.execute(request->handle(), lambda, MemoryView(data.data(), data.size())); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("User aborted VStream read."); + reply->set_status(static_cast(HAILO_STREAM_ABORTED_BY_USER)); + return grpc::Status::OK; + } CHECK_SUCCESS_AS_RPC_STATUS(status, reply, "VStream read failed"); reply->set_data(data.data(), data.size()); reply->set_status(static_cast(HAILO_SUCCESS)); diff --git a/hailort/hailort_service/hailort_rpc_service.hpp b/hailort/hailort_service/hailort_rpc_service.hpp index 3540b9f..a77d701 100644 --- a/hailort/hailort_service/hailort_rpc_service.hpp +++ b/hailort/hailort_service/hailort_rpc_service.hpp @@ -25,17 +25,21 @@ #pragma GCC diagnostic pop #endif +#include + namespace hailort { class HailoRtRpcService final : public ProtoHailoRtRpc::Service { - public: + HailoRtRpcService(); + virtual grpc::Status client_keep_alive(grpc::ServerContext *ctx, const keepalive_Request *request, empty*) override; - virtual grpc::Status get_service_version(grpc::ServerContext *, const get_service_version_Request *request, get_service_version_Reply *reply) override; + virtual grpc::Status VDevice_dup_handle(grpc::ServerContext *ctx, const dup_handle_Request *request, + dup_handle_Reply*) override; virtual grpc::Status VDevice_create(grpc::ServerContext *, const VDevice_create_Request *request, VDevice_create_Reply *reply) override; @@ -90,6 +94,13 @@ public: VStream_get_info_Reply *reply) override; virtual grpc::Status OutputVStream_get_info(grpc::ServerContext*, const VStream_get_info_Request *request, VStream_get_info_Reply *reply) override; + virtual grpc::Status InputVStream_dup_handle(grpc::ServerContext *ctx, const dup_handle_Request *request, + dup_handle_Reply*) override; + virtual grpc::Status OutputVStream_dup_handle(grpc::ServerContext *ctx, const dup_handle_Request *request, + dup_handle_Reply*) override; + + virtual grpc::Status ConfiguredNetworkGroup_dup_handle(grpc::ServerContext *ctx, const dup_handle_Request *request, + dup_handle_Reply*) override; virtual grpc::Status ConfiguredNetworkGroup_release(grpc::ServerContext*, const Release_Request* request, Release_Reply* reply) override; virtual grpc::Status ConfiguredNetworkGroup_make_input_vstream_params(grpc::ServerContext*, @@ -122,12 +133,18 @@ public: virtual grpc::Status ConfiguredNetworkGroup_get_all_vstream_infos(grpc::ServerContext*, const ConfiguredNetworkGroup_get_vstream_infos_Request *request, ConfiguredNetworkGroup_get_vstream_infos_Reply *reply) override; + virtual grpc::Status ConfiguredNetworkGroup_is_scheduled(grpc::ServerContext*, + const ConfiguredNetworkGroup_is_scheduled_Request *request, + ConfiguredNetworkGroup_is_scheduled_Reply *reply) override; virtual grpc::Status ConfiguredNetworkGroup_set_scheduler_timeout(grpc::ServerContext*, const ConfiguredNetworkGroup_set_scheduler_timeout_Request *request, ConfiguredNetworkGroup_set_scheduler_timeout_Reply *reply) override; virtual grpc::Status ConfiguredNetworkGroup_set_scheduler_threshold(grpc::ServerContext*, const ConfiguredNetworkGroup_set_scheduler_threshold_Request *request, ConfiguredNetworkGroup_set_scheduler_threshold_Reply *reply) override; + virtual grpc::Status ConfiguredNetworkGroup_set_scheduler_priority(grpc::ServerContext*, + const ConfiguredNetworkGroup_set_scheduler_priority_Request *request, + ConfiguredNetworkGroup_set_scheduler_priority_Reply *reply) override; virtual grpc::Status ConfiguredNetworkGroup_get_output_vstream_infos(grpc::ServerContext*, const ConfiguredNetworkGroup_get_vstream_infos_Request *request, ConfiguredNetworkGroup_get_vstream_infos_Reply *reply) override; @@ -140,6 +157,13 @@ public: virtual grpc::Status ConfiguredNetworkGroup_get_config_params(grpc::ServerContext*, const ConfiguredNetworkGroup_get_config_params_Request *request, ConfiguredNetworkGroup_get_config_params_Reply *reply) override; + +private: + void keep_alive(); + + std::mutex m_mutex; + std::map> m_clients_pids; + std::unique_ptr m_keep_alive; }; } diff --git a/hailort/hailort_service/hailort_service b/hailort/hailort_service/hailort_service index 8fd5b2b..0259a28 100644 --- a/hailort/hailort_service/hailort_service +++ b/hailort/hailort_service/hailort_service @@ -1,4 +1,4 @@ -# This file contains HailoRT's configurable environment variables for HailoRT Service. +# This file contains HailoRT's configurable environment variables for HailoRT Linux Service. # The environment variables are set to their default values. # To change an environment variable's value, follow the steps: # 1. Change the value of the selected environemt variable in this file @@ -8,5 +8,4 @@ [Service] HAILORT_LOGGER_PATH="/var/log/hailo" HAILO_DISABLE_MULTIPLEXER=0 -HAILO_ENABLE_MULTI_DEVICE_SCHEDULER=0 -SCHEDULER_MONITOR=0 +HAILO_MONITOR=0 diff --git a/hailort/hailort_service/service_resource_manager.hpp b/hailort/hailort_service/service_resource_manager.hpp index d051ac0..488999c 100644 --- a/hailort/hailort_service/service_resource_manager.hpp +++ b/hailort/hailort_service/service_resource_manager.hpp @@ -25,10 +25,9 @@ struct Resource { Resource(uint32_t pid, std::shared_ptr resource) : pid(pid), resource(std::move(resource)) {} - std::shared_timed_mutex resource_mutex; + uint32_t pid; std::shared_ptr resource; - }; template @@ -42,37 +41,51 @@ public: } template - K execute(uint32_t key, Func &lambda, Args... args) + K execute(uint32_t handle, Func &lambda, Args... args) { std::unique_lock lock(m_mutex); - auto resource_expected = resource_lookup(key); + auto resource_expected = resource_lookup(handle); assert(resource_expected); - auto resource = resource_expected.release(); - std::shared_lock resource_lock(resource->resource_mutex); + + assert(contains(m_resources_mutexes, handle)); + std::shared_lock resource_lock(m_resources_mutexes[handle]); lock.unlock(); K ret = lambda(resource->resource, args...); return ret; } - uint32_t register_resource(uint32_t pid, std::shared_ptr const &resource) + uint32_t register_resource(uint32_t pid, const std::shared_ptr &resource) { std::unique_lock lock(m_mutex); - + auto index = m_current_handle_index.load(); // Create a new resource and register - auto index = m_current_handle_index; - m_resources.emplace(m_current_handle_index++, std::make_shared>(pid, std::move(resource))); + m_resources.emplace(m_current_handle_index, std::make_shared>(pid, std::move(resource))); + m_resources_mutexes[m_current_handle_index]; // construct std::shared_timed_mutex + m_current_handle_index++; return index; } - hailo_status release_resource(uint32_t key) + uint32_t dup_handle(uint32_t pid, uint32_t handle) + { + // Keeping this function for future possible usage + (void)pid; + return handle; + } + + hailo_status release_resource(uint32_t handle) { std::unique_lock lock(m_mutex); - auto found = m_resources.find(key); - CHECK(found != m_resources.end(), HAILO_NOT_FOUND, "Failed to release resource with key {}, resource does not exist", key); - std::unique_lock resource_lock(found->second->resource_mutex); - m_resources.erase(key); + auto found = m_resources.find(handle); + CHECK(found != m_resources.end(), HAILO_NOT_FOUND, "Failed to release resource with handle {}, resource does not exist", handle); + assert(contains(m_resources_mutexes, handle)); + auto resource = m_resources[handle]; + { + std::unique_lock resource_lock(m_resources_mutexes[handle]); + m_resources.erase(handle); + } + m_resources_mutexes.erase(handle); return HAILO_SUCCESS; } @@ -80,9 +93,14 @@ public: { std::unique_lock lock(m_mutex); for (auto iter = m_resources.begin(); iter != m_resources.end(); ) { + auto handle = iter->first; if (iter->second->pid == pid) { - std::unique_lock resource_lock(iter->second->resource_mutex); - iter = m_resources.erase(iter); + assert(contains(m_resources_mutexes, handle)); + { + std::unique_lock resource_lock(m_resources_mutexes[handle]); + iter = m_resources.erase(iter); + } + m_resources_mutexes.erase(handle); } else { ++iter; } @@ -94,18 +112,18 @@ private: : m_current_handle_index(0) {} - Expected>> resource_lookup(uint32_t key) + Expected>> resource_lookup(uint32_t handle) { - auto found = m_resources.find(key); - CHECK_AS_EXPECTED(found != m_resources.end(), HAILO_NOT_FOUND, "Failed to find resource with key {}", key); - + auto found = m_resources.find(handle); + CHECK_AS_EXPECTED(found != m_resources.end(), HAILO_NOT_FOUND, "Failed to find resource with handle {}", handle); auto resource = found->second; return resource; } std::mutex m_mutex; - uint32_t m_current_handle_index; + std::atomic m_current_handle_index; std::unordered_map>> m_resources; + std::unordered_map m_resources_mutexes; }; } diff --git a/hailort/hailort_service/hailort_service.cpp b/hailort/hailort_service/unix/hailort_service.cpp similarity index 78% rename from hailort/hailort_service/hailort_service.cpp rename to hailort/hailort_service/unix/hailort_service.cpp index 890e8bd..87c0d0c 100644 --- a/hailort/hailort_service/hailort_service.cpp +++ b/hailort/hailort_service/unix/hailort_service.cpp @@ -4,9 +4,7 @@ * * @file hailort_service.cpp * @brief main for hailort service - * To run as without daemonize the executable: - * 1) Compile with `./build.sh` - * 2) Run `./bin/linux.x86_64.debug/hailort_service standalone` + * To run without daemonization run the hailort_service executable with `standalone`. * * To run as daemon service please follow the steps: * 1) Install the HailoRT: @@ -27,12 +25,13 @@ #include "common/utils.hpp" #include "common/filesystem.hpp" #include "hailo/hailort_common.hpp" +#include "common/os_utils.hpp" #include #include void RunService() { - std::string server_address(hailort::HAILO_DEFAULT_UDS_ADDR); + const std::string server_address = hailort::HAILORT_SERVICE_DEFAULT_ADDR; hailort::HailoRtRpcService service; grpc::ServerBuilder builder; @@ -48,20 +47,20 @@ void write_pid_to_lock_file() { auto status = hailort::Filesystem::create_directory(HAILO_DAEMON_PID_DIR); if (status != HAILO_SUCCESS) { - syslog(LOG_ERR, "Cannot create directory at path, status=%i", status); + HAILORT_OS_LOG_ERROR("Cannot create directory at path, status={}", status); return; } auto locked_file = hailort::LockedFile::create(HAILO_DAEMON_PID_FILE, "wx"); if (HAILO_SUCCESS != locked_file.status()) { - syslog(LOG_ERR, "Failed to lock pid file for hailort service, status=%i", locked_file.status()); + HAILORT_OS_LOG_ERROR("Failed to lock pid file for hailort service, status={}", locked_file.status()); return; } std::string pid = std::to_string(getpid()); auto ret = write(locked_file->get_fd(), pid.c_str(), pid.size()); if (-1 == ret) { - syslog(LOG_ERR, "Failed to write pid to lock file for hailort service, errno=%i", errno); + HAILORT_OS_LOG_ERROR("Failed to write pid to lock file for hailort service, errno={}", errno); return; } } @@ -72,7 +71,7 @@ int main(int argc, char *argv[]) if (!is_standalone) { int ret = daemon(0, 0); if (ret < 0) { - syslog(LOG_ERR, "Failed to create daemon with errno %i", errno); + HAILORT_OS_LOG_ERROR("Failed to create daemon with errno {}", errno); exit(EXIT_FAILURE); } diff --git a/hailort/hailort_service/windows/hailort_service.cpp b/hailort/hailort_service/windows/hailort_service.cpp new file mode 100644 index 0000000..1a1c223 --- /dev/null +++ b/hailort/hailort_service/windows/hailort_service.cpp @@ -0,0 +1,246 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + * + * @file hailort_service.cpp + * @brief main for hailort service + * The service code is based on Microsoft's documenataion: https://learn.microsoft.com/en-us/windows/win32/services/the-complete-service-sample + * + * Running hailort_service: + * To run hailort_service without Windows service control manager (SCM), run hailort_service executable with `standalone`. + * + * To run as a service application please follow the steps: + * 1) Compile and install libhailort: + * `cmake -H. -Bbuild -A=x64 -DCMAKE_BUILD_TYPE=Release -DHAILO_BUILD_SERVICE=1 && cmake --build build --config release --target install` + * + * 2) To install the service, run the `hailort_service` executable with `install`: + * `"C:\Program Files\HailoRT\bin\hailort_service.exe" install` + + * 3) Start the service: + * `sc start hailort_service` + * + * 4) Stop the service: + * `sc stop hailort_service` + * + * 5) Delete service: + * `sc delete hailort_service` +*/ + +#include "hailort_rpc_service.hpp" +#include "rpc/rpc_definitions.hpp" +#include "common/os_utils.hpp" + +#include +#include +#include +#include + +#define SERVICE_NAME ("hailort_service") +static const DWORD HRT_SERVICE_INIT_WAIT_TIME_MS(3000); +static const DWORD HRT_SERVICE_ZERO_WAIT_TIME_MS(0); + +SERVICE_STATUS g_service_status = {0}; +SERVICE_STATUS_HANDLE g_service_status_handle = nullptr; +HANDLE g_stop_event_handle = INVALID_HANDLE_VALUE; +std::unique_ptr g_hailort_rpc_server = nullptr; + +void RunService() +{ + const std::string server_address = hailort::HAILORT_SERVICE_DEFAULT_ADDR; + hailort::HailoRtRpcService service; + + grpc::ServerBuilder builder; + builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + builder.SetMaxReceiveMessageSize(-1); + builder.RegisterService(&service); + g_hailort_rpc_server = builder.BuildAndStart(); + g_hailort_rpc_server->Wait(); +} + +// Installs the service in the SCM database +void install_service() +{ + SC_HANDLE open_sc_manager_handle = nullptr; + SC_HANDLE create_service_handle = nullptr; + TCHAR module_path[MAX_PATH]; + + if (!GetModuleFileName(nullptr, module_path, MAX_PATH)) { + HAILORT_OS_LOG_ERROR("GetModuleFileName() failed. Cannot install hailort service, LE = {}", GetLastError()); + return; + } + + TCHAR quoted_module_path[MAX_PATH]; + StringCbPrintf(quoted_module_path, MAX_PATH, ("\"%s\""), module_path); + + // Get a handle to the SCM database. + open_sc_manager_handle = OpenSCManager( + nullptr, // local computer + nullptr, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (nullptr == open_sc_manager_handle) { + HAILORT_OS_LOG_ERROR("OpenSCManager() failed. Cannot install hailort service, LE = {}", GetLastError()); + return; + } + + // Create the service + create_service_handle = CreateService( + open_sc_manager_handle, // SCM database + SERVICE_NAME, // name of service + SERVICE_NAME, // service name to display + SERVICE_ALL_ACCESS, // desired access + SERVICE_WIN32_OWN_PROCESS, // service type + SERVICE_DEMAND_START, // start type + SERVICE_ERROR_NORMAL, // error control type + quoted_module_path, // path to service's binary + nullptr, // no load ordering group + nullptr, // no tag identifier + nullptr, // no dependencies + nullptr, // LocalSystem account + nullptr); // no password + + if (nullptr == create_service_handle) { + HAILORT_OS_LOG_ERROR("CreateService() failed. Cannot install hailort service, LE = {}", GetLastError()); + CloseServiceHandle(open_sc_manager_handle); + return; + } + + CloseServiceHandle(create_service_handle); + CloseServiceHandle(open_sc_manager_handle); +} + +// Sets the current service status and reports it to the SCM +void report_service_status(DWORD current_state, DWORD win32_exit_code, DWORD wait_hint) +{ + static DWORD check_point = 1; + g_service_status.dwCurrentState = current_state; + g_service_status.dwWin32ExitCode = win32_exit_code; + g_service_status.dwWaitHint = wait_hint; + + if (SERVICE_START_PENDING == current_state) { + // Service is about to start + g_service_status.dwControlsAccepted = 0; + } else { + g_service_status.dwControlsAccepted = SERVICE_ACCEPT_STOP; + } + + if ((SERVICE_RUNNING == current_state) || (SERVICE_STOPPED == current_state)) { + g_service_status.dwCheckPoint = 0; + } else { + g_service_status.dwCheckPoint = check_point++; + } + + // Report the service status to the SCM. + SetServiceStatus(g_service_status_handle, &g_service_status); +} + +// Called by SCM whenever a control code is sent to the service +void control_handler(DWORD control_code) +{ + switch(control_code) { + case SERVICE_CONTROL_STOP: + report_service_status(SERVICE_STOP_PENDING, NO_ERROR, HRT_SERVICE_ZERO_WAIT_TIME_MS); + + // Signal the service to stop. + SetEvent(g_stop_event_handle); + report_service_status(SERVICE_STOPPED, NO_ERROR, HRT_SERVICE_ZERO_WAIT_TIME_MS); + return; + + default: + break; + } +} + +void terminate_server_thread(HANDLE thread_handle) +{ + g_hailort_rpc_server->Shutdown(); + auto rpc_server_wait_res = WaitForSingleObject(thread_handle, INFINITE); + if (WAIT_OBJECT_0 == rpc_server_wait_res) { + CloseHandle(thread_handle); + } else { + HAILORT_OS_LOG_ERROR("Failed waiting on hailort server thread, LE = {}", GetLastError()); + report_service_status(SERVICE_STOPPED, GetLastError(), HRT_SERVICE_ZERO_WAIT_TIME_MS); + } +} + +// The service code +void service_init() +{ + // Create an event. The control handler function signals this event when it receives the stop control code. + g_stop_event_handle = CreateEvent( + nullptr, // default security attributes + TRUE, // manual reset event + FALSE, // not signaled + nullptr); // no name + if (nullptr == g_stop_event_handle) { + report_service_status(SERVICE_STOPPED, GetLastError(), HRT_SERVICE_ZERO_WAIT_TIME_MS); + return; + } + + // Report SCM the running status when initialization is complete. + report_service_status(SERVICE_RUNNING, NO_ERROR, HRT_SERVICE_ZERO_WAIT_TIME_MS); + + // Start a thread that will perform the main task of the service + HANDLE service_thread_handle = CreateThread(nullptr, 0, (LPTHREAD_START_ROUTINE)RunService, nullptr, 0, nullptr); + if (nullptr == service_thread_handle) { + HAILORT_OS_LOG_ERROR("Failed to create hailort_service thread, LE = {}", GetLastError()); + } + + // Wait for stop service signal + auto service_wait_res = WaitForSingleObject(g_stop_event_handle, INFINITE); + if (WAIT_OBJECT_0 == service_wait_res) { + terminate_server_thread(service_thread_handle); + report_service_status(SERVICE_STOPPED, NO_ERROR, HRT_SERVICE_ZERO_WAIT_TIME_MS); + } else { + HAILORT_OS_LOG_ERROR("Failed waiting for signal on hailort_service stop event, LE = {}", GetLastError()); + report_service_status(SERVICE_STOPPED, GetLastError(), HRT_SERVICE_ZERO_WAIT_TIME_MS); + } + CloseHandle(g_stop_event_handle); +} + +// Entry point for the service +void service_main(DWORD /* dwArgc */, LPTSTR * /*lpszArgv*/) +{ + // Register the handler function for the service + g_service_status_handle = RegisterServiceCtrlHandler(SERVICE_NAME, control_handler); + if (!g_service_status_handle) { + HAILORT_OS_LOG_ERROR("RegisterServiceCtrlHandler() failed. Cannot start hailort service, LE = {}", GetLastError()); + return; + } + + g_service_status.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + g_service_status.dwServiceSpecificExitCode = 0; + + // Report initial status to the SCM - service is starting + report_service_status(SERVICE_START_PENDING, NO_ERROR, HRT_SERVICE_INIT_WAIT_TIME_MS); + service_init(); +} + +int main(int argc, TCHAR *argv[]) +{ + const bool is_standalone = ((1 < argc) && (0 == lstrcmpi(argv[1], "standalone"))); + if (is_standalone) { + RunService(); + return 0; + } + + // If command-line parameter is "install", install the service. + // Otherwise, the service is probably being started by the SCM. + if ((0 < argc) && (0 == lstrcmpi(argv[1], "install"))) { + install_service(); + return 0; + } + + // Service is being started by the SCM + SERVICE_TABLE_ENTRY dispatch_table[] = { + {SERVICE_NAME, static_cast(service_main)}, + {nullptr, nullptr} + }; + + // This call returns when the service has stopped (SERVICE_STOPPED). + // The process should simply terminate when the call returns. + if (!StartServiceCtrlDispatcher(dispatch_table)) { + HAILORT_OS_LOG_ERROR("StartServiceCtrlDispatcher() failed. Cannot start hailort service, LE = {}", GetLastError()); + } + return 0; +} \ No newline at end of file diff --git a/hailort/hailort_service/windows/hailort_service_env_vars.bat b/hailort/hailort_service/windows/hailort_service_env_vars.bat new file mode 100644 index 0000000..a615ab4 --- /dev/null +++ b/hailort/hailort_service/windows/hailort_service_env_vars.bat @@ -0,0 +1,11 @@ +@REM This file contains HailoRT's configurable environment variables for HailoRT Windows Service. +@REM The environment variables are set to their default values, and are seperated by the character \0. +@REM To change an environment variable's value, follow the steps: +@REM 1. Change the value of the selected environment variable in this file +@REM 2. Run this script +@REM 3. Restart the service +@REM Running this script requires Administrator permissions. + +reg ADD HKLM\SYSTEM\CurrentControlSet\Services\hailort_service /f /v Environment /t REG_MULTI_SZ /d ^ +HAILORT_LOGGER_PATH="%PROGRAMDATA%\HailoRT_Service\logs"\0^ +HAILO_DISABLE_MULTIPLEXER=0\0 \ No newline at end of file diff --git a/hailort/hailortcli/CMakeLists.txt b/hailort/hailortcli/CMakeLists.txt index 45993e0..cd7ff1d 100644 --- a/hailort/hailortcli/CMakeLists.txt +++ b/hailort/hailortcli/CMakeLists.txt @@ -20,7 +20,6 @@ set(HAILORTCLI_CPP_FILES fw_config_serializer.cpp common.cpp benchmark_command.cpp - temp_measurement.cpp parse_hef_command.cpp graph_printer.cpp mon_command.cpp @@ -30,6 +29,7 @@ set(HAILORTCLI_CPP_FILES run2/live_printer.cpp run2/timer_live_track.cpp run2/network_live_track.cpp + run2/measurement_live_track.cpp ) if(UNIX) @@ -53,9 +53,10 @@ add_executable(hailortcli ${HAILORT_COMMON_CPP_SOURCES} ${PROJECT_SOURCE_DIR}/common/src/firmware_header_utils.c ${PROJECT_SOURCE_DIR}/common/src/md5.c - ${HAILORT_SRC_DIR}/pipeline.cpp - ${HAILO_FULL_OS_DIR}/event.cpp + ${HAILORT_SRC_DIR}/net_flow/pipeline/pipeline.cpp # TODO: link dynamically with libhailort + ${HAILO_FULL_OS_DIR}/event.cpp # TODO: link dynamically with libhailort ) + target_compile_options(hailortcli PRIVATE ${HAILORT_COMPILE_OPTIONS}) set_property(TARGET hailortcli PROPERTY CXX_STANDARD 14) set_property(TARGET hailortcli PROPERTY INSTALL_RPATH "$ORIGIN" "../lib/") # Link with a relative libhailort diff --git a/hailort/hailortcli/common.cpp b/hailort/hailortcli/common.cpp index 16fb5ac..d69fff1 100644 --- a/hailort/hailortcli/common.cpp +++ b/hailort/hailortcli/common.cpp @@ -69,15 +69,14 @@ Expected CliCommon::current_time_to_string() void CliCommon::reset_cursor(size_t lines_count) { for (size_t i = 0; i < lines_count; i++) { - std::cout << FORMAT_CURSOR_UP_LINE; + std::cout << FORMAT_CURSOR_UP_LINE; // Override prev line + std::cout << FORMAT_CLEAR_LINE; // Delete line } } -void CliCommon::clear_lines_down(size_t lines_count) +void CliCommon::clear_terminal() { - for (size_t i = 0; i < lines_count; i++) { - std::cout << FORMAT_CURSOR_DOWN_CLEAR_LINE; - } + std::cout << FORMAT_CLEAR_TERMINAL_CURSOR_FIRST_LINE; } bool CliCommon::is_positive_number(const std::string &s) @@ -91,3 +90,14 @@ bool CliCommon::is_non_negative_number(const std::string &s) bool is_number = (!s.empty()) && (std::all_of(s.begin(), s.end(), ::isdigit)); return is_number && (0 <= std::stoi(s)); } + +AlternativeTerminal::AlternativeTerminal() +{ + std::cout << FORMAT_ENTER_ALTERNATIVE_SCREEN; + CliCommon::clear_terminal(); +} + +AlternativeTerminal::~AlternativeTerminal() +{ + std::cout << FORMAT_EXIT_ALTERNATIVE_SCREEN; +} \ No newline at end of file diff --git a/hailort/hailortcli/common.hpp b/hailort/hailortcli/common.hpp index 4a3c6bd..c549c68 100644 --- a/hailort/hailortcli/common.hpp +++ b/hailort/hailortcli/common.hpp @@ -22,7 +22,11 @@ using namespace hailort; // http://www.climagic.org/mirrors/VT100_Escape_Codes.html #define FORMAT_CLEAR_LINE "\033[2K\r" #define FORMAT_CURSOR_UP_LINE "\033[F" -#define FORMAT_CURSOR_DOWN_CLEAR_LINE "\033[B\33[2K\r" +#define FORMAT_CLEAR_TERMINAL_CURSOR_FIRST_LINE "\033[2J\033[1;1H" +#define FORMAT_ENTER_ALTERNATIVE_SCREEN "\033[?1049h" +#define FORMAT_EXIT_ALTERNATIVE_SCREEN "\033[?1049l" +#define FORMAT_GREEN_PRINT "\x1B[1;32m" +#define FORMAT_NORMAL_PRINT "\x1B[0m" class CliCommon final { @@ -32,9 +36,9 @@ public: static std::string duration_to_string(std::chrono::seconds secs); static Expected current_time_to_string(); static void reset_cursor(size_t number_of_lines); - static void clear_lines_down(size_t number_of_lines); static bool is_positive_number(const std::string &s); static bool is_non_negative_number(const std::string &s); + static void clear_terminal(); }; // Validators @@ -54,6 +58,14 @@ struct FileSuffixValidator : public CLI::Validator { } }; +// This class is an RAII for running in alternative terminal +class AlternativeTerminal final +{ +public: + AlternativeTerminal(); + ~AlternativeTerminal(); +}; + // Based on NLOHMANN_JSON_SERIALIZE_ENUM (json/include/nlohmann/json.hpp) // Accepts a static array instead of building one in the function #define NLOHMANN_JSON_SERIALIZE_ENUM2(ENUM_TYPE, _pair_arr)\ diff --git a/hailort/hailortcli/download_action_list_command.cpp b/hailort/hailortcli/download_action_list_command.cpp index a51c853..be53034 100644 --- a/hailort/hailortcli/download_action_list_command.cpp +++ b/hailort/hailortcli/download_action_list_command.cpp @@ -20,7 +20,7 @@ // TODO - HRT-7364 - add CPU subsystem frequency into the device extended info control // and use it for get the timer's frequency #define NN_CORE_TO_TIMER_FREQ_FACTOR (2) -#define MERCURY_VPU_CORE_CPU_DEFAULT_FREQ_MHZ (200) +#define HAILO15_VPU_CORE_CPU_DEFAULT_FREQ_MHZ (200) constexpr int DownloadActionListCommand::INVALID_NUMERIC_VALUE; @@ -44,9 +44,9 @@ hailo_status DownloadActionListCommand::execute(Device &device, const std::strin auto chip_arch = device.get_architecture(); CHECK_EXPECTED_AS_STATUS(chip_arch); unsigned int clock_cycle = 0; - // TODO - HRT-8046 Implement extended device info for mercury - if (HAILO_ARCH_MERCURY_VPU == chip_arch.value()) { - clock_cycle = MERCURY_VPU_CORE_CPU_DEFAULT_FREQ_MHZ; + // TODO - HRT-8046 Implement extended device info for hailo15 + if (HAILO_ARCH_HAILO15 == chip_arch.value()) { + clock_cycle = HAILO15_VPU_CORE_CPU_DEFAULT_FREQ_MHZ; } else { auto extended_info = device.get_extended_device_information(); CHECK_EXPECTED_AS_STATUS(extended_info); diff --git a/hailort/hailortcli/fw_control_command.cpp b/hailort/hailortcli/fw_control_command.cpp index 23e3681..e4b595f 100644 --- a/hailort/hailortcli/fw_control_command.cpp +++ b/hailort/hailortcli/fw_control_command.cpp @@ -138,10 +138,8 @@ static std::string identity_arch_string(const hailo_device_identity_t &identity) return "HAILO8"; case HAILO_ARCH_HAILO8L: return "HAILO8L"; - case HAILO_ARCH_MERCURY_CA: - return "MERCURY_CA"; - case HAILO_ARCH_MERCURY_VPU: - return "MERCURY_VPU"; + case HAILO_ARCH_HAILO15: + return "HAILO15"; default: return "Unknown"; } diff --git a/hailort/hailortcli/fw_logger_command.cpp b/hailort/hailortcli/fw_logger_command.cpp index efd59e9..e6e3614 100644 --- a/hailort/hailortcli/fw_logger_command.cpp +++ b/hailort/hailortcli/fw_logger_command.cpp @@ -67,7 +67,7 @@ hailo_status FwLoggerCommand::execute_on_device(Device &device) return HAILO_INVALID_OPERATION; } - if (Device::Type::CORE != device.get_type()) { + if (Device::Type::INTEGRATED != device.get_type()) { status = write_logs_to_file(device, ofs, HAILO_CPU_ID_0); if (status != HAILO_SUCCESS){ return status; diff --git a/hailort/hailortcli/graph_printer.hpp b/hailort/hailortcli/graph_printer.hpp index 4443642..f355a53 100644 --- a/hailort/hailortcli/graph_printer.hpp +++ b/hailort/hailortcli/graph_printer.hpp @@ -12,7 +12,8 @@ #include "hailo/hailort.h" #include "hailo/vstream.hpp" -#include "pipeline.hpp" + +#include "net_flow/pipeline/pipeline.hpp" #include "DotWriter.h" @@ -20,6 +21,7 @@ #include #include + namespace hailort { diff --git a/hailort/hailortcli/hailortcli.hpp b/hailort/hailortcli/hailortcli.hpp index 29f5c25..4ed2ab6 100644 --- a/hailort/hailortcli/hailortcli.hpp +++ b/hailort/hailortcli/hailortcli.hpp @@ -27,6 +27,9 @@ using namespace hailort; } \ } while (0) +// Used for run and run2 commands +constexpr size_t OVERALL_LATENCY_TIMESTAMPS_LIST_LENGTH (512); + struct hailo_device_params { std::vector device_ids; }; diff --git a/hailort/hailortcli/infer_stats_printer.cpp b/hailort/hailortcli/infer_stats_printer.cpp index ee9feb2..2e06257 100644 --- a/hailort/hailortcli/infer_stats_printer.cpp +++ b/hailort/hailortcli/infer_stats_printer.cpp @@ -10,7 +10,8 @@ #include "infer_stats_printer.hpp" #include "run_command.hpp" #include "common.hpp" -#include "pipeline.hpp" + +#include "net_flow/pipeline/pipeline.hpp" #include #include @@ -220,11 +221,17 @@ void InferStatsPrinter::print_csv(const std::vector &network_groups for (const auto &pair : inference_result->m_temp_measurements) { if (nullptr != pair.second) { m_results_csv_file << ","; - m_results_csv_file << pair.second->min_value; + if (auto min = pair.second->min()) { + m_results_csv_file << *min; + } m_results_csv_file << ","; - m_results_csv_file << pair.second->average_value; + if (auto mean = pair.second->mean()) { + m_results_csv_file << *mean; + } m_results_csv_file << ","; - m_results_csv_file << pair.second->max_value; + if (auto max = pair.second->max()) { + m_results_csv_file << *max; + } } else { m_results_csv_file << ",,,"; } @@ -447,9 +454,15 @@ void InferStatsPrinter::print_stdout(Expected &inference_result) } auto temp_measure_iter = inference_result->m_temp_measurements.find(pair.first); if ((temp_measure_iter != inference_result->m_temp_measurements.end()) && (nullptr != temp_measure_iter->second)) { - measurement_stream << " Minimum chip temperature: " << temp_measure_iter->second->min_value << "C" << std::endl; - measurement_stream << " Average chip temperature: " << temp_measure_iter->second->average_value << "C" << std::endl; - measurement_stream << " Maximum chip temperature: " << temp_measure_iter->second->max_value << "C" << std::endl; + if (auto min = temp_measure_iter->second->min()) { + measurement_stream << " Minimum chip temperature: " << *min << "C" << std::endl; + } + if (auto mean = temp_measure_iter->second->mean()) { + measurement_stream << " Average chip temperature: " << *mean << "C" << std::endl; + } + if (auto max = temp_measure_iter->second->max()) { + measurement_stream << " Maximum chip temperature: " << *max << "C" << std::endl; + } } if (0 != measurement_stream.rdbuf()->in_avail()) { std::cout << " Device: " << pair.first << std::endl; diff --git a/hailort/hailortcli/inference_progress.cpp b/hailort/hailortcli/inference_progress.cpp index 9a3092b..ef9b432 100644 --- a/hailort/hailortcli/inference_progress.cpp +++ b/hailort/hailortcli/inference_progress.cpp @@ -9,6 +9,7 @@ #include "inference_progress.hpp" #include "infer_stats_printer.hpp" #include "common.hpp" +#include "common/os_utils.hpp" #include #include @@ -40,6 +41,7 @@ InferProgress::InferProgress(const inference_runner_params ¶ms, void InferProgress::start() { m_print_thread = std::thread([this] () { + OsUtils::set_current_thread_name("PROGRESS_BAR"); while (true) { print_progress(true); auto status = m_stop_event->wait(m_print_interval); diff --git a/hailort/hailortcli/inference_result.hpp b/hailort/hailortcli/inference_result.hpp index c1134d7..0396ca6 100644 --- a/hailort/hailortcli/inference_result.hpp +++ b/hailort/hailortcli/inference_result.hpp @@ -11,7 +11,7 @@ #define _HAILO_INFER_RESULT_ #include "power_measurement_command.hpp" -#include "temp_measurement.hpp" +#include "common/device_measurements.hpp" #include "hailo/runtime_statistics.hpp" #include "hailo/vstream.hpp" @@ -309,7 +309,7 @@ public: for (const auto &device : devices) { m_power_measurements.emplace(device.get().get_dev_id(), std::shared_ptr{}); m_current_measurements.emplace(device.get().get_dev_id(), std::shared_ptr{}); - m_temp_measurements.emplace(device.get().get_dev_id(), std::shared_ptr{}); + m_temp_measurements.emplace(device.get().get_dev_id(), std::shared_ptr{}); } } @@ -329,7 +329,7 @@ public: return HAILO_SUCCESS; } - hailo_status set_temp_measurement(const std::string &device_id, std::shared_ptr &&temp_measure) + hailo_status set_temp_measurement(const std::string &device_id, std::shared_ptr &&temp_measure) { auto iter = m_temp_measurements.find(device_id); CHECK(m_temp_measurements.end() != iter, HAILO_INVALID_ARGUMENT); @@ -341,7 +341,7 @@ public: // TODO: create a struct containing all device measurements, and keep only one map std::map> m_power_measurements; std::map> m_current_measurements; - std::map> m_temp_measurements; + std::map> m_temp_measurements; private: std::vector m_network_group_results; diff --git a/hailort/hailortcli/mon_command.cpp b/hailort/hailortcli/mon_command.cpp index a2f47a4..84dce25 100644 --- a/hailort/hailortcli/mon_command.cpp +++ b/hailort/hailortcli/mon_command.cpp @@ -7,12 +7,16 @@ * @brief Monitor of networks - Presents information about the running networks **/ -#include "mon_command.hpp" -#include "common.hpp" #include "hailo/hailort.h" + #include "common/filesystem.hpp" +#include "mon_command.hpp" +#include "common.hpp" + #include +#include +#include #if defined(__GNUC__) #include #endif @@ -20,15 +24,22 @@ namespace hailort { -// TODO: Deal with longer networks names - should use HAILO_MAX_NETWORK_NAME_SIZE but its too long for one line -constexpr size_t NETWORK_NAME_WIDTH = 40; -constexpr size_t STREAM_NAME_WIDTH = 60; -constexpr size_t ACTIVE_TIME_WIDTH = 25; +constexpr size_t STRING_WIDTH = 60; +constexpr size_t NETWORK_GROUP_NAME_WIDTH = STRING_WIDTH; +constexpr size_t DEVICE_ID_WIDTH = STRING_WIDTH; +constexpr size_t STREAM_NAME_WIDTH = STRING_WIDTH; +constexpr size_t UTILIZATION_WIDTH = 25; constexpr size_t NUMBER_WIDTH = 15; constexpr size_t TERMINAL_DEFAULT_WIDTH = 80; -constexpr size_t LINE_LENGTH = 125; +constexpr size_t LINE_LENGTH = NETWORK_GROUP_NAME_WIDTH + STREAM_NAME_WIDTH + UTILIZATION_WIDTH + NUMBER_WIDTH; constexpr std::chrono::milliseconds EPSILON_TIME(500); +inline std::string truncate_str(const std::string &original_str, uint32_t max_length) +{ + static const std::string ELLIPSIS = "... "; + return (original_str.length() > max_length) ? original_str.substr(0, (max_length - ELLIPSIS.length())) + ELLIPSIS : original_str; +} + MonCommand::MonCommand(CLI::App &parent_app) : Command(parent_app.add_subcommand("monitor", "Monitor of networks - Presents information about the running networks. " \ "To enable monitor, set in the application process the environment variable '" + std::string(SCHEDULER_MON_ENV_VAR) + "' to 1.")) @@ -40,65 +51,86 @@ hailo_status MonCommand::execute() LOGGER__ERROR("hailortcli `monitor` command is not supported on Windows"); return HAILO_NOT_IMPLEMENTED; #else - return print_table(); + return run_monitor(); #endif } -size_t MonCommand::print_networks_info_header() +void MonCommand::print_devices_info_header() { std::cout << - std::setw(NETWORK_NAME_WIDTH) << std::left << "Network" << + std::setw(DEVICE_ID_WIDTH) << std::left << "Device ID" << + std::setw(UTILIZATION_WIDTH) << std::left << "Utilization (%)" << + std::setw(STRING_WIDTH) << std::left << "Architecture" << + "\n" << std::left << std::string(LINE_LENGTH, '-') << "\n"; +} + +void MonCommand::print_devices_info_table(const ProtoMon &mon_message) +{ + auto data_line_len = NUMBER_WIDTH + NETWORK_GROUP_NAME_WIDTH + DEVICE_ID_WIDTH; + auto rest_line_len = LINE_LENGTH - data_line_len; + + for (const auto &device_info : mon_message.device_infos()) { + auto device_id = device_info.device_id(); + auto utilization = device_info.utilization(); + auto device_arch = device_info.device_arch(); + + std::cout << std::setprecision(1) << std::fixed << + std::setw(DEVICE_ID_WIDTH) << std::left << device_id << + std::setw(UTILIZATION_WIDTH) << std::left << utilization << + std::setw(STRING_WIDTH) << std::left << device_arch << + std::string(rest_line_len, ' ') << "\n"; + } +} + +void MonCommand::print_networks_info_header() +{ + std::cout << + std::setw(NETWORK_GROUP_NAME_WIDTH) << std::left << "Model" << + std::setw(UTILIZATION_WIDTH) << std::left << "Utilization (%) " << std::setw(NUMBER_WIDTH) << std::left << "FPS" << - std::setw(ACTIVE_TIME_WIDTH) << std::left << "Active Time (%) " << std::setw(NUMBER_WIDTH) << std::left << "PID" << "\n" << std::left << std::string(LINE_LENGTH, '-') << "\n"; - static const uint32_t header_lines_count = 2; - - return header_lines_count; } -size_t MonCommand::print_networks_info_table(const ProtoMon &mon_message) +void MonCommand::print_networks_info_table(const ProtoMon &mon_message) { const uint32_t NUMBER_OBJECTS_COUNT = 3; - auto data_line_len = (NUMBER_WIDTH * NUMBER_OBJECTS_COUNT) + NETWORK_NAME_WIDTH; + auto data_line_len = (NUMBER_WIDTH * NUMBER_OBJECTS_COUNT) + NETWORK_GROUP_NAME_WIDTH; auto rest_line_len = LINE_LENGTH - data_line_len; const std::string &pid = mon_message.pid(); - for (auto net_info : mon_message.networks_infos()) { - auto &net_name = net_info.network_name(); + for (const auto &net_info : mon_message.networks_infos()) { + auto &original_net_name = net_info.network_name(); + auto net_name = truncate_str(original_net_name, NETWORK_GROUP_NAME_WIDTH); auto fps = net_info.fps(); - auto active_time = net_info.active_time(); + auto utilization = net_info.utilization(); std::cout << std::setprecision(1) << std::fixed << - std::setw(NETWORK_NAME_WIDTH) << std::left << net_name << + std::setw(STRING_WIDTH) << std::left << net_name << + std::setw(UTILIZATION_WIDTH) << std::left << utilization << std::setw(NUMBER_WIDTH) << std::left << fps << - std::setw(ACTIVE_TIME_WIDTH) << std::left << active_time << std::setw(NUMBER_WIDTH) << std::left << pid << std::string(rest_line_len, ' ') << "\n"; } - - return mon_message.networks_infos().size(); } -size_t MonCommand::print_frames_header() +void MonCommand::print_frames_header() { std::cout << - std::setw(NETWORK_NAME_WIDTH) << std::left << "Network" << - std::setw(STREAM_NAME_WIDTH) << std::left << "Stream" << + std::setw(STRING_WIDTH) << std::left << "Model" << + std::setw(STRING_WIDTH) << std::left << "Stream" << std::setw(NUMBER_WIDTH) << std::left << "Direction" << std::setw(NUMBER_WIDTH) << std::left << "Frames" << "\n" << std::left << std::string(LINE_LENGTH, '-') << "\n"; - static const size_t header_lines_count = 2; - return header_lines_count; } -size_t MonCommand::print_frames_table(const ProtoMon &mon_message) +void MonCommand::print_frames_table(const ProtoMon &mon_message) { - size_t table_lines_count = 0; - for (auto &net_info : mon_message.net_frames_infos()) { - auto &net_name = net_info.network_name(); - table_lines_count += net_info.streams_frames_infos().size(); - for (auto &streams_frames : net_info.streams_frames_infos()) { - auto &stream_name = streams_frames.stream_name(); + for (const auto &net_info : mon_message.net_frames_infos()) { + auto &original_net_name = net_info.network_name(); + auto net_name = truncate_str(original_net_name, NETWORK_GROUP_NAME_WIDTH); + for (const auto &streams_frames : net_info.streams_frames_infos()) { + auto &stream_name_original = streams_frames.stream_name(); + auto stream_name = truncate_str(stream_name_original, STREAM_NAME_WIDTH); auto stream_direction = (streams_frames.stream_direction() == PROTO__STREAM_DIRECTION__HOST_TO_DEVICE) ? "H2D" : "D2H"; std::string frames; @@ -109,14 +141,12 @@ size_t MonCommand::print_frames_table(const ProtoMon &mon_message) } std::cout << - std::setw(NETWORK_NAME_WIDTH) << std::left << net_name << - std::setw(STREAM_NAME_WIDTH) << std::left << stream_name << + std::setw(STRING_WIDTH) << std::left << net_name << + std::setw(STRING_WIDTH) << std::left << stream_name << std::setw(NUMBER_WIDTH) << std::left << stream_direction << std::setw(NUMBER_WIDTH) << std::left << frames << "\n"; } } - - return table_lines_count; } #if defined(__GNUC__) @@ -133,17 +163,49 @@ Expected get_terminal_line_width() return terminal_line_width; } -hailo_status MonCommand::print_table() +void MonCommand::print_tables(const std::vector &mon_messages, uint32_t terminal_line_width) { + print_devices_info_header(); + for (const auto &mon_message : mon_messages) { + print_devices_info_table(mon_message); + } + + std::cout << std::string(terminal_line_width, ' ') << "\n"; + std::cout << std::string(terminal_line_width, ' ') << "\n"; + + print_networks_info_header(); + + for (const auto &mon_message : mon_messages) { + print_networks_info_table(mon_message); + } + + std::cout << std::string(terminal_line_width, ' ') << "\n"; + std::cout << std::string(terminal_line_width, ' ') << "\n"; + + print_frames_header(); + for (const auto &mon_message : mon_messages) { + print_frames_table(mon_message); + } +} + +static volatile bool keep_running = true; +void signit_handler(int /*dummy*/) +{ + keep_running = false; +} + +hailo_status MonCommand::run_monitor() +{ + // Note: There is no need to unregister to previous SIGINT handler since we finish running after it is called. + signal(SIGINT, signit_handler); + std::chrono::milliseconds time_interval = DEFAULT_SCHEDULER_MON_INTERVAL + EPSILON_TIME; auto terminal_line_width_expected = get_terminal_line_width(); CHECK_EXPECTED_AS_STATUS(terminal_line_width_expected); auto terminal_line_width = terminal_line_width_expected.release(); - size_t last_run_total_lines_count = 0; - bool data_was_printed = false; - while (true) { - size_t total_lines_count = 0; + AlternativeTerminal alt_terminal; + while (keep_running) { bool print_warning_msg = true; // Will change to false only if mon directory is valid and there are updated files in it. auto mon_dir_valid = Filesystem::is_directory(SCHEDULER_MON_TMP_DIR); @@ -160,14 +222,12 @@ hailo_status MonCommand::print_table() auto file = LockedFile::create(mon_file, "r"); if (HAILO_SUCCESS != file.status()) { LOGGER__ERROR("Failed to open and lock file {}, with status: {}", mon_file, file.status()); - total_lines_count++; continue; } ProtoMon mon_message; if (!mon_message.ParseFromFileDescriptor(file->get_fd())) { LOGGER__WARNING("Failed to ParseFromFileDescriptor monitor file {} with errno {}", mon_file, errno); - total_lines_count++; continue; } @@ -175,41 +235,16 @@ hailo_status MonCommand::print_table() } } - total_lines_count += print_networks_info_header(); - for (auto &mon_message : mon_messages) { - total_lines_count += print_networks_info_table(mon_message); - } - - std::cout << std::string(terminal_line_width, ' ') << "\n"; - std::cout << std::string(terminal_line_width, ' ') << "\n"; - total_lines_count += 2; - - total_lines_count += print_frames_header(); - for (auto &mon_message : mon_messages) { - total_lines_count += print_frames_table(mon_message); - } - + print_tables(mon_messages, terminal_line_width); if (print_warning_msg) { - std::cout << "Monitor did not retrieve any files. This occurs when there is no application currently running. If this is not the case, verify that environment variable '" << - SCHEDULER_MON_ENV_VAR << "' is set to 1.\n"; - total_lines_count++; - - if (data_was_printed) { - auto lines_to_clear = last_run_total_lines_count - total_lines_count; - CliCommon::clear_lines_down(lines_to_clear); - total_lines_count += lines_to_clear; - data_was_printed = false; - } - } - else { - data_was_printed = true; - last_run_total_lines_count = total_lines_count; + std::cout << FORMAT_GREEN_PRINT << "Monitor did not retrieve any files. This occurs when there is no application currently running.\n" + << "If this is not the case, verify that environment variable '" << SCHEDULER_MON_ENV_VAR << "' is set to 1.\n" << FORMAT_NORMAL_PRINT; } - CliCommon::reset_cursor(total_lines_count); + CliCommon::clear_terminal(); std::this_thread::sleep_for(DEFAULT_SCHEDULER_MON_INTERVAL); } - + return HAILO_SUCCESS; } #endif diff --git a/hailort/hailortcli/mon_command.hpp b/hailort/hailortcli/mon_command.hpp index 3bfd700..6fe1774 100644 --- a/hailort/hailortcli/mon_command.hpp +++ b/hailort/hailortcli/mon_command.hpp @@ -11,9 +11,10 @@ #define _HAILO_MON_COMMAND_HPP_ #include "hailo/hailort.h" + #include "hailortcli.hpp" #include "command.hpp" -#include "scheduler_mon.hpp" +#include "vdevice/scheduler/scheduler_mon.hpp" #include "CLI/CLI.hpp" @@ -28,11 +29,15 @@ public: virtual hailo_status execute() override; private: - hailo_status print_table(); - size_t print_networks_info_header(); - size_t print_frames_header(); - size_t print_networks_info_table(const ProtoMon &mon_message); - size_t print_frames_table(const ProtoMon &mon_message); + hailo_status run_monitor(); + void print_tables(const std::vector &mon_messages, uint32_t terminal_line_width); + void print_devices_info_header(); + void print_networks_info_header(); + void print_frames_header(); + void print_devices_info_table(const ProtoMon &mon_message); + void print_networks_info_table(const ProtoMon &mon_message); + void print_frames_table(const ProtoMon &mon_message); + hailo_status run_in_alternative_terminal(); }; } /* namespace hailort */ diff --git a/hailort/hailortcli/parse_hef_command.cpp b/hailort/hailortcli/parse_hef_command.cpp index f15ca9e..200e7c0 100644 --- a/hailort/hailortcli/parse_hef_command.cpp +++ b/hailort/hailortcli/parse_hef_command.cpp @@ -11,60 +11,6 @@ #include "common/filesystem.hpp" #include "hailo/hailort_common.hpp" -#define TAB (" ") - -static std::string add_tabs(uint8_t count) -{ - // Each TAB counts as 4 spaces - std::string res = ""; - for (uint8_t i = 0; i < count; i++) { - res = res + TAB; - } - return res; -} - -static std::string get_shape_str(const hailo_stream_info_t &stream_info) -{ - switch (stream_info.format.order) - { - case HAILO_FORMAT_ORDER_HAILO_NMS: - return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) + - "(number of classes: " + std::to_string(stream_info.nms_info.number_of_classes) + - ", max_bboxes_per_class: "+ std::to_string(stream_info.nms_info.max_bboxes_per_class) + ")"; - case HAILO_FORMAT_ORDER_NC: - return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) + - "(" + std::to_string(stream_info.hw_shape.features) + ")"; - case HAILO_FORMAT_ORDER_NHW: - return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) + - "(" + std::to_string(stream_info.hw_shape.height) + "x" + std::to_string(stream_info.hw_shape.width) + ")"; - default: - return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) + - "(" + std::to_string(stream_info.hw_shape.height) + "x" + std::to_string(stream_info.hw_shape.width) + - "x" + std::to_string(stream_info.hw_shape.features) + ")"; - } -} - -static std::string get_shape_str(const hailo_vstream_info_t &vstream_info) -{ - switch (vstream_info.format.order) - { - case HAILO_FORMAT_ORDER_HAILO_NMS: - return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) + - "(number of classes: " + std::to_string(vstream_info.nms_shape.number_of_classes) + - ", max_bboxes_per_class: " + std::to_string(vstream_info.nms_shape.max_bboxes_per_class) + ")"; - case HAILO_FORMAT_ORDER_NC: - return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) + - "(" + std::to_string(vstream_info.shape.features) + ")"; - case HAILO_FORMAT_ORDER_NHW: - return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) + - "(" +std::to_string(vstream_info.shape.height) + "x" + std::to_string(vstream_info.shape.width) + ")"; - default: - return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) + - "(" + std::to_string(vstream_info.shape.height) + "x" + std::to_string(vstream_info.shape.width) + "x" + - std::to_string(vstream_info.shape.features) + ")"; - } -} - ParseHefCommand::ParseHefCommand(CLI::App &parent_app) : Command(parent_app.add_subcommand("parse-hef", "Parse HEF to get information about its components")) { @@ -94,48 +40,9 @@ hailo_status ParseHefCommand::parse_hefs_info(const std::string &hef_path, bool CHECK_EXPECTED_AS_STATUS(hef_exp, "Failed to parse HEF"); auto hef = hef_exp.release(); - auto network_group_infos = hef.get_network_groups_infos(); - CHECK_EXPECTED_AS_STATUS(network_group_infos); - for (auto &network_group_info : network_group_infos.release()) { - auto contexts_str = (network_group_info.is_multi_context ? "Multi Context" : "Single Context"); - std::cout << "Network group name: " << network_group_info.name << " (" << contexts_str << ")" << std::endl; - auto network_infos = hef.get_network_infos(network_group_info.name); - CHECK_EXPECTED_AS_STATUS(network_infos, "Failed to parse networks infos"); - for (auto &network_info : network_infos.value()) { - std::cout << add_tabs(1) << "Network name: " << network_info.name << std::endl; - if (stream_infos) { - std::cout << add_tabs(2) << "Stream infos:" << std::endl; - auto input_stream_infos = hef.get_input_stream_infos(network_info.name); - CHECK_EXPECTED_AS_STATUS(input_stream_infos, "Failed to parse input stream infos"); - for (auto &stream_info : input_stream_infos.value()) { - auto shape_str = get_shape_str(stream_info); - std::cout << add_tabs(3) << "Input " << stream_info.name << " " << shape_str << std::endl; - } - auto output_stream_infos = hef.get_output_stream_infos(network_info.name); - CHECK_EXPECTED_AS_STATUS(output_stream_infos, "Failed to parse output stream infos"); - for (auto &stream_info : output_stream_infos.value()) { - auto shape_str = get_shape_str(stream_info); - std::cout << add_tabs(3) << "Output " << stream_info.name << " " << shape_str << std::endl; - } - } - if (vstream_infos) { - std::cout << add_tabs(2) << "VStream infos:" << std::endl; - auto input_vstream_infos = hef.get_input_vstream_infos(network_info.name); - CHECK_EXPECTED_AS_STATUS(input_vstream_infos, "Failed to parse input vstream infos"); - for (auto &vstream_info : input_vstream_infos.value()) { - auto shape_str = get_shape_str(vstream_info); - std::cout << add_tabs(3) << "Input " << vstream_info.name << " " << shape_str << std::endl; - } - auto output_vstream_infos = hef.get_output_vstream_infos(network_info.name); - CHECK_EXPECTED_AS_STATUS(output_vstream_infos, "Failed to parse output vstream infos"); - for (auto &vstream_info : output_vstream_infos.value()) { - auto shape_str = get_shape_str(vstream_info); - std::cout << add_tabs(3) << "Output " << vstream_info.name << " " << shape_str << std::endl; - } - } - } - } - std::cout << std::endl; + auto hef_info = hef.get_hef_description(stream_infos, vstream_infos); + CHECK_EXPECTED_AS_STATUS(hef_info, "Failed to parse HEF"); + std::cout << hef_info.release(); return HAILO_SUCCESS; } diff --git a/hailort/hailortcli/run2/live_printer.cpp b/hailort/hailortcli/run2/live_printer.cpp index d0bacf3..5e4e866 100644 --- a/hailort/hailortcli/run2/live_printer.cpp +++ b/hailort/hailortcli/run2/live_printer.cpp @@ -9,15 +9,20 @@ #include "live_printer.hpp" #include "../common.hpp" +#include "common/os_utils.hpp" +#include "common/utils.hpp" #include #include + using namespace hailort; LivePrinter::LivePrinter(std::chrono::milliseconds interval) : m_interval(interval), m_stop_event(Event::create_shared(Event::State::not_signalled)), m_tracks(), - m_mutex() + m_mutex(), + m_prev_count(0), + m_enable_ansi_escape_sequences(CursorAdjustment()) { } @@ -27,43 +32,55 @@ LivePrinter::~LivePrinter() if (m_thread.joinable()) { m_thread.join(); } - print(false); + print(); } -void LivePrinter::add(std::shared_ptr track) +void LivePrinter::add(std::shared_ptr track, uint8_t level) { std::unique_lock lock(m_mutex); - m_tracks.emplace_back(track); + if (!contains(m_tracks, level)) { + m_tracks[level] = {}; + } + m_tracks[level].emplace_back(track); } -void LivePrinter::print(bool reset) +void LivePrinter::print() { std::stringstream ss; uint32_t count = 0; { std::unique_lock lock(m_mutex); - for (auto &track : m_tracks) { - count += track->get_text(ss); + for (auto &level_pair : m_tracks) { + for (auto &track : level_pair.second) { + count += track->get_text(ss); + } } } - + CliCommon::reset_cursor(m_prev_count); + // On the first print m_prev_count = 0, so no lines will be deleted std::cout << ss.str() << std::flush; - if (reset) { - CliCommon::reset_cursor(count); - //TODO: what aout leftovers from prev line? - } + m_prev_count = count; } -void LivePrinter::start() +hailo_status LivePrinter::start() { + for (auto &level_pair : m_tracks) { + for (auto &track : level_pair.second) { + CHECK_SUCCESS(track->start()); + } + } + m_thread = std::thread([this] () { + OsUtils::set_current_thread_name("LIVE_PRINTER"); while (true) { - print(true); + print(); auto status = m_stop_event->wait(m_interval); if (HAILO_TIMEOUT != status) { break; } } }); -} \ No newline at end of file + + return HAILO_SUCCESS; +} diff --git a/hailort/hailortcli/run2/live_printer.hpp b/hailort/hailortcli/run2/live_printer.hpp index 0d67364..26ea8ea 100644 --- a/hailort/hailortcli/run2/live_printer.hpp +++ b/hailort/hailortcli/run2/live_printer.hpp @@ -10,12 +10,14 @@ #ifndef _HAILO_HAILORTCLI_RUN2_LIVE_PRINTER_HPP_ #define _HAILO_HAILORTCLI_RUN2_LIVE_PRINTER_HPP_ +#include "common/os_utils.hpp" #include "hailo/event.hpp" #include #include #include #include #include +#include class LivePrinter final { @@ -23,21 +25,30 @@ public: class Track { public: + Track() : m_started(false) + {} + + virtual hailo_status start() = 0; virtual uint32_t get_text(std::stringstream &ss) = 0; + + protected: + bool m_started; }; LivePrinter(std::chrono::milliseconds interval); ~LivePrinter(); - void add(std::shared_ptr track); - void print(bool reset); - void start(); + void add(std::shared_ptr track, uint8_t level); // prints tracks in consecutive order from low-to-high levels + void print(); + hailo_status start(); private: std::chrono::milliseconds m_interval; hailort::EventPtr m_stop_event; - std::vector> m_tracks; + std::map>> m_tracks; std::thread m_thread; std::mutex m_mutex; + uint32_t m_prev_count; + hailort::CursorAdjustment m_enable_ansi_escape_sequences; }; #endif /* _HAILO_HAILORTCLI_RUN2_LIVE_PRINTER_HPP_ */ \ No newline at end of file diff --git a/hailort/hailortcli/run2/measurement_live_track.cpp b/hailort/hailortcli/run2/measurement_live_track.cpp new file mode 100644 index 0000000..cf001bf --- /dev/null +++ b/hailort/hailortcli/run2/measurement_live_track.cpp @@ -0,0 +1,141 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file measurement_live_track.cpp + * @brief Device measurements live track + **/ + +#include "hailo/hailort.h" + +#include "common/device_measurements.hpp" +#include "common/utils.hpp" + +#include "measurement_live_track.hpp" + +#include +#include + + +using namespace hailort; + +Expected> MeasurementLiveTrack::create_shared(Device &device, bool measure_power, bool measure_current, + bool measure_temp) +{ + std::shared_ptr power_measurement = nullptr; + if (measure_power) { + auto power_measurement_exp = PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__POWER); + CHECK_EXPECTED(power_measurement_exp); + power_measurement = power_measurement_exp.release(); + } + + std::shared_ptr current_measurement = nullptr; + if (measure_current) { + auto current_measurement_exp = PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__CURRENT); + CHECK_EXPECTED(current_measurement_exp); + current_measurement = current_measurement_exp.release(); + } + + std::shared_ptr temp_measurement = nullptr; + if (measure_temp) { + auto temp_measurement_exp = TemperatureMeasurement::create_shared(device); + CHECK_EXPECTED(temp_measurement_exp); + temp_measurement = temp_measurement_exp.release(); + } + + auto ptr = make_shared_nothrow(power_measurement, current_measurement, temp_measurement, device.get_dev_id()); + CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +MeasurementLiveTrack::MeasurementLiveTrack(std::shared_ptr power_measurement, + std::shared_ptr current_measurement, std::shared_ptr temp_measurement, + const std::string &device_id) : + LivePrinter::Track(), m_power_measurement(std::move(power_measurement)), m_current_measurement(std::move(current_measurement)), + m_temp_measurement(std::move(temp_measurement)), m_device_id(device_id) +{} + +hailo_status MeasurementLiveTrack::start() +{ + if (m_power_measurement) { + CHECK_SUCCESS(m_power_measurement->start_measurement()); + } + + if (m_current_measurement) { + CHECK_SUCCESS(m_current_measurement->start_measurement()); + } + + if (m_temp_measurement) { + CHECK_SUCCESS(m_temp_measurement->start_measurement()); + } + + m_started = true; + + return HAILO_SUCCESS; +} + +uint32_t MeasurementLiveTrack::get_text(std::stringstream &ss) +{ + if (!m_started) { + return 0; + } + + auto rows_count = 0; + + if (m_power_measurement || m_current_measurement || m_temp_measurement) { + ss << fmt::format("\nMeasurements for device {}\n", m_device_id); + rows_count += 2; + } + + if (m_power_measurement) { + auto measurement_info = m_power_measurement->get_data(); + if (auto min = measurement_info.min()) { + ss << fmt::format("\tMinimum power consumption: {:.2f} {}\n", *min, m_power_measurement->measurement_unit()); + rows_count++; + } + if (auto mean = measurement_info.mean()) { + ss << fmt::format("\tAverage power consumption: {:.2f} {}\n", *mean, m_power_measurement->measurement_unit()); + rows_count++; + } + if (auto max = measurement_info.max()) { + ss << fmt::format("\tMaximum power consumption: {:.2f} {}\n", *max, m_power_measurement->measurement_unit()); + rows_count++; + } + } + + if (m_current_measurement) { + auto measurement_info = m_current_measurement->get_data(); + if (auto min = measurement_info.min()) { + ss << fmt::format("\tMinimum current consumption: {:.2f} {}\n", *min, m_current_measurement->measurement_unit()); + rows_count++; + } + if (auto mean = measurement_info.mean()) { + ss << fmt::format("\tAverage current consumption: {:.2f} {}\n", *mean, m_current_measurement->measurement_unit()); + rows_count++; + } + if (auto max = measurement_info.max()) { + ss << fmt::format("\tMaximum current consumption: {:.2f} {}\n", *max, m_current_measurement->measurement_unit()); + rows_count++; + } + } + + if (m_temp_measurement) { + auto measurement_info = m_temp_measurement->get_data(); + if (auto min = measurement_info.min()) { + ss << fmt::format("\tMinimum chip temperature: {:.2f} {}\n", *min, m_temp_measurement->measurement_unit()); + rows_count++; + } + if (auto mean = measurement_info.mean()) { + ss << fmt::format("\tAverage chip temperature: {:.2f} {}\n", *mean, m_temp_measurement->measurement_unit()); + rows_count++; + } + if (auto max = measurement_info.max()) { + ss << fmt::format("\tMaximum chip temperature: {:.2f} {}\n", *max, m_temp_measurement->measurement_unit()); + rows_count++; + } + } + + return rows_count; +} \ No newline at end of file diff --git a/hailort/hailortcli/run2/measurement_live_track.hpp b/hailort/hailortcli/run2/measurement_live_track.hpp new file mode 100644 index 0000000..17288d9 --- /dev/null +++ b/hailort/hailortcli/run2/measurement_live_track.hpp @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file measurement_live_track.hpp + * @brief Device measurements live track + **/ + +#ifndef _HAILO_HAILORTCLI_RUN2_MEASUREMENT_LIVE_TRACK_HPP_ +#define _HAILO_HAILORTCLI_RUN2_MEASUREMENT_LIVE_TRACK_HPP_ + +#include "hailo/hailort.h" + +#include "common/device_measurements.hpp" + +#include "live_printer.hpp" + + +class MeasurementLiveTrack : public LivePrinter::Track +{ +public: + static hailort::Expected> create_shared(hailort::Device &vdevice, bool measure_power, + bool measure_current, bool measure_temp); + + virtual ~MeasurementLiveTrack() = default; + virtual hailo_status start() override; + virtual uint32_t get_text(std::stringstream &ss) override; + + MeasurementLiveTrack(std::shared_ptr power_measurement, std::shared_ptr current_measurement, + std::shared_ptr temp_measurement, const std::string &device_id); + +private: + std::shared_ptr m_power_measurement; + std::shared_ptr m_current_measurement; + std::shared_ptr m_temp_measurement; + + std::string m_device_id; +}; + +#endif /* _HAILO_HAILORTCLI_RUN2_MEASUREMENT_LIVE_TRACK_HPP_ */ \ No newline at end of file diff --git a/hailort/hailortcli/run2/network_live_track.cpp b/hailort/hailortcli/run2/network_live_track.cpp index e3dc99c..b4e85c2 100644 --- a/hailort/hailortcli/run2/network_live_track.cpp +++ b/hailort/hailortcli/run2/network_live_track.cpp @@ -8,26 +8,64 @@ **/ #include "network_live_track.hpp" +#include "../infer_stats_printer.hpp" + #include #include -NetworkLiveTrack::NetworkLiveTrack(const std::string &name) : - m_name(name), m_count(0), m_last_get_time(std::chrono::steady_clock::now()) +NetworkLiveTrack::NetworkLiveTrack(const std::string &name, std::shared_ptr cng, LatencyMeterPtr overall_latency_meter) : + m_name(name), m_count(0), m_last_get_time(), m_cng(cng), m_overall_latency_meter(overall_latency_meter) { } +hailo_status NetworkLiveTrack::start() +{ + m_last_get_time = std::chrono::steady_clock::now(); + m_count = 0; + m_started = true; + + return HAILO_SUCCESS; +} + uint32_t NetworkLiveTrack::get_text(std::stringstream &ss) { + if (!m_started) { + return 0; + } + auto elapsed_time = std::chrono::steady_clock::now() - m_last_get_time; auto count = m_count.load(); auto fps = count / std::chrono::duration(elapsed_time).count(); - ss << fmt::format("{} - fps: {:.2f}\n", m_name, fps); + ss << fmt::format("{}:\n\t| fps: {:.2f}", m_name, fps); + + auto hw_latency_measurement = m_cng->get_latency_measurement(); + if (hw_latency_measurement) { + ss << fmt::format(" | hw latency: {:.2f} ms", InferResultsFormatUtils::latency_result_to_ms(hw_latency_measurement->avg_hw_latency)); + } + else if (HAILO_NOT_AVAILABLE != hw_latency_measurement.status()) { // HAILO_NOT_AVAILABLE is a valid error, we ignore it + ss << fmt::format(" | hw latency: failed with status={}", hw_latency_measurement.status()); + } - return 1; + if (m_overall_latency_meter) { + auto overall_latency_measurement = m_overall_latency_meter->get_latency(true); + if (overall_latency_measurement) { + ss << fmt::format(" | overall latency: {:.2f} ms", InferResultsFormatUtils::latency_result_to_ms(*overall_latency_measurement)); + } + else if (HAILO_NOT_AVAILABLE != overall_latency_measurement.status()) { // HAILO_NOT_AVAILABLE is a valid error, we ignore it + ss << fmt::format(" | overall latency: failed with status={}", overall_latency_measurement.status()); + } + } + ss << "\n"; + + return 2; } void NetworkLiveTrack::progress() { + if (!m_started) { + return; + } + m_count++; } \ No newline at end of file diff --git a/hailort/hailortcli/run2/network_live_track.hpp b/hailort/hailortcli/run2/network_live_track.hpp index 1eacb52..dec00fd 100644 --- a/hailort/hailortcli/run2/network_live_track.hpp +++ b/hailort/hailortcli/run2/network_live_track.hpp @@ -7,23 +7,32 @@ * @brief Network live track **/ -#include "live_printer.hpp" - #ifndef _HAILO_HAILORTCLI_RUN2_NETWORK_LIVE_TRACK_HPP_ #define _HAILO_HAILORTCLI_RUN2_NETWORK_LIVE_TRACK_HPP_ +#include "hailo/hailort.h" +#include "hailo/network_group.hpp" + +#include "common/latency_meter.hpp" + +#include "live_printer.hpp" + + class NetworkLiveTrack : public LivePrinter::Track { public: - NetworkLiveTrack(const std::string &name); + NetworkLiveTrack(const std::string &name, std::shared_ptr cng, hailort::LatencyMeterPtr overall_latency_meter); virtual ~NetworkLiveTrack() = default; - uint32_t get_text(std::stringstream &ss); + virtual hailo_status start() override; + virtual uint32_t get_text(std::stringstream &ss) override; void progress(); private: std::string m_name; std::atomic m_count; std::chrono::time_point m_last_get_time; + std::shared_ptr m_cng; + hailort::LatencyMeterPtr m_overall_latency_meter; }; #endif /* _HAILO_HAILORTCLI_RUN2_NETWORK_LIVE_TRACK_HPP_ */ \ No newline at end of file diff --git a/hailort/hailortcli/run2/network_runner.cpp b/hailort/hailortcli/run2/network_runner.cpp index 130817c..f2901a8 100644 --- a/hailort/hailortcli/run2/network_runner.cpp +++ b/hailort/hailortcli/run2/network_runner.cpp @@ -7,12 +7,35 @@ * @brief Run network on hailo device **/ -#include "network_runner.hpp" +#include "hailo/hailort.h" +#include "hailo/hailort_common.hpp" +#include "hailo/hailort_defaults.hpp" + #include "common/async_thread.hpp" -#include "hailort_defaults.hpp" //TODO: not API +#include "common/file_utils.hpp" +#include "common/latency_meter.hpp" + +#include "network_runner.hpp" + using namespace hailort; + +class SignalEventScopeGuard final +{ +public: + SignalEventScopeGuard(Event &event) : m_event(event) + {} + + ~SignalEventScopeGuard() + { + m_event.signal(); + } + + Event &m_event; +}; + + //TODO: duplicated static hailo_status wait_for_threads(std::vector> &threads) { @@ -31,10 +54,17 @@ VStreamParams::VStreamParams() : name(), params(HailoRTDefaults::get_vstreams_pa { } +NetworkParams::NetworkParams() : hef_path(), net_group_name(), vstream_params(), scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN), + batch_size(HAILO_DEFAULT_BATCH_SIZE), scheduler_threshold(0), scheduler_timeout_ms(0), framerate(UNLIMITED_FRAMERATE), measure_hw_latency(false), + measure_overall_latency(false) +{ +} + NetworkRunner::NetworkRunner(const NetworkParams ¶ms, const std::string &name, - std::vector &&input_vstreams, std::vector &&output_vstreams) + std::vector &&input_vstreams, std::vector &&output_vstreams, + std::shared_ptr cng, LatencyMeterPtr overall_latency_meter) : m_params(params), m_name(name), m_input_vstreams(std::move(input_vstreams)), - m_output_vstreams(std::move(output_vstreams)) + m_output_vstreams(std::move(output_vstreams)), m_cng(cng), m_overall_latency_meter(overall_latency_meter) { } @@ -51,19 +81,22 @@ Expected> NetworkRunner::create_shared(VDevice &v net_group_name = net_groups_names[0]; } - auto interface = vdevice.get_default_streams_interface(); - CHECK_EXPECTED(interface, "Failed to get default streams interface"); - - auto cfg_params = hef->create_configure_params(*interface, net_group_name); + auto cfg_params = vdevice.create_configure_params(hef.value(), net_group_name); CHECK_EXPECTED(cfg_params); cfg_params->batch_size = params.batch_size; + if (params.measure_hw_latency) { + cfg_params->latency |= HAILO_LATENCY_MEASURE; + } auto cfgr_net_groups = vdevice.configure(hef.value(), {{net_group_name, cfg_params.value()}}); CHECK_EXPECTED(cfgr_net_groups); assert(1 == cfgr_net_groups->size()); auto cfgr_net_group = cfgr_net_groups.value()[0]; - CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_threshold(params.scheduler_threshold)); - CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_timeout(std::chrono::milliseconds(params.scheduler_timeout_ms))); + if (HAILO_SCHEDULING_ALGORITHM_NONE!= params.scheduling_algorithm) { + CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_threshold(params.scheduler_threshold)); + CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_timeout(std::chrono::milliseconds(params.scheduler_timeout_ms))); + CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_priority(params.scheduler_priority)); + } std::map vstreams_params; for (auto &vstream_params : params.vstream_params) { @@ -72,23 +105,69 @@ Expected> NetworkRunner::create_shared(VDevice &v auto vstreams = create_vstreams(*cfgr_net_group, vstreams_params); CHECK_EXPECTED(vstreams); - auto net_runner = make_shared_nothrow(params, net_group_name, std::move(vstreams->first), std::move(vstreams->second)); + LatencyMeterPtr overall_latency_meter = nullptr; + if (params.measure_overall_latency) { + CHECK_AS_EXPECTED((1 == vstreams->first.size()), HAILO_INVALID_OPERATION, + "Overall latency measurement over multiple inputs network is not supported"); + + std::set output_names; + for (auto &output_vstream : vstreams->second) { + output_names.insert(output_vstream.name()); + } + + overall_latency_meter = make_shared_nothrow(output_names, OVERALL_LATENCY_TIMESTAMPS_LIST_LENGTH); + CHECK_NOT_NULL_AS_EXPECTED(overall_latency_meter, HAILO_OUT_OF_HOST_MEMORY); + } + auto net_runner = make_shared_nothrow(params, net_group_name, std::move(vstreams->first), + std::move(vstreams->second), cfgr_net_group, overall_latency_meter); CHECK_NOT_NULL_AS_EXPECTED(net_runner, HAILO_OUT_OF_HOST_MEMORY); return net_runner; } -hailo_status NetworkRunner::run_input_vstream(InputVStream &vstream) +Expected NetworkRunner::create_dataset_from_input_file(const std::string &file_path, + const InputVStream &input_vstream) { - auto dataset = Buffer::create(vstream.get_frame_size(), 0xAB); - CHECK_EXPECTED_AS_STATUS(dataset); + auto buffer = read_binary_file(file_path); + CHECK_EXPECTED(buffer); + CHECK_AS_EXPECTED(0 == (buffer->size() % input_vstream.get_frame_size()), HAILO_INVALID_ARGUMENT, + "Input file ({}) size {} must be a multiple of the frame size {} ({})", + file_path, buffer->size(), input_vstream.get_frame_size(), input_vstream.name()); + + auto buffer_ptr = make_shared_nothrow(buffer.release()); + CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY); + + return buffer_ptr; +} + + +Expected NetworkRunner::create_constant_dataset(const InputVStream &input_vstream) +{ + const uint8_t const_byte = 0xAB; + auto constant_buffer = Buffer::create_shared(input_vstream.get_frame_size(), const_byte); + CHECK_EXPECTED(constant_buffer); + + return constant_buffer.release(); +} + +hailo_status NetworkRunner::run_input_vstream(InputVStream &vstream, Event &shutdown_event, BufferPtr dataset, + LatencyMeterPtr overall_latency_meter) +{ + auto signal_event_scope_guard = SignalEventScopeGuard(shutdown_event); + auto last_write_time = std::chrono::steady_clock::now(); auto framerate_interval = std::chrono::duration(1) / m_params.framerate; + size_t buffer_offset = 0; while(true) { - auto status = vstream.write(MemoryView(dataset.value())); + if (overall_latency_meter) { + overall_latency_meter->add_start_sample(std::chrono::steady_clock::now().time_since_epoch()); + } + auto status = vstream.write(MemoryView((dataset->data() + buffer_offset), vstream.get_frame_size())); if (status == HAILO_STREAM_ABORTED_BY_USER) { return status; } CHECK_SUCCESS(status); + buffer_offset += vstream.get_frame_size(); + buffer_offset %= dataset->size(); if (m_params.framerate != UNLIMITED_FRAMERATE) { auto elapsed_time = std::chrono::steady_clock::now() - last_write_time; @@ -99,8 +178,11 @@ hailo_status NetworkRunner::run_input_vstream(InputVStream &vstream) return HAILO_SUCCESS; } -hailo_status NetworkRunner::run_output_vstream(OutputVStream &vstream, bool first, std::shared_ptr net_live_track) +hailo_status NetworkRunner::run_output_vstream(OutputVStream &vstream, bool first, std::shared_ptr net_live_track, + Event &shutdown_event, LatencyMeterPtr overall_latency_meter) { + auto signal_event_scope_guard = SignalEventScopeGuard(shutdown_event); + auto result = Buffer::create(vstream.get_frame_size()); CHECK_EXPECTED_AS_STATUS(result); while(true) { @@ -109,6 +191,9 @@ hailo_status NetworkRunner::run_output_vstream(OutputVStream &vstream, bool firs return status; } CHECK_SUCCESS(status); + if (overall_latency_meter) { + overall_latency_meter->add_end_sample(vstream.name(), std::chrono::steady_clock::now().time_since_epoch()); + } if (first) { net_live_track->progress(); } @@ -116,27 +201,54 @@ hailo_status NetworkRunner::run_output_vstream(OutputVStream &vstream, bool firs return HAILO_SUCCESS; } -hailo_status NetworkRunner::run(Event &shutdown_event, LivePrinter &live_printer) +hailo_status NetworkRunner::run(Event &shutdown_event, LivePrinter &live_printer, Barrier &barrier) { + auto ang = std::unique_ptr(nullptr); + if (HAILO_SCHEDULING_ALGORITHM_NONE == m_params.scheduling_algorithm) { + auto ang_exp = m_cng->activate(); + if (!ang_exp) { + barrier.terminate(); + } + CHECK_EXPECTED_AS_STATUS(ang_exp); + ang = ang_exp.release(); + } + + auto net_live_track = std::make_shared(m_name, m_cng, m_overall_latency_meter); + live_printer.add(net_live_track, 1); //support progress over multiple outputs + barrier.arrive_and_wait(); + std::vector> threads; for (auto &input_vstream : m_input_vstreams) { - threads.emplace_back(std::make_unique>([this, &input_vstream](){ - return run_input_vstream(input_vstream); + BufferPtr dataset = nullptr; + for (auto ¶ms : m_params.vstream_params) { + if ((input_vstream.name() == params.name) && (!params.input_file_path.empty())) { + auto dataset_exp = create_dataset_from_input_file(params.input_file_path, input_vstream); + CHECK_EXPECTED_AS_STATUS(dataset_exp); + dataset = dataset_exp.release(); + } + } + if (nullptr == dataset) { + auto dataset_exp = create_constant_dataset(input_vstream); + CHECK_EXPECTED_AS_STATUS(dataset_exp); + dataset = dataset_exp.release(); + } + + threads.emplace_back(std::make_unique>("SEND", [this, &input_vstream, &shutdown_event, + dataset](){ + return run_input_vstream(input_vstream, shutdown_event, dataset, m_overall_latency_meter); })); } - auto net_live_track = std::make_shared(m_name); - live_printer.add(net_live_track);//support progress over multiple outputs - - bool first = true;//TODO: check with multiple outputs + bool first = true; //TODO: check with multiple outputs for (auto &output_vstream : m_output_vstreams) { - threads.emplace_back(std::make_unique>([&output_vstream, first, net_live_track](){ - return run_output_vstream(output_vstream, first, net_live_track); + threads.emplace_back(std::make_unique>("RECV", [this, &output_vstream, first, net_live_track, + &shutdown_event](){ + return run_output_vstream(output_vstream, first, net_live_track, shutdown_event, m_overall_latency_meter); })); first = false; } - //TODO: signal a barrier that we should start infer and timer. return threads and move stop outside? + //TODO: return threads and move stop outside? CHECK_SUCCESS(shutdown_event.wait(HAILO_INFINITE_TIMEOUT)); stop(); return wait_for_threads(threads); diff --git a/hailort/hailortcli/run2/network_runner.hpp b/hailort/hailortcli/run2/network_runner.hpp index d59cf23..dda0651 100644 --- a/hailort/hailortcli/run2/network_runner.hpp +++ b/hailort/hailortcli/run2/network_runner.hpp @@ -10,11 +10,16 @@ #ifndef _HAILO_HAILORTCLI_RUN2_NETWORK_RUNNER_HPP_ #define _HAILO_HAILORTCLI_RUN2_NETWORK_RUNNER_HPP_ +#include "common/barrier.hpp" + #include "hailo/vdevice.hpp" #include "hailo/vstream.hpp" #include "hailo/event.hpp" #include "hailo/network_group.hpp" #include "hailo/expected.hpp" +#include "hailo/buffer.hpp" + +#include "../hailortcli.hpp" #include "live_printer.hpp" #include "network_live_track.hpp" @@ -30,43 +35,58 @@ struct VStreamParams std::string name; hailo_vstream_params_t params; + std::string input_file_path; }; struct NetworkParams { + NetworkParams(); + std::string hef_path; std::string net_group_name; std::vector vstream_params; + hailo_scheduling_algorithm_t scheduling_algorithm; // Network parameters uint16_t batch_size; uint32_t scheduler_threshold; uint32_t scheduler_timeout_ms; + uint8_t scheduler_priority; // Run parameters uint32_t framerate; + + bool measure_hw_latency; + bool measure_overall_latency; }; class NetworkRunner { public: NetworkRunner(const NetworkParams ¶ms, const std::string &name, - std::vector &&input_vstreams, std::vector &&output_vstreams); + std::vector &&input_vstreams, std::vector &&output_vstreams, + std::shared_ptr cng, hailort::LatencyMeterPtr overall_latency_meter); static hailort::Expected> create_shared(hailort::VDevice &vdevice, const NetworkParams ¶ms); - hailo_status run(hailort::Event &shutdown_event, LivePrinter &live_printer); + hailo_status run(hailort::Event &shutdown_event, LivePrinter &live_printer, hailort::Barrier &barrier); void stop(); private: static hailort::Expected, std::vector>> create_vstreams( hailort::ConfiguredNetworkGroup &net_group, const std::map ¶ms); - hailo_status run_input_vstream(hailort::InputVStream &vstream); - static hailo_status run_output_vstream(hailort::OutputVStream &vstream, bool first, std::shared_ptr net_live_track); + hailo_status run_input_vstream(hailort::InputVStream &vstream, hailort::Event &shutdown_event, hailort::BufferPtr dataset, + hailort::LatencyMeterPtr overall_latency_meter); + static hailo_status run_output_vstream(hailort::OutputVStream &vstream, bool first, std::shared_ptr net_live_track, + hailort::Event &shutdown_event, hailort::LatencyMeterPtr overall_latency_meter); +static hailort::Expected create_constant_dataset(const hailort::InputVStream &input_vstream); +static hailort::Expected create_dataset_from_input_file(const std::string &file_path, const hailort::InputVStream &input_vstream); const NetworkParams &m_params;//TODO: copy instead of ref? std::string m_name; std::vector m_input_vstreams; std::vector m_output_vstreams; + std::shared_ptr m_cng; + hailort::LatencyMeterPtr m_overall_latency_meter; }; #endif /* _HAILO_HAILORTCLI_RUN2_NETWORK_RUNNER_HPP_ */ \ No newline at end of file diff --git a/hailort/hailortcli/run2/run2_command.cpp b/hailort/hailortcli/run2/run2_command.cpp index 01d492f..e4c1070 100644 --- a/hailort/hailortcli/run2/run2_command.cpp +++ b/hailort/hailortcli/run2/run2_command.cpp @@ -10,8 +10,10 @@ #include "run2_command.hpp" #include "live_printer.hpp" #include "timer_live_track.hpp" +#include "measurement_live_track.hpp" #include "network_runner.hpp" +#include "common/barrier.hpp" #include "common/async_thread.hpp" #include "hailo/vdevice.hpp" #include "hailo/hef.hpp" @@ -91,6 +93,10 @@ VStreamApp::VStreamApp(const std::string &description, const std::string &name, add_option("name", m_params.name, "vStream name") ->check(VStreamNameValidator(hef_path_option, net_group_name_option)); + add_option("--input-file", m_params.input_file_path, + "Input file path. If not given, random data will be used. File format should be raw binary data with size that is a factor of the input shape size") + ->default_val(""); + auto format_opt_group = add_option_group("Format"); format_opt_group->add_option("--type", m_params.params.user_buffer_format.type, "Format type") ->transform(HailoCheckedTransformer({ @@ -116,14 +122,16 @@ VStreamApp::VStreamApp(const std::string &description, const std::string &name, { "nchw", HAILO_FORMAT_ORDER_NCHW }, { "yuy2", HAILO_FORMAT_ORDER_YUY2 }, { "nv12", HAILO_FORMAT_ORDER_NV12 }, - { "nv21", HAILO_FORMAT_ORDER_NV21 } + { "nv21", HAILO_FORMAT_ORDER_NV21 }, + { "rgb4", HAILO_FORMAT_ORDER_RGB4 }, + { "i420", HAILO_FORMAT_ORDER_I420 } })) ->default_val("auto"); add_flag_callback(format_opt_group, "-q,--quantized,!--no-quantized", "Whether or not data is quantized", [this](bool result){ m_params.params.user_buffer_format.flags = result ? - m_params.params.user_buffer_format.flags | HAILO_FORMAT_FLAGS_QUANTIZED : + static_cast(m_params.params.user_buffer_format.flags | HAILO_FORMAT_FLAGS_QUANTIZED) : static_cast(m_params.params.user_buffer_format.flags & (~HAILO_FORMAT_FLAGS_QUANTIZED));}) ->run_callback_for_default() ->default_val(true); // default_val() must be after run_callback_for_default() @@ -188,10 +196,14 @@ NetworkApp::NetworkApp(const std::string &description, const std::string &name) net_params->add_option("--batch-size", m_params.batch_size, "Batch size")->default_val(HAILO_DEFAULT_BATCH_SIZE); net_params->add_option("--scheduler-threshold", m_params.scheduler_threshold, "Scheduler threshold")->default_val(0); net_params->add_option("--scheduler-timeout", m_params.scheduler_timeout_ms, "Scheduler timeout in milliseconds")->default_val(0); + net_params->add_option("--scheduler-priority", m_params.scheduler_priority, "Scheduler priority")->default_val(HAILO_SCHEDULER_PRIORITY_NORMAL); auto run_params = add_option_group("Run Parameters"); run_params->add_option("--framerate", m_params.framerate, "Input vStreams framerate")->default_val(UNLIMITED_FRAMERATE); + // TODO: support multiple scheduling algorithms + m_params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN; + add_vstream_app_subcom(hef_path_option, net_group_name_option); } @@ -235,19 +247,74 @@ public: const std::vector& get_network_params(); std::chrono::seconds get_time_to_run(); + std::vector get_dev_ids(); + uint32_t get_device_count(); + bool get_measure_power(); + bool get_measure_current(); + bool get_measure_temp(); + bool get_multi_process_service(); + const std::string &get_group_id(); + + void set_scheduling_algorithm(hailo_scheduling_algorithm_t scheduling_algorithm); + void set_measure_latency(); private: void add_net_app_subcom(); std::vector m_network_params; uint32_t m_time_to_run; + std::vector m_device_id; + uint32_t m_device_count; + bool m_multi_process_service; + std::string m_group_id; + + bool m_measure_hw_latency; + bool m_measure_overall_latency; + + bool m_measure_power; + bool m_measure_current; + bool m_measure_temp; }; + Run2::Run2() : CLI::App("Run networks (preview)", "run2") { add_net_app_subcom(); add_option("-t,--time-to-run", m_time_to_run, "Time to run (seconds)") ->default_val(DEFAULT_TIME_TO_RUN_SECONDS) ->check(CLI::PositiveNumber); + + auto vdevice_options_group = add_option_group("VDevice Options"); + + auto dev_id_opt = vdevice_options_group->add_option("-s,--device-id", m_device_id, + "Device id, same as returned from `hailortcli scan` command. For multiple devices, use space as separator."); + + vdevice_options_group->add_option("--device-count", m_device_count, "VDevice device count") + ->default_val(HAILO_DEFAULT_DEVICE_COUNT) + ->check(CLI::PositiveNumber) + ->excludes(dev_id_opt); + + vdevice_options_group->add_flag("--multi-process-service", m_multi_process_service, "VDevice multi process service") + ->default_val(false); + + vdevice_options_group->add_option("--group-id", m_group_id, "VDevice group id") + ->default_val(HAILO_DEFAULT_VDEVICE_GROUP_ID); + + auto measurement_options_group = add_option_group("Measurement Options"); + + auto measure_power_opt = measurement_options_group->add_flag("--measure-power", m_measure_power, "Measure power consumption") + ->default_val(false); + + measurement_options_group->add_flag("--measure-current", m_measure_current, "Measure current")->excludes(measure_power_opt) + ->default_val(false); + + measurement_options_group->add_flag("--measure-latency", m_measure_hw_latency, "Measure network latency") + ->default_val(false); + + measurement_options_group->add_flag("--measure-overall-latency", m_measure_overall_latency, "Measure overall latency measurement") + ->default_val(false); + + measurement_options_group->add_flag("--measure-temp", m_measure_temp, "Measure chip temperature") + ->default_val(false); } void Run2::add_net_app_subcom() @@ -286,6 +353,65 @@ std::chrono::seconds Run2::get_time_to_run() return std::chrono::seconds(m_time_to_run); } +bool Run2::get_measure_power() +{ + return m_measure_power; +} + +bool Run2::get_measure_current() +{ + return m_measure_current; +} + +bool Run2::get_measure_temp() +{ + return m_measure_temp; +} + +std::vector Run2::get_dev_ids() +{ + std::vector res; + res.reserve(m_device_id.size()); + for (auto &id_str : m_device_id) { + hailo_device_id_t id = {}; + std::memset(id.id, 0, sizeof(id.id)); + std::strncpy(id.id, id_str.c_str(), sizeof(id.id) - 1); + res.push_back(id); + } + return res; +} + +uint32_t Run2::get_device_count() +{ + return m_device_count; +} + +void Run2::set_scheduling_algorithm(hailo_scheduling_algorithm_t scheduling_algorithm) +{ + for (auto ¶ms: m_network_params) { + params.scheduling_algorithm = scheduling_algorithm; + } +} + +void Run2::set_measure_latency() +{ + for (auto ¶ms: m_network_params) { + params.measure_hw_latency = m_measure_hw_latency; + params.measure_overall_latency = m_measure_overall_latency; + } +} + +bool Run2::get_multi_process_service() +{ + return m_multi_process_service; +} + +const std::string &Run2::get_group_id() +{ + return m_group_id; +} + + /** Run2Command */ Run2Command::Run2Command(CLI::App &parent_app) : Command(parent_app.add_subcommand(std::make_shared())) { @@ -304,10 +430,19 @@ static hailo_status wait_for_threads(std::vector> & return last_error_status; } +bool is_valid_ip(const std::string &ip) +{ + int a,b,c,d; + return (4 == sscanf(ip.c_str(),"%d.%d.%d.%d", &a, &b, &c, &d)) && + IS_FIT_IN_UINT8(a) && IS_FIT_IN_UINT8(b) && IS_FIT_IN_UINT8(c) && IS_FIT_IN_UINT8(d); +} + hailo_status Run2Command::execute() { Run2 *app = reinterpret_cast(m_app); + app->set_measure_latency(); + if (0 == app->get_network_params().size()) { LOGGER__ERROR("Nothing to run"); return HAILO_INVALID_OPERATION; @@ -316,9 +451,26 @@ hailo_status Run2Command::execute() LOGGER__WARN("\"hailortcli run2\" is in preview. It is recommended to use \"hailortcli run\" command for a single network group"); } - // TODO: support multi-device. maybe get all by default? hailo_vdevice_params_t vdevice_params = {}; CHECK_SUCCESS(hailo_init_vdevice_params(&vdevice_params)); + auto dev_ids = app->get_dev_ids(); + if (!dev_ids.empty()) { + vdevice_params.device_count = static_cast(dev_ids.size()); + vdevice_params.device_ids = dev_ids.data(); + + // Disable scheduler for eth VDevice + if ((1 == dev_ids.size()) && (is_valid_ip(dev_ids[0].id))) { + vdevice_params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_NONE; + CHECK(1 == app->get_network_params().size(), HAILO_INVALID_OPERATION, "On Ethernet inference only one model is allowed"); + app->set_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_NONE); + } + } else { + vdevice_params.device_count = app->get_device_count(); + } + + vdevice_params.group_id = app->get_group_id().c_str(); + vdevice_params.multi_process_service = app->get_multi_process_service(); + auto vdevice = VDevice::create(vdevice_params); CHECK_EXPECTED_AS_STATUS(vdevice); @@ -327,22 +479,43 @@ hailo_status Run2Command::execute() for (auto &net_params : app->get_network_params()) { auto net_runner = NetworkRunner::create_shared(*vdevice->get(), net_params); CHECK_EXPECTED_AS_STATUS(net_runner); + net_runners.emplace_back(net_runner.release()); } - LivePrinter live_printer(std::chrono::seconds(1)); - live_printer.add(std::make_shared(app->get_time_to_run())); + auto live_printer = std::make_unique(std::chrono::seconds(1)); + + live_printer->add(std::make_shared(app->get_time_to_run()), 0); auto shutdown_event = Event::create(Event::State::not_signalled); CHECK_EXPECTED_AS_STATUS(shutdown_event); std::vector> threads; + Barrier barrier(net_runners.size() + 1); // We wait for all nets to finish activation + this thread to start sampling for (auto &net_runner : net_runners) { - threads.emplace_back(std::make_unique>([&net_runner, &shutdown_event, &live_printer](){ - return net_runner->run(shutdown_event.value(), live_printer); + threads.emplace_back(std::make_unique>("NG_INFER", [&net_runner, &shutdown_event, + &live_printer, &barrier](){ + return net_runner->run(shutdown_event.value(), *live_printer, barrier); })); } + + auto physical_devices = vdevice.value()->get_physical_devices(); + CHECK_EXPECTED_AS_STATUS(physical_devices); + + for (auto &device : physical_devices.value()) { + auto measurement_live_track = MeasurementLiveTrack::create_shared(device.get(), app->get_measure_power(), + app->get_measure_current(), app->get_measure_temp()); + CHECK_EXPECTED_AS_STATUS(measurement_live_track); + live_printer->add(measurement_live_track.release(), 2); + } + // TODO: wait for all nets before starting timer. start() should update TimerLiveTrack to start. or maybe append here but first in vector... - live_printer.start(); - std::this_thread::sleep_for(app->get_time_to_run()); + barrier.arrive_and_wait(); + CHECK_SUCCESS(live_printer->start()); + auto status = shutdown_event->wait(app->get_time_to_run()); + if (HAILO_TIMEOUT != status) { + // if shutdown_event is signaled its because one of the send/recv threads failed + LOGGER__ERROR("Encountered error during inference. See log for more information."); + } + live_printer.reset(); // Ensures that the final print will include real values and not with values of when streams are already aborted. shutdown_event->signal(); return wait_for_threads(threads); } \ No newline at end of file diff --git a/hailort/hailortcli/run2/timer_live_track.cpp b/hailort/hailortcli/run2/timer_live_track.cpp index 33aeb2b..05fd73d 100644 --- a/hailort/hailortcli/run2/timer_live_track.cpp +++ b/hailort/hailortcli/run2/timer_live_track.cpp @@ -13,14 +13,25 @@ #include TimerLiveTrack::TimerLiveTrack(std::chrono::milliseconds duration) : - m_duration(duration), m_start(std::chrono::steady_clock::now()) + LivePrinter::Track(), m_duration(duration), m_start_time() { } +hailo_status TimerLiveTrack::start() +{ + m_start_time = std::chrono::steady_clock::now(); + m_started = true; + + return HAILO_SUCCESS; +} + uint32_t TimerLiveTrack::get_text(std::stringstream &ss) { + if (!m_started) { + return 0; + } static const uint32_t MAX_PROGRESS_BAR_WIDTH = 20; - auto elapsed_time = std::chrono::steady_clock::now() - m_start; + auto elapsed_time = std::chrono::steady_clock::now() - m_start_time; auto eta = std::chrono::seconds(std::max(0, static_cast(std::round(std::chrono::duration(m_duration - elapsed_time).count())))); // std::chrono::round is from C++17 auto elapsed_percentage = std::min(100, static_cast(std::round(std::chrono::duration(100 * elapsed_time / m_duration).count()))); auto progress_bar_width = std::max(1, std::min(MAX_PROGRESS_BAR_WIDTH, diff --git a/hailort/hailortcli/run2/timer_live_track.hpp b/hailort/hailortcli/run2/timer_live_track.hpp index ce9d548..af6e7e7 100644 --- a/hailort/hailortcli/run2/timer_live_track.hpp +++ b/hailort/hailortcli/run2/timer_live_track.hpp @@ -17,11 +17,12 @@ class TimerLiveTrack : public LivePrinter::Track public: TimerLiveTrack(std::chrono::milliseconds duration); virtual ~TimerLiveTrack() = default; + virtual hailo_status start() override; virtual uint32_t get_text(std::stringstream &ss) override; private: std::chrono::milliseconds m_duration; - std::chrono::time_point m_start; + std::chrono::time_point m_start_time; }; #endif /* _HAILO_HAILORTCLI_RUN2_TIMER_LIVE_TRACK_HPP_ */ \ No newline at end of file diff --git a/hailort/hailortcli/run_command.cpp b/hailort/hailortcli/run_command.cpp index ef4bc82..05be13c 100644 --- a/hailort/hailortcli/run_command.cpp +++ b/hailort/hailortcli/run_command.cpp @@ -11,7 +11,6 @@ #include "hailortcli.hpp" #include "inference_progress.hpp" #include "infer_stats_printer.hpp" -#include "temp_measurement.hpp" #include "graph_printer.hpp" #if defined(__GNUC__) // TODO: Support on windows (HRT-5919) @@ -25,10 +24,13 @@ #include "common/barrier.hpp" #include "common/latency_meter.hpp" #include "common/filesystem.hpp" +#include "common/device_measurements.hpp" +#include "hailo/hailort.h" #include "hailo/network_group.hpp" #include "hailo/hef.hpp" #include "hailo/vstream.hpp" #include "hailo/vdevice.hpp" +#include "hailo/transform.hpp" #include "spdlog/fmt/fmt.h" @@ -43,7 +45,6 @@ std::condition_variable wait_for_exit_cv; They're useful for simple interprocess communication. */ #define USER_SIGNAL (SIGUSR1) -constexpr size_t OVERALL_LATENCY_TIMESTAMPS_LIST_LENGTH (512); constexpr uint32_t DEFAULT_TIME_TO_RUN_SECONDS = 5; #ifndef HAILO_EMULATOR constexpr std::chrono::milliseconds TIME_TO_WAIT_FOR_CONFIG(300); @@ -226,8 +227,8 @@ static void add_run_command_params(CLI::App *run_subcommand, inference_runner_pa "Collect runtime data to be used by the Profiler"); static const char *JSON_SUFFIX = ".json"; collect_runtime_data_subcommand->add_option("--output-path", params.runtime_data.runtime_data_output_path, - fmt::format("Runtime data output file path\n'{}' will be replaced with the current running hef", - RUNTIME_DATA_OUTPUT_PATH_HEF_PLACE_HOLDER)) + fmt::format("Runtime data output file path\n'{}' will be replaced with the current running hef", RUNTIME_DATA_OUTPUT_PATH_HEF_PLACE_HOLDER) + + "\nIn case of multiple-devices, _ will be added as prefix to each file") ->default_val(fmt::format("runtime_data_{}.json", RUNTIME_DATA_OUTPUT_PATH_HEF_PLACE_HOLDER)) ->check(FileSuffixValidator(JSON_SUFFIX)); collect_runtime_data_subcommand->add_option("--batch-to-measure", params.runtime_data.batch_to_measure_str, @@ -286,9 +287,6 @@ static void add_run_command_params(CLI::App *run_subcommand, inference_runner_pa params.time_to_run = DEFAULT_TIME_TO_RUN_SECONDS; } - PARSE_CHECK(((!params.runtime_data.collect_runtime_data) || (params.vdevice_params.device_count == 1)), - "Passing runtime data is not supported for multiple devices"); - PARSE_CHECK((!(params.runtime_data.collect_runtime_data && params.vdevice_params.multi_process_service)), "Passing runtime data is not supported for multi process service"); @@ -670,7 +668,7 @@ static hailo_status run_streaming_impl(std::shared_ptr c auto first = true; for (auto& recv_object : recv_objects) { auto &frames_recieved = frames_recieved_per_output[output_index]; - results.emplace_back(std::make_unique>( + results.emplace_back(std::make_unique>("RECV", [network_progress_bar, params, &recv_object, &output_buffers, first, &barrier, &overall_latency_meter, &frames_recieved, batch_size]() { auto res = recv_loop(params, recv_object.get(), network_progress_bar, barrier, overall_latency_meter, @@ -685,7 +683,7 @@ static hailo_status run_streaming_impl(std::shared_ptr c ++output_index; } for (auto &send_object : send_objects) { - results.emplace_back(std::make_unique>( + results.emplace_back(std::make_unique>("SEND", [params, &send_object, &input_dataset, &barrier, &overall_latency_meter, batch_size]() -> hailo_status { auto res = send_loop(params, send_object.get(), input_dataset, barrier, overall_latency_meter, batch_size); if (HAILO_SUCCESS != res) { @@ -804,7 +802,7 @@ static Expected run_streaming(const std::vector>( + networks_threads_status[network_group_index].emplace_back(std::make_unique>(fmt::format("NG_INFER {}", network_group_index), [network_group_index, &configured_net_groups, &input_datasets, &output_buffers, ¶ms, &send_objects_per_network_group, &recv_objects_per_network_group, network_name, &progress_bar, &networks_results]() { return run_streaming_impl(configured_net_groups[network_group_index], input_datasets[network_group_index], @@ -1108,9 +1106,10 @@ Expected activate_and_run_single_device( } bool should_measure_temp = params.measure_temp; - TemperatureMeasurement temp_measure(device); + auto temp_measure = TemperatureMeasurement::create_shared(device); + CHECK_EXPECTED(temp_measure); if (should_measure_temp) { - auto status = temp_measure.start_measurement(); + auto status = temp_measure.value()->start_measurement(); CHECK_SUCCESS_AS_EXPECTED(status, "Failed to get chip's temperature"); } @@ -1136,8 +1135,8 @@ Expected activate_and_run_single_device( } if (should_measure_temp) { - temp_measure.stop_measurement(); - auto temp_measure_p = make_shared_nothrow(temp_measure.get_data()); + temp_measure.value()->stop_measurement(); + auto temp_measure_p = make_shared_nothrow(temp_measure.value()->get_data()); CHECK_NOT_NULL_AS_EXPECTED(temp_measure_p, HAILO_OUT_OF_HOST_MEMORY); auto status = inference_result.set_temp_measurement(device.get_dev_id(), std::move(temp_measure_p)); CHECK_SUCCESS_AS_EXPECTED(status); @@ -1257,11 +1256,11 @@ Expected activate_and_run_vdevice( std::map> temp_measurements; if (params.measure_temp) { for (auto &device : physical_devices) { - auto temp_measure = make_shared_nothrow(device); - CHECK_NOT_NULL_AS_EXPECTED(temp_measure, HAILO_OUT_OF_HOST_MEMORY); - auto status = temp_measure->start_measurement(); + auto temp_measure = TemperatureMeasurement::create_shared(device); + CHECK_EXPECTED(temp_measure); + auto status = temp_measure.value()->start_measurement(); CHECK_SUCCESS_AS_EXPECTED(status, "Failed starting temperature measurement on device {}", device.get().get_dev_id()); - temp_measurements.emplace(device.get().get_dev_id(), std::move(temp_measure)); + temp_measurements.emplace(device.get().get_dev_id(), temp_measure.release()); } } @@ -1298,7 +1297,7 @@ Expected activate_and_run_vdevice( if (params.measure_temp) { for(const auto &temp_measure_pair : temp_measurements) { temp_measure_pair.second->stop_measurement(); - auto temp_measure_p = make_shared_nothrow(temp_measure_pair.second->get_data()); + auto temp_measure_p = make_shared_nothrow(temp_measure_pair.second->get_data()); CHECK_NOT_NULL_AS_EXPECTED(temp_measure_p, HAILO_OUT_OF_HOST_MEMORY); auto status = inference_result.set_temp_measurement(temp_measure_pair.first, std::move(temp_measure_p)); CHECK_SUCCESS_AS_EXPECTED(status); @@ -1384,8 +1383,9 @@ Expected run_command_hef_vdevice(const inference_runner_params &par } if (params.runtime_data.collect_runtime_data) { - const auto runtime_data_output_path = format_runtime_data_output_path( - params.runtime_data.runtime_data_output_path, params.hef_path); + auto output_path = (1 == physical_devices.size()) ? params.runtime_data.runtime_data_output_path : + (std::string(device.get().get_dev_id()) + "_" + params.runtime_data.runtime_data_output_path); + const auto runtime_data_output_path = format_runtime_data_output_path(output_path, params.hef_path); DownloadActionListCommand::execute(device.get(), runtime_data_output_path, network_group_list.value(), params.hef_path); } diff --git a/hailort/hailortcli/run_command.hpp b/hailort/hailortcli/run_command.hpp index 5eaf834..e00199e 100644 --- a/hailort/hailortcli/run_command.hpp +++ b/hailort/hailortcli/run_command.hpp @@ -13,7 +13,7 @@ #include "hailortcli.hpp" #include "common.hpp" #include "power_measurement_command.hpp" -#include "temp_measurement.hpp" +#include "common/device_measurements.hpp" #include "CLI/CLI.hpp" #include "inference_result.hpp" diff --git a/hailort/hailortcli/temp_measurement.cpp b/hailort/hailortcli/temp_measurement.cpp deleted file mode 100644 index fbba350..0000000 --- a/hailort/hailortcli/temp_measurement.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file temp_measurement.cpp - * @brief Measure temperature of Hailo chip - **/ - -#include "temp_measurement.hpp" - -constexpr std::chrono::milliseconds DEFAULT_TEMPERATURE_MEASUREMENTS_INTERVAL(1000); - -static float32_t calc_avg(uint32_t old_samples_count, float32_t old_avg, uint32_t new_samples_count, float32_t new_value) -{ - float32_t old_samples = static_cast(old_samples_count); - float32_t new_samples = static_cast(new_samples_count); - float32_t total_samples_count = old_samples + new_samples; - return (((old_avg * old_samples) + (new_value * new_samples)) / total_samples_count); -} - -TemperatureMeasurement::TemperatureMeasurement(Device &device) : - m_device(device), - m_is_thread_running(false), - m_data() -{} - -TemperatureMeasurement::~TemperatureMeasurement() -{ - stop_measurement(); -} - -hailo_status TemperatureMeasurement::start_measurement() -{ - // Checking temperature sensor before starting thread - auto temp_info = m_device.get_chip_temperature(); - CHECK_EXPECTED_AS_STATUS(temp_info); - - m_is_thread_running = true; - m_thread = std::thread([this] () { - while (m_is_thread_running.load()) { - auto temp_info = m_device.get_chip_temperature(); - if (temp_info.status() != HAILO_SUCCESS) { - LOGGER__ERROR("Failed to get chip's temperature, status = {}", temp_info.status()); - m_is_thread_running = false; - break; - } - - TempMeasurementData new_data = {}; - auto old_data = m_data; - - float32_t ts_avg = ((temp_info->ts0_temperature + temp_info->ts1_temperature) / 2); - new_data.max_value = std::max(old_data.max_value, ts_avg); - new_data.min_value = (old_data.min_value == 0) ? ts_avg : std::min(old_data.min_value, ts_avg); - new_data.average_value = calc_avg(old_data.sample_count, old_data.average_value, temp_info->sample_count, ts_avg); - new_data.sample_count = old_data.sample_count + temp_info->sample_count; - - { - std::unique_lock lock(m_mutex); - m_data = new_data; - } - - std::this_thread::sleep_for(DEFAULT_TEMPERATURE_MEASUREMENTS_INTERVAL); - } - }); - - return HAILO_SUCCESS; -} - -void TemperatureMeasurement::stop_measurement() -{ - m_is_thread_running = false; - - if (m_thread.joinable()) { - m_thread.join(); - } -} - -const TempMeasurementData TemperatureMeasurement::get_data() -{ - std::unique_lock lock(m_mutex); - return m_data; -} \ No newline at end of file diff --git a/hailort/hailortcli/temp_measurement.hpp b/hailort/hailortcli/temp_measurement.hpp deleted file mode 100644 index b991645..0000000 --- a/hailort/hailortcli/temp_measurement.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file temp_measurement.hpp - * @brief Measure temperature of Hailo chip - **/ - -#ifndef _HAILO_TEMP_MEASUREMENT_HPP_ -#define _HAILO_TEMP_MEASUREMENT_HPP_ - -#include "hailortcli.hpp" -#include "command.hpp" -#include "hailo/hailort.h" -#include "hailo/device.hpp" -#include "CLI/CLI.hpp" - -#include - -struct TempMeasurementData { - float32_t average_value; - float32_t min_value; - float32_t max_value; - uint32_t sample_count; -}; - - -class TemperatureMeasurement final { -public: - TemperatureMeasurement(Device &device); - virtual ~TemperatureMeasurement(); - - hailo_status start_measurement(); - void stop_measurement(); - const TempMeasurementData get_data(); - -private: - void measure_temp(); - - Device &m_device; - std::thread m_thread; - std::atomic_bool m_is_thread_running; - std::mutex m_mutex; - TempMeasurementData m_data; -}; - -#endif /* _HAILO_TEMP_MEASUREMENT_HPP_ */ diff --git a/hailort/libhailort/CMakeLists.txt b/hailort/libhailort/CMakeLists.txt index 21a18f0..3993b61 100644 --- a/hailort/libhailort/CMakeLists.txt +++ b/hailort/libhailort/CMakeLists.txt @@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.0.0) # set(CMAKE_C_CLANG_TIDY "clang-tidy;-checks=*") set(HAILORT_MAJOR_VERSION 4) -set(HAILORT_MINOR_VERSION 12) -set(HAILORT_REVISION_VERSION 1) +set(HAILORT_MINOR_VERSION 13) +set(HAILORT_REVISION_VERSION 0) # Add the cmake folder so the modules there are found set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) @@ -48,6 +48,7 @@ add_library(readerwriterqueue INTERFACE) target_include_directories(readerwriterqueue INTERFACE ${HAILO_EXTERNAL_DIR}/readerwriterqueue) add_subdirectory(src) +set(NET_FLOW_INFRA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/infra/net_flow") if(HAILO_BUILD_EXAMPLES) add_subdirectory(examples) diff --git a/hailort/libhailort/bindings/gstreamer/CMakeLists.txt b/hailort/libhailort/bindings/gstreamer/CMakeLists.txt index 4a14f32..e0c06c9 100644 --- a/hailort/libhailort/bindings/gstreamer/CMakeLists.txt +++ b/hailort/libhailort/bindings/gstreamer/CMakeLists.txt @@ -8,7 +8,7 @@ if(NOT CMAKE_HOST_UNIX) message(FATAL_ERROR "Only unix hosts are supported, stopping build") endif() -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) # GST_PLUGIN_DEFINE needs PACKAGE to be defined set(GST_HAILO_PACKAGE_NAME "hailo") @@ -32,6 +32,8 @@ add_library(gsthailo SHARED gst-hailo/metadata/tensor_meta.cpp gst-hailo/hailo_events/hailo_events.cpp) +set_property(TARGET gsthailo PROPERTY CXX_STANDARD 14) + set_target_properties(gsthailo PROPERTIES PUBLIC_HEADER "gst-hailo/metadata/tensor_meta.hpp" ) diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp index 592125b..73d126e 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp @@ -48,7 +48,7 @@ using namespace hailort; #define DEFAULT_VDEVICE_KEY (0) #define MIN_VALID_VDEVICE_KEY (1) -#define HAILO_SUPPORTED_FORMATS "{ RGB, RGBA, YUY2, NV12, NV21 }" +#define HAILO_SUPPORTED_FORMATS "{ RGB, RGBA, YUY2, NV12, NV21, I420 }" #define HAILO_VIDEO_CAPS GST_VIDEO_CAPS_MAKE(HAILO_SUPPORTED_FORMATS) #define HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS (0) diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailodevicestats.hpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailodevicestats.hpp index 90c244d..204600f 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailodevicestats.hpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailodevicestats.hpp @@ -25,6 +25,7 @@ #include #include +#include G_BEGIN_DECLS diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp index c4c0ea8..7b4f755 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp @@ -52,6 +52,29 @@ gst_scheduling_algorithm_get_type (void) return scheduling_algorithm_type; } +#define GST_TYPE_HAILO_FORMAT_TYPE (gst_hailo_format_type_get_type ()) +static GType +gst_hailo_format_type_get_type (void) +{ + static GType format_type_enum = 0; + + /* Tightly coupled to hailo_format_type_t */ + + if (!format_type_enum) { + static GEnumValue format_types[] = { + { HAILO_FORMAT_TYPE_AUTO, "auto", "HAILO_FORMAT_TYPE_AUTO"}, + { HAILO_FORMAT_TYPE_UINT8, "uint8", "HAILO_FORMAT_TYPE_UINT8"}, + { HAILO_FORMAT_TYPE_UINT16, "uint16", "HAILO_FORMAT_TYPE_UINT16"}, + { HAILO_FORMAT_TYPE_FLOAT32, "float32", "HAILO_FORMAT_TYPE_FLOAT32"}, + { HAILO_FORMAT_TYPE_MAX_ENUM, NULL, NULL }, + }; + + format_type_enum = g_enum_register_static ("GstHailoFormatTypes", format_types); + } + + return format_type_enum; +} + constexpr std::chrono::milliseconds WAIT_FOR_FLUSH_TIMEOUT_MS(1000); static void gst_hailonet_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec); @@ -80,6 +103,10 @@ enum PROP_SCHEDULER_TIMEOUT_MS, PROP_SCHEDULER_THRESHOLD, PROP_MULTI_PROCESS_SERVICE, + PROP_INPUT_QUANTIZED, + PROP_OUTPUT_QUANTIZED, + PROP_INPUT_FORMAT_TYPE, + PROP_OUTPUT_FORMAT_TYPE, }; G_DEFINE_TYPE(GstHailoNet, gst_hailonet, GST_TYPE_BIN); @@ -150,8 +177,7 @@ static void gst_hailonet_class_init(GstHailoNetClass *klass) g_param_spec_enum("scheduling-algorithm", "Scheduling policy for automatic network group switching", "Controls the Model Scheduler algorithm of HailoRT. " "Gets values from the enum GstHailoSchedulingAlgorithms. " "Using Model Scheduler algorithm different than HAILO_SCHEDULING_ALGORITHM_NONE, excludes the property 'is-active'. " - "When using the same VDevice across multiple hailonets, all should have the same 'scheduling-algorithm'. " - "To run with more than one device, set env variable 'HAILO_ENABLE_MULTI_DEVICE_SCHEDULER' to 1.", + "When using the same VDevice across multiple hailonets, all should have the same 'scheduling-algorithm'. ", GST_TYPE_SCHEDULING_ALGORITHM, HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); g_object_class_install_property(gobject_class, PROP_SCHEDULER_TIMEOUT_MS, @@ -165,6 +191,24 @@ static void gst_hailonet_class_init(GstHailoNetClass *klass) g_param_spec_boolean("multi-process-service", "Should run over HailoRT service", "Controls wether to run HailoRT over its service. " "To use this property, the service should be active and scheduling-algorithm should be set. Defaults to false.", HAILO_DEFAULT_MULTI_PROCESS_SERVICE, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_INPUT_QUANTIZED, + g_param_spec_boolean("input-quantized", "Is the input quantized or not", "Passing `true` under the argument means that the input data sent to the stream is quantized to begin with." + "This will result in an input stream that doesn't quantize the input data. Passing `false` under the argument, will lead to input data being quantized.", + true, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_OUTPUT_QUANTIZED, + g_param_spec_boolean("output-quantized", "Should the output be quantized or de-quantized","Passing `true` under the argument means that the output data received from the stream is to remain quantized" + "(such as it is upon exiting the device). This will result in an output stream that doesn't de-quantize the output data. Passing `false` under the argument will lead to output data being de-quantized.", + true, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_INPUT_FORMAT_TYPE, + g_param_spec_enum("input-format-type", "Input format type", "Input format type(auto, float32, uint16, uint8). Default value is auto." + "Gets values from the enum GstHailoFormatType. ", + GST_TYPE_HAILO_FORMAT_TYPE, HAILO_FORMAT_TYPE_AUTO, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_OUTPUT_FORMAT_TYPE, + g_param_spec_enum("output-format-type", "Output format type", "Output format type(auto, float32, uint16, uint8). Default value is auto." + "Gets values from the enum GstHailoFormatType. ", + GST_TYPE_HAILO_FORMAT_TYPE, HAILO_FORMAT_TYPE_AUTO, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); // See information about the "flush" signal in the element description g_signal_new( "flush", @@ -457,6 +501,34 @@ void HailoNetImpl::set_property(GObject *object, guint property_id, const GValue } m_props.m_multi_process_service = g_value_get_boolean(value); break; + case PROP_INPUT_QUANTIZED: + if (m_was_configured) { + g_warning("The network was already configured so changing the quantized flag will not take place!"); + break; + } + m_props.m_input_quantized = g_value_get_boolean(value); + break; + case PROP_OUTPUT_QUANTIZED: + if (m_was_configured) { + g_warning("The network was already configured so changing the quantized flag will not take place!"); + break; + } + m_props.m_output_quantized = g_value_get_boolean(value); + break; + case PROP_INPUT_FORMAT_TYPE: + if (m_was_configured) { + g_warning("The network was already configured so changing the format type will not take place!"); + break; + } + m_props.m_input_format_type = static_cast(g_value_get_enum(value)); + break; + case PROP_OUTPUT_FORMAT_TYPE: + if (m_was_configured) { + g_warning("The network was already configured so changing the format type will not take place!"); + break; + } + m_props.m_output_format_type = static_cast(g_value_get_enum(value)); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -527,6 +599,18 @@ void HailoNetImpl::get_property(GObject *object, guint property_id, GValue *valu case PROP_MULTI_PROCESS_SERVICE: g_value_set_boolean(value, m_props.m_multi_process_service.get()); break; + case PROP_INPUT_QUANTIZED: + g_value_set_boolean(value, m_props.m_input_quantized.get()); + break; + case PROP_OUTPUT_QUANTIZED: + g_value_set_boolean(value, m_props.m_output_quantized.get()); + break; + case PROP_INPUT_FORMAT_TYPE: + g_value_set_enum(value, m_props.m_input_format_type.get()); + break; + case PROP_OUTPUT_FORMAT_TYPE: + g_value_set_enum(value, m_props.m_output_format_type.get()); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -613,7 +697,8 @@ hailo_status HailoNetImpl::configure_network_group() GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting scheduler threshold failed, status = %d", status); } - auto vstreams = m_net_group_handle->create_vstreams(m_props.m_network_name.get(), m_props.m_scheduling_algorithm.get(), m_output_formats); + auto vstreams = m_net_group_handle->create_vstreams(m_props.m_network_name.get(), m_props.m_scheduling_algorithm.get(), m_output_formats, static_cast(m_props.m_input_quantized.get()), + static_cast(m_props.m_output_quantized.get()), m_props.m_input_format_type.get(), m_props.m_output_format_type.get()); GST_CHECK_EXPECTED_AS_STATUS(vstreams, m_element, RESOURCE, "Creating vstreams failed, status = %d", status); GST_HAILOSEND(m_hailosend)->impl->set_input_vstreams(std::move(vstreams->first)); @@ -728,7 +813,7 @@ gboolean HailoNetImpl::src_pad_event(GstEvent *event) auto parsed_event = HailoSetOutputFormatEvent::parse(event); if (HAILO_SUCCESS != parsed_event.status()) { - return FALSE; + return FALSE; } m_output_formats = std::move(parsed_event->formats); diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp index ccb4d4b..0c3e6f8 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp @@ -54,7 +54,9 @@ public: HailoNetProperties() : m_device_id(nullptr), m_hef_path(nullptr), m_network_name(nullptr), m_batch_size(HAILO_DEFAULT_BATCH_SIZE), m_is_active(false), m_device_count(0), m_vdevice_key(DEFAULT_VDEVICE_KEY), m_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN), m_scheduler_timeout_ms(HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS), m_scheduler_threshold(HAILO_DEFAULT_SCHEDULER_THRESHOLD), - m_multi_process_service(HAILO_DEFAULT_MULTI_PROCESS_SERVICE) + m_multi_process_service(HAILO_DEFAULT_MULTI_PROCESS_SERVICE), m_input_quantized(true), m_output_quantized(true), m_input_format_type(HAILO_FORMAT_TYPE_AUTO), + m_output_format_type(HAILO_FORMAT_TYPE_AUTO) + {} HailoElemProperty m_device_id; @@ -68,6 +70,10 @@ public: HailoElemProperty m_scheduler_timeout_ms; HailoElemProperty m_scheduler_threshold; HailoElemProperty m_multi_process_service; + HailoElemProperty m_input_quantized; + HailoElemProperty m_output_quantized; + HailoElemProperty m_input_format_type; + HailoElemProperty m_output_format_type; }; class HailoNetImpl final diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailorecv.cpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailorecv.cpp index 5a0624b..e7955da 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailorecv.cpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailorecv.cpp @@ -281,7 +281,7 @@ hailo_status HailoRecvImpl::set_output_vstreams(std::vector &&out for (auto &out_vstream : m_output_vstreams) { GstHailoBufferPool *hailo_pool = GST_HAILO_BUFFER_POOL(g_object_new(GST_TYPE_HAILO_BUFFER_POOL, NULL)); gst_object_ref_sink(hailo_pool); - memcpy(hailo_pool->vstream_name, out_vstream.name().c_str(), sizeof(hailo_pool->vstream_name)); + strncpy(hailo_pool->vstream_name, out_vstream.name().c_str(), out_vstream.name().length() + 1); hailo_pool->element_name = GST_ELEMENT_NAME(GST_ELEMENT_PARENT(m_element)); GstBufferPool *pool = GST_BUFFER_POOL(hailo_pool); diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailosend.cpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailosend.cpp index e6a85d2..184886c 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailosend.cpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailosend.cpp @@ -33,6 +33,7 @@ GST_DEBUG_CATEGORY_STATIC(gst_hailosend_debug_category); #define YUY2_FEATURES_SIZE (2) #define NV12_FEATURES_SIZE (3) #define NV21_FEATURES_SIZE (3) +#define I420_FEATURES_SIZE (3) static void gst_hailosend_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec); static void gst_hailosend_get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec); @@ -64,7 +65,7 @@ static void gst_hailosend_class_init(GstHailoSendClass *klass) gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, gst_caps_from_string(HAILO_VIDEO_CAPS))); gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass), - "hailosend element", "Hailo/Filter/Video", "Send RGB/RGBA/YUY2 video to HailoRT", PLUGIN_AUTHOR); + "hailosend element", "Hailo/Filter/Video", "Send RGB/RGBA/YUY2/NV12/NV21/I420 video to HailoRT", PLUGIN_AUTHOR); element_class->change_state = GST_DEBUG_FUNCPTR(gst_hailosend_change_state); @@ -238,6 +239,12 @@ GstCaps *HailoSendImpl::get_caps(GstBaseTransform */*trans*/, GstPadDirection /* "Features of input vstream %s is not %d for NV21 format! (features=%d)", m_input_vstream_infos[0].name, NV21_FEATURES_SIZE, m_input_vstream_infos[0].shape.features); break; + case HAILO_FORMAT_ORDER_I420: + format = "I420"; + GST_CHECK(I420_FEATURES_SIZE == m_input_vstream_infos[0].shape.features, NULL, m_element, STREAM, + "Features of input vstream %s is not %d for I420 format! (features=%d)", m_input_vstream_infos[0].name, I420_FEATURES_SIZE, + m_input_vstream_infos[0].shape.features); + break; default: GST_ELEMENT_ERROR(m_element, RESOURCE, FAILED, ("Input VStream %s has an unsupported format order! order = %d", m_input_vstream_infos[0].name, m_input_vstream_infos[0].format.order), (NULL)); diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.cpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.cpp index 7a04149..fb0aecd 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.cpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.cpp @@ -17,11 +17,15 @@ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ + +#include "hailo/hailort_common.hpp" + #include "network_group_handle.hpp" #include #include + std::unordered_set> NetworkGroupHandle::m_vdevices; NetworkGroupConfigManager NetworkGroupHandle::m_net_group_config_manager; NetworkGroupActivationManager NetworkGroupHandle::m_net_group_activation_manager; @@ -177,7 +181,8 @@ hailo_status NetworkGroupHandle::set_scheduler_threshold(const char *network_nam } Expected, std::vector>> NetworkGroupHandle::create_vstreams(const char *network_name, - hailo_scheduling_algorithm_t scheduling_algorithm, const std::vector &output_formats) + hailo_scheduling_algorithm_t scheduling_algorithm, const std::vector &output_formats, bool input_quantized, + bool output_quantized, hailo_format_type_t input_format_type, hailo_format_type_t output_format_type) { GST_CHECK(nullptr != network_name, make_unexpected(HAILO_INVALID_ARGUMENT), m_element, RESOURCE, "Got nullptr in network name!"); @@ -191,11 +196,10 @@ Expected, std::vector>> Netwo auto expected_input_vstream_infos = hef()->get_input_vstream_infos(network_name); GST_CHECK_EXPECTED(expected_input_vstream_infos, m_element, RESOURCE, "Failed getting input vstream infos, status = %d", expected_input_vstream_infos.status()); - auto expected_input_params_map = m_cng->make_input_vstream_params(true, HAILO_FORMAT_TYPE_AUTO, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS, + auto expected_input_params_map = m_cng->make_input_vstream_params(input_quantized, input_format_type, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, m_network_name); GST_CHECK_EXPECTED(expected_input_params_map, m_element, RESOURCE, "Failed making input vstream params, status = %d", expected_input_params_map.status()); - // In RGB formats, Gstreamer is padding each row to 4. auto &&input_params_map = expected_input_params_map.release(); auto &&input_infos = expected_input_vstream_infos.release(); @@ -215,7 +219,7 @@ Expected, std::vector>> Netwo GST_CHECK(1 == input_vstreams->size(), make_unexpected(HAILO_INVALID_OPERATION), m_element, RESOURCE, "hailosend element supports only HEFs with one input for now!"); - auto output_params_map = m_cng->make_output_vstream_params(true, HAILO_FORMAT_TYPE_AUTO, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS, + auto output_params_map = m_cng->make_output_vstream_params(output_quantized, output_format_type, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, m_network_name); GST_CHECK_EXPECTED(output_params_map, m_element, RESOURCE, "Failed making output vstream params, status = %d", output_params_map.status()); @@ -244,11 +248,7 @@ Expected, std::vector>> Netwo Expected NetworkGroupHandle::get_configure_params(Hef &hef, const VDevice &vdevice, const char *net_group_name, uint16_t batch_size) { - auto stream_interface = vdevice.get_default_streams_interface(); - GST_CHECK_EXPECTED(stream_interface, m_element, RESOURCE, - "Failed default stream interface configure params, status = %d", stream_interface.status()); - - auto params = hef.create_configure_params(*stream_interface, net_group_name); + auto params = vdevice.create_configure_params(hef, net_group_name); GST_CHECK_EXPECTED(params, m_element, RESOURCE, "Failed creating configure params, status = %d", params.status()); params->batch_size = batch_size; diff --git a/hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.hpp b/hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.hpp index 246118f..665d0ac 100644 --- a/hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.hpp +++ b/hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.hpp @@ -83,7 +83,8 @@ public: bool multi_process_service, const char *hef_path); hailo_status configure_network_group(const char *net_group_name, hailo_scheduling_algorithm_t scheduling_algorithm, uint16_t batch_size); Expected, std::vector>> create_vstreams(const char *network_name, - hailo_scheduling_algorithm_t scheduling_algorithm, const std::vector &output_formats); + hailo_scheduling_algorithm_t scheduling_algorithm, const std::vector &output_formats, bool input_quantized, + bool output_quantized, hailo_format_type_t input_format_type, hailo_format_type_t output_format_type); hailo_status activate_network_group(); Expected remove_network_group(); diff --git a/hailort/libhailort/bindings/python/examples/hef_infer_pipeline_vstream.py b/hailort/libhailort/bindings/python/examples/hef_infer_pipeline_vstream.py deleted file mode 100644 index 2eea36c..0000000 --- a/hailort/libhailort/bindings/python/examples/hef_infer_pipeline_vstream.py +++ /dev/null @@ -1,34 +0,0 @@ -from hailo_platform import (HEF, VDevice, ConfigureParams, InferVStreams, InputVStreamParams, - OutputVStreamParams, FormatType) -from hailo_platform.pyhailort.pyhailort import HailoStreamInterface -import numpy as np -import argparse - -def parse_args(): - parser = argparse.ArgumentParser(description='Streaming API example') - parser.add_argument('hef_path', type=str, help='Path of the HEF to run') - parser.add_argument('-n', '--num-frames', type=int, default=10, help='Number of frames to send') - return parser.parse_args() - -def main(): - args = parse_args() - with VDevice() as target: - hef = HEF(args.hef_path) - configure_params = ConfigureParams.create_from_hef(hef, interface=HailoStreamInterface.PCIe) - network_groups = target.configure(hef, configure_params) - network_group = network_groups[0] - network_group_params = network_group.create_params() - input_vstreams_params = InputVStreamParams.make(network_group, quantized=False, format_type=FormatType.FLOAT32) - output_vstreams_params = OutputVStreamParams.make(network_group, quantized=True, format_type=FormatType.AUTO) - with InferVStreams(network_group, input_vstreams_params, output_vstreams_params) as infer_pipeline: - input_names_to_shape = {vstream_info.name: vstream_info.shape for vstream_info in hef.get_input_vstream_infos()} - input_data = {name : 1 + np.ndarray([args.num_frames] + list(shape), dtype=np.float32) for name, shape in input_names_to_shape.items()} - with network_group.activate(network_group_params): - _ = infer_pipeline.infer(input_data) - fps = args.num_frames / infer_pipeline.get_hw_time() - - print('Inference ran successfully') - print(f'FPS: {fps}') - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/hailort/libhailort/bindings/python/examples/hef_infer_virtual_stream.py b/hailort/libhailort/bindings/python/examples/hef_infer_virtual_stream.py deleted file mode 100644 index f9dd2b6..0000000 --- a/hailort/libhailort/bindings/python/examples/hef_infer_virtual_stream.py +++ /dev/null @@ -1,69 +0,0 @@ -import argparse -import time -import numpy as np - -from multiprocessing import Process -from hailo_platform import (HEF, VDevice, HailoStreamInterface, ConfigureParams, InputVStreamParams, InputVStreams, - OutputVStreamParams, OutputVStreams) - -def send(configured_network, num_frames): - vstreams_params = InputVStreamParams.make(configured_network) - configured_network.wait_for_activation(1000) - with InputVStreams(configured_network, vstreams_params) as vstreams: - vstream_to_buffer = {vstream: np.ndarray([1] + list(vstream.shape), dtype=vstream.dtype) for vstream in vstreams} - for _ in range(num_frames): - for vstream, buff in vstream_to_buffer.items(): - vstream.send(buff) - # Flushing is not mandatory here - for vstream in vstreams: - vstream.flush() - -def recv(configured_network, vstreams_params, num_frames): - configured_network.wait_for_activation(1000) - with OutputVStreams(configured_network, vstreams_params) as vstreams: - for _ in range(num_frames): - for vstream in vstreams: - _ = vstream.recv() - -def recv_all(configured_network, num_frames): - vstreams_params_groups = OutputVStreamParams.make_groups(configured_network) - recv_procs = [] - for vstreams_params in vstreams_params_groups: - proc = Process(target=recv, args=(configured_network, vstreams_params, num_frames)) - proc.start() - recv_procs.append(proc) - - for proc in recv_procs: - proc.join() - -def parse_args(): - parser = argparse.ArgumentParser(description='vStream API example') - parser.add_argument('hef_path', type=str, help='Path of the HEF to run') - parser.add_argument('-n', '--num-frames', type=int, default=1000, help='Number of frames to send') - return parser.parse_args() - -def main(): - args = parse_args() - hef = HEF(args.hef_path) - - with VDevice() as device: - configure_params = ConfigureParams.create_from_hef(hef, interface=HailoStreamInterface.PCIe) - network_group = device.configure(hef, configure_params)[0] - network_group_params = network_group.create_params() - send_process = Process(target=send, args=(network_group, args.num_frames)) - recv_process = Process(target=recv_all, args=(network_group, args.num_frames)) - - time_before = time.time() - recv_process.start() - send_process.start() - with network_group.activate(network_group_params): - send_process.join() - recv_process.join() - - fps = args.num_frames / (time.time() - time_before) - - print('Inference ran successfully') - print(f'FPS: {fps}') - -if __name__ == '__main__': - main() diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py b/hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py index b307e7d..18d66a1 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py @@ -26,7 +26,7 @@ from hailo_platform.pyhailort.pyhailort import (HEF, ConfigureParams, InputVStreams, OutputVStreams, InferVStreams, HailoStreamDirection, HailoFormatFlags, HailoCpuId, Device, VDevice, DvmTypes, PowerMeasurementTypes, SamplingPeriod, AveragingFactor, MeasurementBufferIndex, - HailoRTException, YOLOv5PostProcessingOp) + HailoRTException, YOLOv5PostProcessOp, HailoSchedulingAlgorithm) def _verify_pyhailort_lib_exists(): python_version = "".join(str(i) for i in sys.version_info[:2]) @@ -62,4 +62,4 @@ __all__ = ['EthernetDevice', 'DvmTypes', 'PowerMeasurementTypes', 'MipiIspImageInOrder', 'MipiIspImageOutDataType', 'join_drivers_path', 'IspLightFrequency', 'HailoPowerMode', 'Endianness', 'HailoStreamInterface', 'InputVStreamParams', 'OutputVStreamParams', 'InputVStreams', 'OutputVStreams', 'InferVStreams', 'HailoStreamDirection', 'HailoFormatFlags', 'HailoCpuId', - 'Device', 'VDevice', 'HailoRTException', 'YOLOv5PostProcessingOp'] + 'Device', 'VDevice', 'HailoRTException', 'YOLOv5PostProcessOp', 'HailoSchedulingAlgorithm'] diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/control_object.py b/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/control_object.py index 879f817..a1f5095 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/control_object.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/control_object.py @@ -9,7 +9,6 @@ from hailo_platform.pyhailort.pyhailort import (Control, InternalPcieDevice, Exc import hailo_platform.pyhailort._pyhailort as _pyhailort - class ControlObjectException(Exception): """Raised on illegal ContolObject operation.""" pass @@ -38,6 +37,9 @@ class UdpHcpControl(HcpControl): ignore_socket_errors (bool, optional): Ignore socket error (might be usefull for debugging). """ # In the C API we define the total amount of attempts, instead of the amount of retries. + + # TODO: HRT-9987 - Add this deprecation warning + # default_logger().warning("UdpHcpControl is deprecated! Please Use Control object") max_number_of_attempts = retries + 1 response_timeout_milliseconds = int(response_timeout_seconds * 1000) if device is None: @@ -55,6 +57,8 @@ class PcieHcpControl(HcpControl): def __init__(self, device=None, device_info=None): """Initializes a new HailoPcieController object.""" + # TODO: HRT-9987 - Add this deprecation warning + # default_logger().warning("PcieHcpControl is deprecated! Please Use Control object") if device_info is None: device_info = InternalPcieDevice.scan_devices()[0] diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/hw_object.py b/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/hw_object.py index 5871dca..32483c5 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/hw_object.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/hw_object.py @@ -27,7 +27,8 @@ class HailoHWObjectException(Exception): class HailoHWObject(object): - """Abstract Hailo hardware device representation.""" + # TODO: HRT-9987 - Add (deprecated) to this doc + """Abstract Hailo hardware device representation""" NAME = InferenceTargets.UNINITIALIZED IS_HARDWARE = True @@ -43,6 +44,9 @@ class HailoHWObject(object): self._is_device_used = False self._hef_loaded = False + # TODO: HRT-9987 - Add this deprecation warning + # self._logger.warning("HailoHWObject is deprecated! Please use VDevice/Device object.") + # TODO: HRT-6310 Remove this. def __eq__(self, other): return type(self).NAME == other @@ -50,14 +54,16 @@ class HailoHWObject(object): @property def name(self): """str: The name of this target. Valid values are defined by :class:`~hailo_platform.pyhailort.hw_object.InferenceTargets`""" - # self._logger.warning("HailoHWObject name property is deprecated! Please use VDevice object with device_id.") + # TODO: HRT-9987 - Add this deprecation warning + # self._logger.warning("HailoHWObject name property is deprecated! Please use VDevice/Device object with device_id.") return type(self).NAME @property def is_hardware(self): """bool: Indicates this target runs on a physical hardware device.""" # TODO: SDK should implement in Target - # self._logger.warning("HailoHWObject is_hardware property is deprecated! Please use VDevice object, or derive from it.") + # TODO: HRT-9987 - Add this deprecation warning + # self._logger.warning("HailoHWObject is_hardware property is deprecated! Please use VDevice/Device object, or derive from it.") return type(self).IS_HARDWARE @property @@ -74,6 +80,7 @@ class HailoHWObject(object): Returns: list of str: Sorted list of the output layer names. """ + # TODO: HRT-9987 - Add this deprecation warning # self._logger.warning("HailoHWObject sorted_output_layer_names property is deprecated! Please use ConfiguredNetwork get_sorted_output_names.") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to sorted_output_layer_names is only allowed when there is a single loaded network group") @@ -81,8 +88,9 @@ class HailoHWObject(object): @contextmanager def use_device(self, *args, **kwargs): - # self._logger.warning("HailoHWObject use_device context manager is deprecated! Please use VDevice object.") - """A context manager that wraps the usage of the device (deprecated).""" + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs + # self._logger.warning("HailoHWObject use_device context manager is deprecated! Please use VDevice/Device object.") + """A context manager that wraps the usage of the device.""" self._is_device_used = True yield self._is_device_used = False @@ -93,6 +101,7 @@ class HailoHWObject(object): Returns: dict: Keys are device output names and values are lists of layers' names. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoHWObject get_output_device_layer_to_original_layer_map function is deprecated!") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to layer names is only allowed when there is a single loaded network group") @@ -105,6 +114,7 @@ class HailoHWObject(object): Returns: dict: Keys are the names of the layers and values are device outputs names. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoHWObject get_original_layer_to_device_layer_map function is deprecated!") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to layer names is only allowed when there is a single loaded network group") @@ -114,30 +124,35 @@ class HailoHWObject(object): @property def device_input_layers(self): """Get a list of the names of the device's inputs.""" + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoHWObject device_input_layers function is deprecated! Please use ConfiguredNetwork object.") return [layer.name for layer in self.get_input_stream_infos()] @property def device_output_layers(self): """Get a list of the names of the device's outputs.""" + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoHWObject device_output_layers function is deprecated! Please use ConfiguredNetwork object.") return [layer.name for layer in self.get_output_stream_infos()] def hef_loaded(self): """Return True if this object has loaded the model HEF to the hardware device.""" # TODO: SDK should implement in Target - # self._logger.warning("HailoHWObject hef_loaded function is deprecated! Please use VDevice object, or derive from it.") + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs + # self._logger.warning("HailoHWObject hef_loaded function is deprecated! Please use VDevice/Device object, or derive from it.") return self._hef_loaded def outputs_count(self): """Return the amount of output tensors that are returned from the hardware device for every input image. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoHWObject outputs_count function is deprecated! Please use ConfiguredNetwork object.") return len(self.get_output_vstream_infos()) def _clear_shapes(self): # TODO: SDK should implement in Target + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoHWObject _clear_shapes function is deprecated! Please use ConfiguredNetwork object.") self._hw_consts = None @@ -148,6 +163,7 @@ class HailoHWObject(object): Returns: str: Model name. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoHWObject model_name property is deprecated! Please use ConfiguredNetwork object.") if len(self._loaded_network_groups) == 1: return self._loaded_network_groups[0].name @@ -160,6 +176,7 @@ class HailoHWObject(object): Returns: Tuple of output shapes, sorted by the output names. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoHWObject get_output_shapes function is deprecated! Please use ConfiguredNetwork object.") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Calling get_output_shapes is only allowed when there is a single loaded network group") @@ -167,6 +184,7 @@ class HailoHWObject(object): class HailoChipObject(HailoHWObject): + # TODO: HRT-9987 - Add (deprecated) to this docs """Hailo hardware device representation""" def __init__(self): @@ -195,6 +213,7 @@ class HailoChipObject(HailoHWObject): Returns: dict of :obj:'numpy.dtype': where the key is model input_layer name, and the value is dtype as the device expect to get for this input. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_all_input_layers_dtype function is deprecated! Please use ConfiguredNetwork object.") return {stream.name: HailoRTTransformUtils.get_dtype(stream.data_bytes) for stream in self.get_input_stream_infos()} @@ -208,6 +227,7 @@ class HailoChipObject(HailoHWObject): If there is exactly one configured network group, returns a list of :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with all the information objects of all input vstreams """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_input_vstream_infos function is deprecated! Please use ConfiguredNetwork object.") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to network vstream info is only allowed when there is a single loaded network group") @@ -223,6 +243,7 @@ class HailoChipObject(HailoHWObject): If there is exactly one configured network group, returns a list of :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with all the information objects of all output vstreams """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_output_vstream_infos function is deprecated! Please use ConfiguredNetwork object.") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to network vstream info is only allowed when there is a single loaded network group") @@ -238,6 +259,7 @@ class HailoChipObject(HailoHWObject): If there is exactly one configured network group, returns a list of :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with all the information objects of all input and output vstreams """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_all_vstream_infos function is deprecated! Please use ConfiguredNetwork object.") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to network vstream info is only allowed when there is a single loaded network group") @@ -254,6 +276,7 @@ class HailoChipObject(HailoHWObject): :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with information objects of all input low-level streams. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_input_stream_infos function is deprecated! Please use ConfiguredNetwork object.") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to network stream info is only allowed when there is a single loaded network group") @@ -270,6 +293,7 @@ class HailoChipObject(HailoHWObject): :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with information objects of all output low-level streams. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_output_stream_infos function is deprecated! Please use ConfiguredNetwork object.") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to network stream info is only allowed when there is a single loaded network group") @@ -285,6 +309,7 @@ class HailoChipObject(HailoHWObject): If there is exactly one configured network group, returns a list of :obj:`hailo_platform.pyhailort._pyhailort.StreamInfo`: with all the information objects of all input and output streams """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_all_stream_infos function is deprecated! Please use ConfiguredNetwork object.") if len(self._loaded_network_groups) != 1: raise HailoHWObjectException("Access to network stream info is only allowed when there is a single loaded network group") @@ -328,6 +353,7 @@ class HailoChipObject(HailoHWObject): Returns: Tuple of integers representing the input_shape. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_input_shape function is deprecated! Please use ConfiguredNetwork object.") if name is None: name = self.get_input_vstream_infos()[0].name @@ -348,6 +374,7 @@ class HailoChipObject(HailoHWObject): Returns: int: The index of the layer name in the output list. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # self._logger.warning("HailoChipObject get_index_from_name function is deprecated! Please use ConfiguredNetwork object.") try: return self.sorted_output_layer_names.index(name) @@ -371,7 +398,8 @@ class HailoChipObject(HailoHWObject): class EthernetDevice(HailoChipObject): - """Represents any Hailo hardware device that supports UDP control and dataflow.""" + # TODO: HRT-9987 - Add (deprecated) to this docs + """Represents any Hailo hardware device that supports UDP control and dataflow""" NAME = InferenceTargets.UDP_CONTROLLER @@ -414,6 +442,7 @@ class EthernetDevice(HailoChipObject): Returns: list of str: IPs of scanned devices. """ + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs # default_logger().warning("EthernetDevice scan_devices method is deprecated! Please use scan() of Device object.") udp_scanner = HailoUdpScan() return udp_scanner.scan_devices(interface_name, timeout_seconds=timeout_seconds) @@ -435,12 +464,14 @@ class EthernetDevice(HailoChipObject): @property def remote_ip(self): """Return the IP of the remote device.""" - # self._logger.warning("EthernetDevice remote_ip method is deprecated! Please use Device object.") + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs + # self._logger.warning("EthernetDevice remote_ip method is deprecated! Please use VDevice/Device object.") return self._remote_ip class PcieDevice(HailoChipObject): - """Hailo PCIe production device representation.""" + # TODO: HRT-9987 - Add (deprecated) to this docs + """Hailo PCIe production device representation""" NAME = InferenceTargets.PCIE_CONTROLLER @@ -455,7 +486,8 @@ class PcieDevice(HailoChipObject): :func:`PcieDevice.scan_devices` to get list of all available devices. """ super(PcieDevice, self).__init__() - # self._logger.warning("PcieDevice deprecated! Please use VDevice object.") + # TODO: HRT-9987 - Add this deprecation warning + # self._logger.warning("PcieDevice is deprecated! Please use VDevice/Device object.") gc.collect() # PcieDevice __del__ function tries to release self._device. @@ -479,7 +511,8 @@ class PcieDevice(HailoChipObject): Returns: list of :obj:`hailo_platform.pyhailort.pyhailort.PcieDeviceInfo` """ - # default_logger().warning("PcieDevice scan_devices method is deprecated! Please use scan() of Device object.") + # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs + # default_logger().warning("PcieDevice scan_devices method is deprecated! Please use Device object.") return InternalPcieDevice.scan_devices() def _open_device(self, device_info): diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py b/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py index 629291b..6b85acf 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py @@ -48,6 +48,8 @@ class HailoSocket(object): MIN_UDP_PADDED_PAYLOAD_SIZE = HailoSocketDefs.MIN_UDP_PADDED_PAYLOAD_SIZE() MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP = HailoSocketDefs.MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP() +class HailoSchedulingAlgorithm(_pyhailort.SchedulingAlgorithm): + pass class HailoRTException(Exception): pass @@ -104,46 +106,47 @@ class ExceptionWrapper(object): def __exit__(self, exception_type, value, traceback): if value is not None: if exception_type is _pyhailort.HailoRTStatusException: - self._raise_indicative_status_exception(int(value.args[0])) + self._raise_indicative_status_exception(value) else: raise - def _raise_indicative_status_exception(self, error_code): + def _raise_indicative_status_exception(self, libhailort_exception): + error_code = int(libhailort_exception.args[0]) string_error_code = get_status_message(error_code) if string_error_code == "HAILO_ETH_RECV_FAILURE": - raise UdpRecvError("Failed to receive data") + raise UdpRecvError("Failed to receive data") from libhailort_exception if string_error_code == "HAILO_UNSUPPORTED_CONTROL_PROTOCOL_VERSION": - raise InvalidProtocolVersionException("HailoRT has failed because an invalid protocol version was received from device") + raise InvalidProtocolVersionException("HailoRT has failed because an invalid protocol version was received from device") from libhailort_exception if string_error_code == "HAILO_FW_CONTROL_FAILURE": - raise HailoRTFirmwareControlFailedException("libhailort control operation failed") + raise HailoRTFirmwareControlFailedException("libhailort control operation failed") from libhailort_exception if string_error_code == "HAILO_UNSUPPORTED_OPCODE": - raise HailoRTUnsupportedOpcodeException("HailoRT has failed because an unsupported opcode was sent to device") + raise HailoRTUnsupportedOpcodeException("HailoRT has failed because an unsupported opcode was sent to device") from libhailort_exception if string_error_code == "HAILO_INVALID_FRAME": - raise HailoRTInvalidFrameException("An invalid frame was received") + raise HailoRTInvalidFrameException("An invalid frame was received") from libhailort_exception if string_error_code == "HAILO_TIMEOUT": - raise HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred") + raise HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred") from libhailort_exception if string_error_code == "HAILO_STREAM_ABORTED_BY_HW": - raise HailoRTStreamAborted("Stream aborted due to an external event") + raise HailoRTStreamAborted("Stream aborted due to an external event") from libhailort_exception if string_error_code == "HAILO_INVALID_OPERATION": - raise HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information") + raise HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information") from libhailort_exception if string_error_code == "HAILO_INVALID_ARGUMENT": - raise HailoRTInvalidArgumentException("Invalid argument. See hailort.log for more information") + raise HailoRTInvalidArgumentException("Invalid argument. See hailort.log for more information") from libhailort_exception if string_error_code == "HAILO_NOT_FOUND": - raise HailoRTNotFoundException("Item not found. See hailort.log for more information") + raise HailoRTNotFoundException("Item not found. See hailort.log for more information") from libhailort_exception if string_error_code == "HAILO_INVALID_HEF": - raise HailoRTInvalidHEFException("Invalid HEF. See hailort.log for more information") + raise HailoRTInvalidHEFException("Invalid HEF. See hailort.log for more information") from libhailort_exception if string_error_code == "HAILO_ETH_FAILURE": - raise HailoRTEthException("Ethernet failure. See hailort.log for more information") - if string_error_code == "HAILO_PCIE_DRIVER_FAIL": - raise HailoRTPCIeDriverException("PCIe driver failure. run 'dmesg | grep hailo' for more information") + raise HailoRTEthException("Ethernet failure. See hailort.log for more information") from libhailort_exception + if string_error_code == "HAILO_DRIVER_FAIL": + raise HailoRTPCIeDriverException("PCIe driver failure. run 'dmesg | grep hailo' for more information") from libhailort_exception if string_error_code == "HAILO_NETWORK_GROUP_NOT_ACTIVATED": - raise HailoRTNetworkGroupNotActivatedException("Network group is not activated") + raise HailoRTNetworkGroupNotActivatedException("Network group is not activated") from libhailort_exception else: - raise HailoRTException("libhailort failed with error: {} ({})".format(error_code, string_error_code)) + raise HailoRTException("libhailort failed with error: {} ({})".format(error_code, string_error_code)) from libhailort_exception def get_status_message(status_code): status_str = _pyhailort.get_status_message(status_code) @@ -523,6 +526,8 @@ class ConfiguredNetwork(object): def __init__(self, configured_network, target, hef): self._configured_network = configured_network + self._input_vstreams_holders = [] + self._output_vstreams_holders = [] self._target = target self._hef = hef @@ -540,6 +545,7 @@ class ConfiguredNetwork(object): :class:`ActivatedNetworkContextManager`: Context manager that returns the activated network group. """ + # TODO: HRT-9988 - Add deprecation warning when changing to service by default network_group_params = network_group_params or self.create_params() with ExceptionWrapper(): @@ -670,11 +676,39 @@ class ConfiguredNetwork(object): with ExceptionWrapper(): return self._configured_network.get_udp_rates_dict(int(fps), int(max_supported_rate_bytes)) + def _before_fork(self): + if self._configured_network is not None: + self._configured_network.before_fork() + for input_vstreams in self._input_vstreams_holders: + input_vstreams.before_fork() + for output_vstreams in self._output_vstreams_holders: + output_vstreams.before_fork() + + def _after_fork_in_parent(self): + if self._configured_network is not None: + self._configured_network.after_fork_in_parent() + for input_vstreams in self._input_vstreams_holders: + input_vstreams.after_fork_in_parent() + for output_vstreams in self._output_vstreams_holders: + output_vstreams.after_fork_in_parent() + + def _after_fork_in_child(self): + if self._configured_network is not None: + self._configured_network.after_fork_in_child() + for input_vstreams in self._input_vstreams_holders: + input_vstreams.after_fork_in_child() + for output_vstreams in self._output_vstreams_holders: + output_vstreams.after_fork_in_child() + def _create_input_vstreams(self, input_vstreams_params): - return self._configured_network.InputVStreams(input_vstreams_params) + input_vstreams_holder = self._configured_network.InputVStreams(input_vstreams_params) + self._input_vstreams_holders.append(input_vstreams_holder) + return input_vstreams_holder def _create_output_vstreams(self, output_vstreams_params): - return self._configured_network.OutputVStreams(output_vstreams_params) + output_vstreams_holder = self._configured_network.OutputVStreams(output_vstreams_params) + self._output_vstreams_holders.append(output_vstreams_holder) + return output_vstreams_holder def get_stream_names_from_vstream_name(self, vstream_name): """Get stream name from vstream name for a specific network group. @@ -700,6 +734,38 @@ class ConfiguredNetwork(object): with ExceptionWrapper(): return self._hef.get_vstream_names_from_stream_name(stream_name, self.name) + def set_scheduler_timeout(self, timeout_ms, network_name=None): + """Sets the maximum time period that may pass before getting run time from the scheduler, + even without reaching the minimum required send requests (e.g. threshold - see set_scheduler_threshold()), + as long as at least one send request has been sent. + This time period is measured since the last time the scheduler gave this network group run time. + + Args: + timeout_ms (int): Timeout in milliseconds. + """ + name = network_name if network_name is not None else self.name + return self._configured_network.set_scheduler_timeout(timeout_ms, name) + + def set_scheduler_threshold(self, threshold): + """Sets the minimum number of send requests required before the network is considered ready to get run time from the scheduler. + If at least one send request has been sent, but the threshold is not reached within a set time period (e.g. timeout - see hailo_set_scheduler_timeout()), + the scheduler will consider the network ready regardless. + + Args: + threshold (int): Threshold in number of frames. + """ + return self._configured_network.set_scheduler_threshold(threshold) + + def set_scheduler_priority(self, priority): + """Sets the priority of the network. + When the model scheduler will choose the next network, networks with higher priority will be prioritized in the selection. + bigger number represent higher priority. + + Args: + priority (int): Priority as a number between HAILO_SCHEDULER_PRIORITY_MIN - HAILO_SCHEDULER_PRIORITY_MAX. + """ + return self._configured_network.set_scheduler_priority(priority) + class ActivatedNetworkContextManager(object): """A context manager that returns the activated network group upon enter.""" @@ -1075,7 +1141,8 @@ class HailoRTTransformUtils(object): class InternalEthernetDevice(object): def __init__(self, address, port, response_timeout_seconds=10, max_number_of_attempts=3): - # default_logger().warning("InternalEthernetDevice is deprecated! Please use Device object.") + # TODO: HRT-9987 - Add this deprecation warning + # default_logger().warning("InternalEthernetDevice is deprecated! Please use VDevice object.") self.device = None self._address = address self._port = port @@ -1104,7 +1171,6 @@ class PcieDeviceInfo(_pyhailort.PcieDeviceInfo): def __init__(self, bus, device, func, domain=None): super(PcieDeviceInfo, self).__init__() - # default_logger().warning("PcieDeviceInfo is deprecated! Please use Device object with device_id.") self.bus = bus self.device = device self.func = func @@ -1141,7 +1207,6 @@ class PcieDeviceInfo(_pyhailort.PcieDeviceInfo): class InternalPcieDevice(object): def __init__(self, device_info=None): - # self._logger.warning("InternalPcieDevice deprecated! Please use Device object.") self.device = None if device_info is None: device_info = InternalPcieDevice.scan_devices()[0] @@ -1235,8 +1300,8 @@ class HailoFormatFlags(_pyhailort.FormatFlags): SUPPORTED_PROTOCOL_VERSION = 2 SUPPORTED_FW_MAJOR = 4 -SUPPORTED_FW_MINOR = 12 -SUPPORTED_FW_REVISION = 1 +SUPPORTED_FW_MINOR = 13 +SUPPORTED_FW_REVISION = 0 MEGA_MULTIPLIER = 1000.0 * 1000.0 @@ -1245,7 +1310,7 @@ class DeviceArchitectureTypes(IntEnum): HAILO8_A0 = 0 HAILO8 = 1 HAILO8L = 2 - MERCURY_CA = 3 + HAILO15 = 3 def __str__(self): return self.name @@ -1301,8 +1366,8 @@ class BoardInformation(object): if ((device_arch == DeviceArchitectureTypes.HAILO8) or (device_arch == DeviceArchitectureTypes.HAILO8L)): return 'hailo8' - elif device_arch == DeviceArchitectureTypes.MERCURY_CA: - return 'mercury' + elif device_arch == DeviceArchitectureTypes.HAILO15: + return 'hailo15' else: raise HailoRTException("Unsupported device architecture.") @@ -1549,7 +1614,6 @@ class SupportedFeatures(object): def _is_feature_enabled(self, feature): return (self.supported_features & feature) != 0 - class Control: """The control object of this device, which implements the control API of the Hailo device. Should be used only from Device.control""" @@ -1557,7 +1621,7 @@ class Control: WORD_SIZE = 4 def __init__(self, device: '_pyhailort.Device'): - self._device = device + self.__device = device self._logger = default_logger() # TODO: should remove? @@ -1566,12 +1630,19 @@ class Control: self._identify_info = self.identify() + @property + def _device(self): + if not self.__device.is_valid(): + raise HailoRTInvalidOperationException("The device in use has been released. " + "This can happen if 'device.release()' has been called, or one-liner usage of control 'Device().control.XX()'") + return self.__device + @property def device_id(self): """Getter for the device_id. Returns: - str: A string ID of the device. BDF for PCIe devices, IP address for Ethernet devices, "Core" for core devices. + str: A string ID of the device. BDF for PCIe devices, IP address for Ethernet devices, "Integrated" for integrated nnc devices. """ return self._device.device_id @@ -1597,7 +1668,7 @@ class Control: """reloads the device firmware (soft reset)""" with ExceptionWrapper(): return self._device.reset(_pyhailort.ResetDeviceMode.SOFT) - + def forced_soft_reset(self): """reloads the device firmware (forced soft reset)""" with ExceptionWrapper(): @@ -1638,7 +1709,7 @@ class Control: configure_params_by_name (dict, optional): Maps between each net_group_name to configure_params. In case of a mismatch with net_groups_names, default params will be used. - """ + """ with ExceptionWrapper(): return self._device.configure(hef._hef, configure_params_by_name) @@ -1672,9 +1743,7 @@ class Control: measurement types, please look at :class:`~hailo_platform.pyhailort.pyhailort.PowerMeasurementTypes`. """ - if ((self._identify_info.device_architecture != DeviceArchitectureTypes.HAILO8) and - (self._identify_info.device_architecture != DeviceArchitectureTypes.HAILO8L)): - raise HailoRTException("Invalid device architecture: {}".format(self._identify_info.device_architecture)) + with ExceptionWrapper(): return self._device.power_measurement(dvm, measurement_type) @@ -1751,16 +1820,14 @@ class Control: For all supported measurement types view :class:`~hailo_platform.pyhailort.pyhailort.PowerMeasurementTypes`. """ - if ((self._identify_info.device_architecture != DeviceArchitectureTypes.HAILO8) and - (self._identify_info.device_architecture != DeviceArchitectureTypes.HAILO8L)): - raise HailoRTException("Invalid device architecture: {}".format(self._identify_info.device_architecture)) + with ExceptionWrapper(): return self._device.get_power_measurement(buffer_index, should_clear) def _examine_user_config(self): with ExceptionWrapper(): return self._device.examine_user_config() - + def read_user_config(self): """Read the user configuration section as binary data. @@ -1778,11 +1845,11 @@ class Control: """ with ExceptionWrapper(): return self._device.write_user_config(configuration) - + def _erase_user_config(self): with ExceptionWrapper(): return self._device.erase_user_config() - + def read_board_config(self): """Read the board configuration section as binary data. @@ -1808,7 +1875,7 @@ class Control: :class:`~hailo_platform.pyhailort.pyhailort.BoardInformation` """ with ExceptionWrapper(): - response = self._device.identify() + response = self._device.identify() board_information = BoardInformation(response.protocol_version, response.fw_version.major, response.fw_version.minor, response.fw_version.revision, response.logger_version, response.board_name, response.is_release, response.extended_context_switch_buffer, @@ -1898,7 +1965,7 @@ class Control: c_slave = self._create_c_i2c_slave(slave) with ExceptionWrapper(): return self._device.i2c_write(c_slave, register_address, data, len(data)) - + def i2c_read(self, slave, register_address, data_length): """Read data from an I2C slave. @@ -1914,7 +1981,7 @@ class Control: c_slave = self._create_c_i2c_slave(slave) with ExceptionWrapper(): return self._device.i2c_read(c_slave, register_address, data_length) - + def read_register(self, address): """Read the value of a register from a given address. @@ -1948,7 +2015,7 @@ class Control: register_value = self.read_register(address) register_value &= ~(1 << bit_index) self.write_memory(address, struct.pack('!I', register_value)) - + def firmware_update(self, firmware_binary, should_reset=True): """Update firmware binary on the flash. @@ -1989,7 +2056,7 @@ class Control: with ExceptionWrapper(): return self._device.sensor_store_config(section_index, reset_data_size, sensor_type, config_file_path, config_height, config_width, config_fps, config_name) - + def store_isp_config(self, reset_config_size, isp_static_config_file_path, isp_runtime_config_file_path, config_height=0, config_width=0, config_fps=0, config_name=None): """Store sensor isp configuration to Hailo chip flash memory. @@ -2018,7 +2085,7 @@ class Control: """ with ExceptionWrapper(): return self._device.sensor_get_sections_info() - + def sensor_set_generic_i2c_slave(self, slave_address, register_address_size, bus_index, should_hold_bus, endianness): """Set a generic I2C slave for sensor usage. @@ -2181,7 +2248,7 @@ class Control: class Device: - """ Hailo device object representation. """ + """ Hailo device object representation (for inference use VDevice)""" @classmethod def scan(cls): @@ -2246,7 +2313,7 @@ class Device: """Getter for the device_id. Returns: - str: A string ID of the device. BDF for PCIe devices, IP address for Ethernet devices, "Core" for core devices. + str: A string ID of the device. BDF for PCIe devices, IP address for Ethernet devices, "Integrated" for integrated nnc devices. """ return self._device_id @@ -2258,7 +2325,7 @@ class Device: configure_params_by_name (dict, optional): Maps between each net_group_name to configure_params. If not provided, default params will be applied """ if self._creation_pid != os.getpid(): - raise HailoRTException("VDevice can only be configured from the process it was created in.") + raise HailoRTException("Device can only be configured from the process it was created in.") with ExceptionWrapper(): configured_apps = self._device.configure(hef._hef, configure_params_by_name) configured_networks = [ConfiguredNetwork(configured_app, self, hef) for configured_app in configured_apps] @@ -2315,9 +2382,10 @@ class VDevice(object): params (:obj:`hailo_platform.pyhailort.pyhailort.VDeviceParams`, optional): VDevice params, call :func:`VDevice.create_params` to get default params. Excludes 'device_ids'. device_ids (list of str, optional): devices ids to create VDevice from, call :func:`Device.scan` to get - list of all available devices. Excludes 'params'. + list of all available devices. Excludes 'params'. Cannot be used together with device_id. """ gc.collect() + self._logger = default_logger() # VDevice __del__ function tries to release self._vdevice. # to avoid AttributeError if the __init__ func fails, we set it to None first. @@ -2330,21 +2398,37 @@ class VDevice(object): self._creation_pid = os.getpid() self._device_ids = device_ids - if self._device_ids is not None: - if self._params is not None: - raise HailoRTException("VDevice can be created from params or device ids. Both parameters were passed to the c'tor") self._open_vdevice() + def _before_fork(self): + if self._vdevice is not None: + self._vdevice.before_fork() + for configured_network in self._loaded_network_groups: + configured_network._before_fork() + + def _after_fork_in_parent(self): + if self._vdevice is not None: + self._vdevice.after_fork_in_parent() + for configured_network in self._loaded_network_groups: + configured_network._after_fork_in_parent() + + def _after_fork_in_child(self): + if self._vdevice is not None: + self._vdevice.after_fork_in_child() + for configured_network in self._loaded_network_groups: + configured_network._after_fork_in_child() + def _open_vdevice(self): - if self._device_ids is not None: - with ExceptionWrapper(): - self._vdevice = _pyhailort.VDevice.create_from_ids(self._device_ids) - else: - if self._params is None: - self._params = VDevice.create_params() - with ExceptionWrapper(): - self._vdevice = _pyhailort.VDevice.create(self._params) + if self._params is None: + self._params = VDevice.create_params() + if sys.platform != "win32" and self._params.multi_process_service: + os.register_at_fork(before=lambda: self._before_fork()) + os.register_at_fork(after_in_parent=lambda: self._after_fork_in_parent()) + os.register_at_fork(after_in_child=lambda: self._after_fork_in_child()) + with ExceptionWrapper(): + device_ids = [] if self._device_ids is None else self._device_ids + self._vdevice = _pyhailort.VDevice.create(self._params, device_ids) def __enter__(self): return self @@ -2496,7 +2580,7 @@ class OutputVStreamParams(object): @staticmethod def make(configured_network, quantized=True, format_type=None, timeout_ms=None, queue_size=None, network_name=None): """Create output virtual stream params from a configured network group. These params determine the format of the - data that will be fed into the network group. + data that will be returned from the network group. Args: configured_network (:class:`ConfiguredNetwork`): The configured network group for which @@ -2537,12 +2621,12 @@ class OutputVStreamParams(object): @staticmethod def make_from_network_group(configured_network, quantized=True, format_type=None, timeout_ms=None, queue_size=None, network_name=None): """Create output virtual stream params from a configured network group. These params determine the format of the - data that will be fed into the network group. + data that will be returned from the network group. Args: configured_network (:class:`ConfiguredNetwork`): The configured network group for which the params are created. - quantized (bool): Whether the data fed into the chip is already quantized. True means + quantized (bool): Whether the data returned from the chip is already quantized. True means the data is already quantized. False means it's HailoRT's responsibility to quantize (scale) the data. Defaults to True. format_type (:class:`~hailo_platform.pyhailort.pyhailort.FormatType`): The @@ -2566,12 +2650,12 @@ class OutputVStreamParams(object): @staticmethod def make_groups(configured_network, quantized=True, format_type=None, timeout_ms=None, queue_size=None): """Create output virtual stream params from a configured network group. These params determine the format of the - data that will be fed into the network group. The params groups are splitted with respect to their underlying streams for multi process usges. + data that will be returned from the network group. The params groups are splitted with respect to their underlying streams for multi process usges. Args: configured_network (:class:`ConfiguredNetwork`): The configured network group for which the params are created. - quantized (bool): Whether the data fed into the chip is already quantized. True means + quantized (bool): Whether the data returned from the chip is already quantized. True means the data is already quantized. False means it's HailoRT's responsibility to quantize (scale) the data. Defaults to True. format_type (:class:`~hailo_platform.pyhailort.pyhailort.FormatType`): The @@ -2660,6 +2744,19 @@ class InputVStream(object): with ExceptionWrapper(): return self._send_object.info + def _before_fork(self): + if self._send_object is not None: + self._send_object.before_fork() + + def _after_fork_in_parent(self): + if self._send_object is not None: + self._send_object.after_fork_in_parent() + + def _after_fork_in_child(self): + if self._send_object is not None: + self._send_object.after_fork_in_child() + + class InputVStreams(object): """Input vstreams pipelines that allows to send data, to be used as a context manager.""" @@ -2709,6 +2806,19 @@ class InputVStreams(object): def __iter__(self): return iter(self._vstreams.values()) + def _before_fork(self): + for vstream in self._vstreams.values(): + vstream._before_fork() + + def _after_fork_in_parent(self): + for vstream in self._vstreams.values(): + vstream._after_fork_in_parent() + + def _after_fork_in_child(self): + for vstream in self._vstreams.values(): + vstream._after_fork_in_child() + + class OutputLayerUtils(object): def __init__(self, hef, vstream_name, pipeline, net_group_name=""): self._hef = hef @@ -2833,6 +2943,19 @@ class OutputVStream(object): with ExceptionWrapper(): return self._recv_object.info + def _before_fork(self): + if self._recv_object is not None: + self._recv_object.before_fork() + + def _after_fork_in_parent(self): + if self._recv_object is not None: + self._recv_object.after_fork_in_parent() + + def _after_fork_in_child(self): + if self._recv_object is not None: + self._recv_object.after_fork_in_child() + + class OutputVStreams(object): """Output virtual streams pipelines that allows to receive data, to be used as a context manager.""" @@ -2896,13 +3019,26 @@ class OutputVStreams(object): def __iter__(self): return iter(self._vstreams.values()) -class YOLOv5PostProcessingOp(object): + def _before_fork(self): + for vstream in self._vstreams.values(): + vstream._before_fork() + + def _after_fork_in_parent(self): + for vstream in self._vstreams.values(): + vstream._after_fork_in_parent() + + def _after_fork_in_child(self): + for vstream in self._vstreams.values(): + vstream._after_fork_in_child() + + +class YOLOv5PostProcessOp(object): def __init__(self, anchors, shapes, formats, quant_infos, image_height, image_width, confidence_threshold, iou_threshold, num_of_classes, - should_dequantize, max_boxes, should_sigmoid, one_class_per_bbox=True): + max_boxes, cross_classes=True): - self._op = _pyhailort.YOLOv5PostProcessingOp.create(anchors, shapes, formats, quant_infos, image_height, image_width, confidence_threshold, - iou_threshold, num_of_classes, should_dequantize, max_boxes, should_sigmoid, one_class_per_bbox) + self._op = _pyhailort.YOLOv5PostProcessOp.create(anchors, shapes, formats, quant_infos, image_height, image_width, confidence_threshold, + iou_threshold, num_of_classes, max_boxes, cross_classes) def execute(self, net_flow_tensors): return self._op.execute(net_flow_tensors) \ No newline at end of file diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/hailocli_commands.py b/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/hailocli_commands.py index f38f1b6..4c6568f 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/hailocli_commands.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/hailocli_commands.py @@ -1,29 +1,33 @@ import os -import subprocess import pathlib +import subprocess import sys +import pkg_resources + import hailo_platform from hailo_platform.tools.hailocli.base_utils import HailortCliUtil -import pkg_resources """ HailoRTCLI matching commands in Hailo-CLI tool. """ + class BenchmarkCommandCLI(HailortCliUtil): def __init__(self, parser): super().__init__(parser, 'benchmark') - + class FWConfigCommandCLI(HailortCliUtil): """CLI tool for changing the FW configuration (User Config)""" + def __init__(self, parser): super().__init__(parser, 'fw-config') class BoardConfigCommandCLI(HailortCliUtil): """CLI tool for changing the FW configuration (Board Config)""" + def __init__(self, parser): super().__init__(parser, 'board-config') @@ -47,7 +51,7 @@ class MeasurePowerCommandCLI(HailortCliUtil): def __init__(self, parser): super().__init__(parser, 'measure-power') - + class RunCommandCLI(HailortCliUtil): def __init__(self, parser): super().__init__(parser, 'run') @@ -56,30 +60,27 @@ class RunCommandCLI(HailortCliUtil): class SensorConfigCommandCLI(HailortCliUtil): def __init__(self, parser): super().__init__(parser, 'sensor-config') - - + + class FWUpdaterCLI(HailortCliUtil): """Cli tool for firmware updates""" + def __init__(self, parser): super().__init__(parser, 'fw-update') - + class SSBUpdaterCLI(HailortCliUtil): """Cli tool for second stage boot updates""" + def __init__(self, parser): super().__init__(parser, 'ssb-update') - + class UDPRateLimiterCLI(HailortCliUtil): """CLI tool for UDP rate limitation.""" - def __init__(self, parser): - super().__init__(parser, 'udp-rate-limiter') - -class VersionCLI(HailortCliUtil): - """CLI tool for hailort version.""" def __init__(self, parser): - super().__init__(parser, '--version') + super().__init__(parser, 'udp-rate-limiter') class TutorialRequired(Exception): @@ -87,7 +88,6 @@ class TutorialRequired(Exception): class TutorialRunnerCLI(): - TUTORIALS_DIR = os.path.join(pathlib.Path(hailo_platform.__file__).parent.parent, 'hailo_tutorials/notebooks/') TUTORIALS_REQUIREMENTS = ["jupyter"] ERROR_MSG = """ @@ -109,7 +109,7 @@ class TutorialRunnerCLI(): working_set.require(req) except pkg_resources.DistributionNotFound: missing_pkgs.append(req) - + if missing_pkgs: sys.tracebacklimit = 0 raise TutorialRequired(f"\n{self.ERROR_MSG}\n {'; '.join([f'pip install {pkg}' for pkg in missing_pkgs])}") diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/main.py b/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/main.py index a37e642..b6620c6 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/main.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/main.py @@ -1,14 +1,19 @@ #!/usr/bin/env python import argparse -import argcomplete import sys +import argcomplete + import hailo_platform -from hailo_platform.tools.hailocli.base_utils import HailortCliUtil, Helper, HailortCliUtilError -from hailo_platform.tools.hailocli.hailocli_commands import (FWUpdaterCLI, SSBUpdaterCLI, ControlCommandCLI, ScanCommandCLI, - LoggerCommandCLI, MeasurePowerCommandCLI, RunCommandCLI, SensorConfigCommandCLI, - FWConfigCommandCLI, BenchmarkCommandCLI, UDPRateLimiterCLI, MonitorCommandCLI, ParseHEFCommandCLI, TutorialRunnerCLI) +from hailo_platform.tools.hailocli.base_utils import HailortCliUtil, HailortCliUtilError, Helper +from hailo_platform.tools.hailocli.hailocli_commands import (BenchmarkCommandCLI, ControlCommandCLI, FWConfigCommandCLI, + FWUpdaterCLI, LoggerCommandCLI, MeasurePowerCommandCLI, + MonitorCommandCLI, ParseHEFCommandCLI, RunCommandCLI, + SSBUpdaterCLI, ScanCommandCLI, SensorConfigCommandCLI, + TutorialRunnerCLI, UDPRateLimiterCLI) +from hailo_platform.tools.hailocli.version_action import CustomVersionAction + # Note: PlatformCommands are external dependencies in phase2-sdk/demos repo; don't change! class PlatformCommands: @@ -25,15 +30,16 @@ class PlatformCommands: 'sensor-config': ('Sensor configuration tool', SensorConfigCommandCLI), 'run': ('Run a compiled network', RunCommandCLI), 'benchmark': ('Measure basic performance on compiled network', BenchmarkCommandCLI), - 'monitor': ("Monitor of networks - Presents information about the running networks. To enable monitor, set in the application process the environment variable 'SCHEDULER_MONITOR' to 1.", MonitorCommandCLI), + 'monitor': ("Monitor of networks - Presents information about the running networks. To enable monitor, set in the application process the environment variable 'HAILO_MONITOR' to 1.", MonitorCommandCLI), 'parse-hef': (' Parse HEF to get information about its components', ParseHEFCommandCLI), 'measure-power': ('Measures power consumption', MeasurePowerCommandCLI), 'tutorial': ('Runs the tutorials in jupyter notebook', TutorialRunnerCLI), - #'--version': ('Print program version and exit', VersionCLI) } def __init__(self): self.parser = argparse.ArgumentParser(description=self._get_generic_description()) + self.parser.register('action', 'custom_version', CustomVersionAction) + self.parser.add_argument('--version', action='custom_version') self.subparsers = self.parser.add_subparsers(help='Hailo utilities aimed to help with everything you need') self.COMMANDS = {} self.COMMANDS.update(type(self).PLATFORM_COMMANDS) @@ -60,6 +66,7 @@ class PlatformCommands: # Dependency injection for testing def _run(self, argv): self.COMMANDS['help'] = ('show the list of commands', Helper(self.COMMANDS)) + # Create the commands and let them set the arguments commands = {} for command_name, (help_, command_class) in self.COMMANDS.items(): @@ -79,10 +86,10 @@ class PlatformCommands: command_name = argv[0] if (command_name in commands) and isinstance(commands[command_name], HailortCliUtil): # HailortCliUtil just passes the rest of the argv to hailortcli - try : + try: return commands[command_name].run(argv[1:]) except HailortCliUtilError as e: - print('\n'+ str(e)) + print('\n' + str(e)) return # This isn't a HailortCliUtil commnad, parse with argparse diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/version_action.py b/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/version_action.py new file mode 100644 index 0000000..8003a9d --- /dev/null +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/version_action.py @@ -0,0 +1,31 @@ +import argparse + +import hailo_platform + + +class CustomVersionAction(argparse.Action): + def __init__(self, + option_strings, + dest=argparse.SUPPRESS, + default=argparse.SUPPRESS, + help="show program's version number and exit"): + super(CustomVersionAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + @staticmethod + def _print_version(): + print(f'HailoRT v{hailo_platform.__version__}') + + try: + import hailo_sdk_client + print(f'Hailo Dataflow Compiler v{hailo_sdk_client.__version__}') + except ImportError: + pass + + def __call__(self, parser, namespace, values, option_string=None): + self._print_version() + parser.exit() diff --git a/hailort/libhailort/bindings/python/platform/hailo_platform/tools/udp_rate_limiter.py b/hailort/libhailort/bindings/python/platform/hailo_platform/tools/udp_rate_limiter.py index 75967ef..b597e32 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_platform/tools/udp_rate_limiter.py +++ b/hailort/libhailort/bindings/python/platform/hailo_platform/tools/udp_rate_limiter.py @@ -36,7 +36,7 @@ def get_max_supported_kbps(hw_arch="hailo8"): class RateLimiterWrapper(object): """UDPRateLimiter wrapper enabling ``with`` statements.""" - def __init__(self, network_group, fps=1, fps_factor=1.0): + def __init__(self, network_group, fps=1, fps_factor=1.0, remote_ip=None, hw_arch=None): """RateLimiterWrapper constructor. Args: @@ -48,10 +48,18 @@ class RateLimiterWrapper(object): if not isinstance(network_group, ConfiguredNetwork): return RateLimiterException("The API was changed. RateLimiterWrapper accept ConfiguredNetwork instead of ActivatedNetwork") self._network_group = network_group - self._remote_ip = network_group._target.remote_ip + if remote_ip is not None: + self._remote_ip = remote_ip + else: + # this line should be removed. this parameter will be removed from the object + self._remote_ip = network_group._target.device_id self._fps = fps self._fps_factor = fps_factor - self._hw_arch = network_group._target._hw_arch + if hw_arch is not None: + self._hw_arch = hw_arch + else: + # this line should be removed. this parameter will be removed from the object + self._hw_arch = network_group._target._hw_arch if hasattr(network_group._target, '_hw_arch') else None self._rates_dict = {} self._tc_dict = {} diff --git a/hailort/libhailort/bindings/python/platform/hailo_tutorials/notebooks/HRT_0_Inference_Tutorial.ipynb b/hailort/libhailort/bindings/python/platform/hailo_tutorials/notebooks/HRT_0_Inference_Tutorial.ipynb index 038add2..8a7785c 100644 --- a/hailort/libhailort/bindings/python/platform/hailo_tutorials/notebooks/HRT_0_Inference_Tutorial.ipynb +++ b/hailort/libhailort/bindings/python/platform/hailo_tutorials/notebooks/HRT_0_Inference_Tutorial.ipynb @@ -43,12 +43,12 @@ "source": [ "import numpy as np\n", "from multiprocessing import Process\n", - "from hailo_platform import (HEF, PcieDevice, HailoStreamInterface, InferVStreams, ConfigureParams,\n", + "from hailo_platform import (HEF, VDevice, HailoStreamInterface, InferVStreams, ConfigureParams,\n", " InputVStreamParams, OutputVStreamParams, InputVStreams, OutputVStreams, FormatType)\n", "\n", "# The target can be used as a context manager (\"with\" statement) to ensure it's released on time.\n", "# Here it's avoided for the sake of simplicity\n", - "target = PcieDevice()\n", + "target = VDevice()\n", "\n", "# Loading compiled HEFs to device:\n", "model_name = 'resnet_v1_18'\n", diff --git a/hailort/libhailort/bindings/python/platform/setup.py b/hailort/libhailort/bindings/python/platform/setup.py index c03c4c2..445e867 100644 --- a/hailort/libhailort/bindings/python/platform/setup.py +++ b/hailort/libhailort/bindings/python/platform/setup.py @@ -69,6 +69,6 @@ if __name__ == "__main__": "linux_aarch64", ], url="https://hailo.ai/", - version="4.12.1", + version="4.13.0", zip_safe=False, ) diff --git a/hailort/libhailort/bindings/python/src/CMakeLists.txt b/hailort/libhailort/bindings/python/src/CMakeLists.txt index 90cba95..bd032c4 100644 --- a/hailort/libhailort/bindings/python/src/CMakeLists.txt +++ b/hailort/libhailort/bindings/python/src/CMakeLists.txt @@ -24,6 +24,7 @@ pybind11_add_module(_pyhailort hef_api.cpp vstream_api.cpp quantization_api.cpp + ${HAILORT_OPS_CPP_SOURCES} ${HAILORT_COMMON_CPP_SOURCES} ) @@ -44,6 +45,10 @@ target_link_libraries(_pyhailort PRIVATE libhailort spdlog::spdlog) if(WIN32) target_link_libraries(_pyhailort PRIVATE Ws2_32 Iphlpapi Shlwapi) endif() +if(HAILO_BUILD_SERVICE) + target_link_libraries(_pyhailort PRIVATE grpc++_unsecure hailort_rpc_grpc_proto hef_proto) +endif() + target_compile_options(_pyhailort PRIVATE ${HAILORT_COMPILE_OPTIONS}) exclude_archive_libs_symbols(_pyhailort) diff --git a/hailort/libhailort/bindings/python/src/bindings_common.hpp b/hailort/libhailort/bindings/python/src/bindings_common.hpp index 95b60a9..513175b 100644 --- a/hailort/libhailort/bindings/python/src/bindings_common.hpp +++ b/hailort/libhailort/bindings/python/src/bindings_common.hpp @@ -13,9 +13,14 @@ #include "hailo/hailort.h" #include "hailo/hailort_common.hpp" #include "hailo/network_group.hpp" -#include "utils.hpp" + #include "common/logger_macros.hpp" +#include "utils.hpp" + +#include + + namespace hailort { class HailoRTBindingsCommon diff --git a/hailort/libhailort/bindings/python/src/device_api.cpp b/hailort/libhailort/bindings/python/src/device_api.cpp index b2423e1..c6f4a5b 100644 --- a/hailort/libhailort/bindings/python/src/device_api.cpp +++ b/hailort/libhailort/bindings/python/src/device_api.cpp @@ -126,7 +126,7 @@ bool DeviceWrapper::get_overcurrent_state() py::bytes DeviceWrapper::read_memory(uint32_t address, uint32_t length) { std::unique_ptr response = make_unique_nothrow(length, '\x00'); - VALIDATE_NOT_NULL(response); + VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY); MemoryView data_view(const_cast(reinterpret_cast(response->data())), length); auto status = device().read_memory(address, data_view); @@ -150,7 +150,7 @@ void DeviceWrapper::test_chip_memories() void DeviceWrapper::i2c_write(hailo_i2c_slave_config_t *slave_config, uint32_t register_address, py::bytes data, uint32_t length) { - VALIDATE_NOT_NULL(slave_config); + VALIDATE_NOT_NULL(slave_config, HAILO_INVALID_ARGUMENT); std::string data_str(data); MemoryView data_view = MemoryView::create_const(data_str.c_str(), length); @@ -160,10 +160,10 @@ void DeviceWrapper::i2c_write(hailo_i2c_slave_config_t *slave_config, uint32_t r py::bytes DeviceWrapper::i2c_read(hailo_i2c_slave_config_t *slave_config, uint32_t register_address, uint32_t length) { - VALIDATE_NOT_NULL(slave_config); + VALIDATE_NOT_NULL(slave_config, HAILO_INVALID_ARGUMENT); std::unique_ptr response = make_unique_nothrow(length, '\x00'); - VALIDATE_NOT_NULL(response); + VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY); MemoryView data_view(const_cast(reinterpret_cast(response->data())), length); auto status = device().i2c_read(*slave_config, register_address, data_view); @@ -231,7 +231,7 @@ py::bytes DeviceWrapper::read_user_config() std::unique_ptr response = make_unique_nothrow( const_cast(reinterpret_cast(config_buffer->data())), config_buffer->size()); - VALIDATE_NOT_NULL(response); + VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY); return *response; } @@ -257,7 +257,7 @@ py::bytes DeviceWrapper::read_board_config() std::unique_ptr response = make_unique_nothrow( const_cast(reinterpret_cast(config_buffer->data())), config_buffer->size()); - VALIDATE_NOT_NULL(response); + VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY); return *response; } @@ -309,7 +309,7 @@ py::bytes DeviceWrapper::sensor_get_sections_info() std::unique_ptr response = make_unique_nothrow( const_cast(reinterpret_cast(buffer->data())), buffer->size()); - VALIDATE_NOT_NULL(response); + VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY); return *response; } @@ -446,7 +446,7 @@ void DeviceWrapper::direct_write_memory(uint32_t address, py::bytes buffer) { const auto buffer_str = static_cast(buffer); hailo_status status = device().direct_write_memory(address, buffer_str.c_str(), - (uint32_t) (buffer_str.length())); + (uint32_t)(buffer_str.length())); VALIDATE_STATUS(status); } @@ -478,6 +478,8 @@ void DeviceWrapper::set_sleep_state(hailo_sleep_state_t sleep_state) void DeviceWrapper::add_to_python_module(py::module &m) { py::class_(m, "Device") + .def("is_valid", &DeviceWrapper::is_valid) + // Scan .def("scan", &DeviceWrapper::scan) diff --git a/hailort/libhailort/bindings/python/src/device_api.hpp b/hailort/libhailort/bindings/python/src/device_api.hpp index 1d3c18e..b216cce 100644 --- a/hailort/libhailort/bindings/python/src/device_api.hpp +++ b/hailort/libhailort/bindings/python/src/device_api.hpp @@ -11,13 +11,15 @@ #ifndef _DEVICE_API_HPP_ #define _DEVICE_API_HPP_ -#include "utils.hpp" -#include "hailo/hailort.hpp" +#include "hailo/hailort.h" +#include "hailo/device.hpp" + #include "common/socket.hpp" + +#include "utils.hpp" #include "hef_api.hpp" #include -#include namespace hailort @@ -53,16 +55,21 @@ public: Device& device() { - VALIDATE_NOT_NULL(m_device); + VALIDATE_NOT_NULL(m_device, HAILO_INTERNAL_FAILURE); return *(m_device.get()); } const Device& device() const { - VALIDATE_NOT_NULL(m_device); + VALIDATE_NOT_NULL(m_device, HAILO_INTERNAL_FAILURE); return *(m_device.get()); } + bool is_valid() + { + return (nullptr != m_device); + } + Device& operator*() // Used for control_internals { return device(); diff --git a/hailort/libhailort/bindings/python/src/hef_api.cpp b/hailort/libhailort/bindings/python/src/hef_api.cpp index f940f31..f255768 100644 --- a/hailort/libhailort/bindings/python/src/hef_api.cpp +++ b/hailort/libhailort/bindings/python/src/hef_api.cpp @@ -207,9 +207,10 @@ ActivatedAppContextManagerWrapper::ActivatedAppContextManagerWrapper(ConfiguredN const ActivatedNetworkGroup& ActivatedAppContextManagerWrapper::enter() { auto activated = m_net_group.activate(m_network_group_params); - VALIDATE_EXPECTED(activated); - - m_activated_net_group = activated.release(); + if (activated.status() != HAILO_NOT_IMPLEMENTED) { + VALIDATE_EXPECTED(activated); + m_activated_net_group = activated.release(); + } return std::ref(*m_activated_net_group); } @@ -273,7 +274,9 @@ void HefWrapper::initialize_python_module(py::module &m) .def("wait_for_activation", [](ConfiguredNetworkGroup& self, uint32_t timeout_ms) { auto status = self.wait_for_activation(std::chrono::milliseconds(timeout_ms)); - VALIDATE_STATUS(status); + if (status != HAILO_NOT_IMPLEMENTED) { + VALIDATE_STATUS(status); + } }) .def("InputVStreams", [](ConfiguredNetworkGroup &self, std::map &input_vstreams_params) { @@ -295,6 +298,49 @@ void HefWrapper::initialize_python_module(py::module &m) return py::cast(results.value()); }) + .def("before_fork", [](ConfiguredNetworkGroup& self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.before_fork(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + }) + .def("after_fork_in_parent", [](ConfiguredNetworkGroup& self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.after_fork_in_parent(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + }) + .def("after_fork_in_child", [](ConfiguredNetworkGroup& self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.after_fork_in_child(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + }) + .def("set_scheduler_timeout", [](ConfiguredNetworkGroup& self, int timeout, const std::string &network_name="") + { + auto timeout_mili = std::chrono::milliseconds(timeout); + auto status = self.set_scheduler_timeout(timeout_mili, network_name); + VALIDATE_STATUS(status); + }) + .def("set_scheduler_threshold", [](ConfiguredNetworkGroup& self, uint32_t threshold) + { + auto status = self.set_scheduler_threshold(threshold); + VALIDATE_STATUS(status); + }) + .def("set_scheduler_priority", [](ConfiguredNetworkGroup& self, uint8_t priority) + { + auto status = self.set_scheduler_priority(priority); + VALIDATE_STATUS(status); + }) ; ActivatedAppContextManagerWrapper::add_to_python_module(m); diff --git a/hailort/libhailort/bindings/python/src/internal/CMakeLists.txt b/hailort/libhailort/bindings/python/src/internal/CMakeLists.txt index ccf091f..292a765 100644 --- a/hailort/libhailort/bindings/python/src/internal/CMakeLists.txt +++ b/hailort/libhailort/bindings/python/src/internal/CMakeLists.txt @@ -1,11 +1,13 @@ -cmake_minimum_required(VERSION 3.0.0) +cmake_minimum_required(VERSION 3.15.0) pybind11_add_module(_pyhailort_internal SHARED pyhailort_internal.cpp control_api.cpp - ${HAILORT_SRCS_ABS} + $ ) +add_dependencies(_pyhailort_internal libhailort) + set_target_properties(_pyhailort_internal PROPERTIES CXX_STANDARD 14 CXX_STANDARD_REQUIRED YES @@ -22,7 +24,6 @@ target_include_directories(_pyhailort_internal ) target_link_libraries(_pyhailort_internal PRIVATE - libhailort hef_proto spdlog::spdlog readerwriterqueue diff --git a/hailort/libhailort/bindings/python/src/internal/control_api.cpp b/hailort/libhailort/bindings/python/src/internal/control_api.cpp index b53f4cc..07d790c 100644 --- a/hailort/libhailort/bindings/python/src/internal/control_api.cpp +++ b/hailort/libhailort/bindings/python/src/internal/control_api.cpp @@ -93,7 +93,7 @@ void ControlWrapper::validate_firmware_update(DeviceWrapper &device, py::bytes m py::bytes ControlWrapper::sensor_get_config(DeviceWrapper &device, uint32_t section_index, uint32_t offset, uint32_t data_length) { std::unique_ptr response = make_unique_nothrow(data_length, '\x00'); - VALIDATE_NOT_NULL(response); + VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY); auto status = Control::sensor_get_config(*device, section_index, offset, data_length, (uint8_t*)(response->data())); VALIDATE_STATUS(status); diff --git a/hailort/libhailort/bindings/python/src/internal/control_api.hpp b/hailort/libhailort/bindings/python/src/internal/control_api.hpp index 8106f34..ffcf26d 100644 --- a/hailort/libhailort/bindings/python/src/internal/control_api.hpp +++ b/hailort/libhailort/bindings/python/src/internal/control_api.hpp @@ -11,7 +11,7 @@ #ifndef _CONTROL_API_HPP_ #define _CONTROL_API_HPP_ -#include "control.hpp" +#include "device_common/control.hpp" #include "utils.hpp" #include "device_api.hpp" diff --git a/hailort/libhailort/bindings/python/src/internal/pyhailort_internal.cpp b/hailort/libhailort/bindings/python/src/internal/pyhailort_internal.cpp index 18f4c49..b81794f 100644 --- a/hailort/libhailort/bindings/python/src/internal/pyhailort_internal.cpp +++ b/hailort/libhailort/bindings/python/src/internal/pyhailort_internal.cpp @@ -1,3 +1,15 @@ + + +#include "hailo/hailort.h" + +#include "transform/transform_internal.hpp" +#include "bindings_common.hpp" + +#include "pyhailort_internal.hpp" +#include "control_api.hpp" +#include "utils.hpp" +#include "utils.h" + #include #include #include @@ -6,22 +18,15 @@ #include #include -#include "pyhailort_internal.hpp" -#include "control_api.hpp" -#include "utils.hpp" -#include "utils.h" - -#include "hailo/hailort.h" -#include "transform_internal.hpp" -#include "bindings_common.hpp" - namespace hailort { +// TODO: Remove (HRT-9944) +// Duplicated for hailo post process test with python API. +static const uint32_t TEST_NUM_OF_CLASSES = 80; -static const uint32_t TEST_NUM_OF_CLASSES2 = 80; -py::array PyhailortInternal::get_yolov5_post_process_expected_buffer() +Expected get_expected_buffer_float32() { static const uint32_t DETECTION_CLASS_ID_1 = 0; static const float32_t CLASS_ID_1_DETECTION_COUNT = 5; @@ -105,13 +110,13 @@ py::array PyhailortInternal::get_yolov5_post_process_expected_buffer() }; static const uint32_t DETECTION_COUNT = 9; - auto buffer_size = (DETECTION_COUNT * sizeof(hailo_bbox_float32_t)) + (TEST_NUM_OF_CLASSES2 * sizeof(float32_t)); - auto buffer_expected = hailort::Buffer::create(buffer_size, 0); - // CATCH_REQUIRE_EXPECTED(buffer_expected); + auto buffer_size = (DETECTION_COUNT * sizeof(hailo_bbox_float32_t)) + (TEST_NUM_OF_CLASSES * sizeof(float32_t)); + auto buffer_expected = Buffer::create(buffer_size, 0); + CHECK_EXPECTED(buffer_expected); auto buffer = buffer_expected.release(); size_t offset = 0; - for (uint32_t class_index = 0; class_index < TEST_NUM_OF_CLASSES2; class_index++) { + for (uint32_t class_index = 0; class_index < TEST_NUM_OF_CLASSES; class_index++) { if (DETECTION_CLASS_ID_1 == class_index) { memcpy(buffer.data() + offset, &CLASS_ID_1_DETECTION_COUNT, sizeof(CLASS_ID_1_DETECTION_COUNT)); offset += sizeof(CLASS_ID_1_DETECTION_COUNT); @@ -160,12 +165,20 @@ py::array PyhailortInternal::get_yolov5_post_process_expected_buffer() } } + return buffer; +} + +py::array PyhailortInternal::get_yolov5_post_process_expected_buffer() +{ + auto buffer = get_expected_buffer_float32(); + VALIDATE_EXPECTED(buffer); + // Note: The ownership of the buffer is transferred to Python wrapped as a py::array. // When the py::array isn't referenced anymore in Python and is destructed, the py::capsule's dtor // is called too (and it deletes the raw buffer) auto type = py::dtype(HailoRTBindingsCommon::convert_format_type_to_string(HAILO_FORMAT_TYPE_FLOAT32)); - auto shape = *py::array::ShapeContainer({buffer.size()}); - const auto unmanaged_addr = buffer.release(); + auto shape = *py::array::ShapeContainer({buffer->size()}); + const auto unmanaged_addr = buffer.release().release(); return py::array(type, shape, unmanaged_addr, py::capsule(unmanaged_addr, [](void *p) { delete reinterpret_cast(p); })); } @@ -261,10 +274,10 @@ bool PyhailortInternal::is_output_transformation_required( py::list PyhailortInternal::get_all_layers_info(const HefWrapper &hef, const std::string &net_group_name) { - auto network_group_metadata = hef.hef_ptr()->pimpl->get_network_group_metadata(net_group_name); - VALIDATE_EXPECTED(network_group_metadata); + auto core_op_metadata = hef.hef_ptr()->pimpl->get_core_op_metadata(net_group_name); + VALIDATE_EXPECTED(core_op_metadata); - return py::cast(network_group_metadata->get_all_layer_infos()); + return py::cast(core_op_metadata->get_all_layer_infos()); } PYBIND11_MODULE(_pyhailort_internal, m) { diff --git a/hailort/libhailort/bindings/python/src/internal/pyhailort_internal.hpp b/hailort/libhailort/bindings/python/src/internal/pyhailort_internal.hpp index 94f58d5..e8d0994 100644 --- a/hailort/libhailort/bindings/python/src/internal/pyhailort_internal.hpp +++ b/hailort/libhailort/bindings/python/src/internal/pyhailort_internal.hpp @@ -10,6 +10,11 @@ #ifndef _PYHAILORT_INTERNAL_ #define _PYHAILORT_INTERNAL_ +#include "hef/hef_internal.hpp" + +#include "hef_api.hpp" +#include "utils.hpp" +#include "utils.h" #include #include #include @@ -18,10 +23,6 @@ #include #include -#include "hef_internal.hpp" -#include "hef_api.hpp" -#include "utils.hpp" -#include "utils.h" namespace hailort { diff --git a/hailort/libhailort/bindings/python/src/net_flow_api.hpp b/hailort/libhailort/bindings/python/src/net_flow_api.hpp index 2bf26ab..df29b36 100644 --- a/hailort/libhailort/bindings/python/src/net_flow_api.hpp +++ b/hailort/libhailort/bindings/python/src/net_flow_api.hpp @@ -10,42 +10,70 @@ #ifndef _HAILO_NET_FLOW_API_HPP_ #define _HAILO_NET_FLOW_API_HPP_ +#include "hailo/hailort.h" + +#include "net_flow/ops/yolo_post_process.hpp" + #include "utils.hpp" -#include "hailo/hailort.hpp" #include "bindings_common.hpp" -#include "net_flow/ops/yolo_post_processing.hpp" + namespace hailort { namespace net_flow { -class YOLOv5PostProcessingOpWrapper +class YOLOv5PostProcessOpWrapper { public: - static YOLOv5PostProcessingOpWrapper create(const std::vector> &anchors, + static YOLOv5PostProcessOpWrapper create(const std::vector> &anchors, const std::vector &shapes, const std::vector &formats, const std::vector &quant_infos, float32_t image_height, float32_t image_width, float32_t confidence_threshold, - float32_t iou_threshold, uint32_t num_of_classes, bool should_dequantize, uint32_t max_boxes, bool should_sigmoid, - bool one_class_per_bbox=true) + float32_t iou_threshold, uint32_t num_of_classes, uint32_t max_boxes, + bool cross_classes=true) { - auto op = YOLOv5PostProcessingOp::create(anchors, shapes, formats, quant_infos, image_height, image_width, - confidence_threshold, iou_threshold, num_of_classes, should_dequantize, max_boxes, should_sigmoid, one_class_per_bbox); + std::map inputs_metadata; + std::map outputs_metadata; + + net_flow::NmsPostProcessConfig nms_post_process_config{}; + nms_post_process_config.nms_score_th = confidence_threshold; + nms_post_process_config.nms_iou_th = iou_threshold; + nms_post_process_config.max_proposals_per_class = max_boxes; + nms_post_process_config.classes = num_of_classes; + nms_post_process_config.background_removal = false; + nms_post_process_config.background_removal_index = 0; + nms_post_process_config.cross_classes = cross_classes; + net_flow::YoloPostProcessConfig yolo_post_process_config{}; + yolo_post_process_config.image_height = image_height; + yolo_post_process_config.image_width = image_width; + // Each layer anchors vector is structured as {w,h} pairs. + for (size_t i = 0; i < anchors.size(); ++i) { + auto name = std::to_string(i); + yolo_post_process_config.anchors.insert({name, anchors[i]}); + BufferMetaData input_metadata = { + shapes[i], + shapes[i], + formats[i], + quant_infos[i] + }; + inputs_metadata.insert({name, input_metadata}); + } + auto op = YOLOv5PostProcessOp::create(inputs_metadata, outputs_metadata, nms_post_process_config, yolo_post_process_config); VALIDATE_EXPECTED(op); - - return YOLOv5PostProcessingOpWrapper(op.release(), num_of_classes, max_boxes); + + return YOLOv5PostProcessOpWrapper(op.release(), num_of_classes, max_boxes); } static void add_to_python_module(py::module &m) { - py::class_(m, "YOLOv5PostProcessingOp") - .def("create", &YOLOv5PostProcessingOpWrapper::create) - .def("execute",[](YOLOv5PostProcessingOpWrapper &self, const std::vector &tensors) + py::class_(m, "YOLOv5PostProcessOp") + .def("create", &YOLOv5PostProcessOpWrapper::create) + .def("execute",[](YOLOv5PostProcessOpWrapper &self, const std::vector &tensors) { - std::vector data_views; - data_views.reserve(tensors.size()); - for (auto &tensor : tensors) { - data_views.push_back(MemoryView(const_cast(reinterpret_cast(tensor.data())), tensor.nbytes())); + std::map data_views; + for (size_t i = 0; i < tensors.size(); ++i) { + data_views.insert({std::to_string(i), + MemoryView(const_cast(reinterpret_cast(tensors[i].data())), tensors[i].nbytes())}); } hailo_nms_info_t nms_info = { @@ -64,7 +92,9 @@ public: auto buffer = Buffer::create(HailoRTCommon::get_nms_host_frame_size(nms_info, output_format), 0); VALIDATE_STATUS(buffer.status()); - auto status = self.m_post_processing_op.execute(data_views, MemoryView(buffer.value().data(), buffer.value().size())); + std::map outputs; + outputs.insert({"", MemoryView(buffer.value().data(), buffer.value().size())}); + auto status = self.m_post_processing_op->execute(data_views, outputs); VALIDATE_STATUS(status); // Note: The ownership of the buffer is transferred to Python wrapped as a py::array. @@ -80,19 +110,19 @@ public: } private: - YOLOv5PostProcessingOpWrapper(YOLOv5PostProcessingOp &&post_processing_op, uint32_t num_of_classes, uint32_t max_bboxes) + YOLOv5PostProcessOpWrapper(std::shared_ptr post_processing_op, uint32_t num_of_classes, uint32_t max_bboxes) : m_post_processing_op(post_processing_op), m_num_of_classes(num_of_classes), m_max_boxes(max_bboxes) {} - YOLOv5PostProcessingOp m_post_processing_op; + std::shared_ptr m_post_processing_op; uint32_t m_num_of_classes = 0; uint32_t m_max_boxes = 0; }; void NetFlow_api_initialize_python_module(py::module &m) { - YOLOv5PostProcessingOpWrapper::add_to_python_module(m); + YOLOv5PostProcessOpWrapper::add_to_python_module(m); } diff --git a/hailort/libhailort/bindings/python/src/pyhailort.cpp b/hailort/libhailort/bindings/python/src/pyhailort.cpp index 21b11f9..2890c91 100644 --- a/hailort/libhailort/bindings/python/src/pyhailort.cpp +++ b/hailort/libhailort/bindings/python/src/pyhailort.cpp @@ -8,7 +8,8 @@ #include using namespace std; -#include "hailo/hailort.hpp" +#include "hailo/hailort.h" +#include "hailo/hailort_defaults.hpp" #include "hef_api.hpp" #include "vstream_api.hpp" @@ -23,7 +24,6 @@ using namespace std; #include "bindings_common.hpp" #include "sensor_config_exports.h" -#include "hailort_defaults.hpp" #if defined(__GNUC__) #include "common/os/posix/traffic_control.hpp" #endif @@ -44,6 +44,7 @@ bool hailo_format_equals(hailo_format_t &first, hailo_format_t &second){ (first.order == second.order) && (first.flags == second.flags)); } + class UdpScan { public: UdpScan() = default; @@ -231,7 +232,7 @@ PYBIND11_MODULE(_pyhailort, m) { .value("HAILO8_A0", HAILO_ARCH_HAILO8_A0) .value("HAILO8", HAILO_ARCH_HAILO8) .value("HAILO8L", HAILO_ARCH_HAILO8L) - .value("MERCURY_CA", HAILO_ARCH_MERCURY_CA) + .value("HAILO15", HAILO_ARCH_HAILO15) ; /* TODO: SDK-15648 */ @@ -539,6 +540,9 @@ PYBIND11_MODULE(_pyhailort, m) { .value("YYUV", HAILO_FORMAT_ORDER_HAILO_YYUV) .value("NV21", HAILO_FORMAT_ORDER_NV21) .value("YYVU", HAILO_FORMAT_ORDER_HAILO_YYVU) + .value("RGB4", HAILO_FORMAT_ORDER_RGB4) + .value("I420", HAILO_FORMAT_ORDER_I420) + .value("YYYYUV", HAILO_FORMAT_ORDER_HAILO_YYYYUV) ; py::enum_(m, "FormatFlags", py::arithmetic()) @@ -727,11 +731,11 @@ PYBIND11_MODULE(_pyhailort, m) { .def(py::init<>()) ; - py::class_(m, "CoreInputStreamParams") + py::class_(m, "IntegratedInputStreamParams") .def(py::init<>()) ; - py::class_(m, "CoreOutputStreamParams") + py::class_(m, "IntegratedOutputStreamParams") .def(py::init<>()) ; @@ -746,7 +750,7 @@ PYBIND11_MODULE(_pyhailort, m) { py::enum_(m, "StreamInterface") .value("PCIe", HAILO_STREAM_INTERFACE_PCIE) - .value("CORE", HAILO_STREAM_INTERFACE_CORE) + .value("INTEGRATED", HAILO_STREAM_INTERFACE_INTEGRATED) .value("ETH", HAILO_STREAM_INTERFACE_ETH) .value("MIPI", HAILO_STREAM_INTERFACE_MIPI) ; @@ -772,12 +776,18 @@ PYBIND11_MODULE(_pyhailort, m) { py::class_(m, "ActivateNetworkGroupParams") .def(py::init<>()) .def_static("default", []() { - return HailoRTDefaults::get_network_group_params(); + return HailoRTDefaults::get_active_network_group_params(); }); ; + py::enum_(m, "SchedulingAlgorithm") + .value("NONE", HAILO_SCHEDULING_ALGORITHM_NONE) + .value("ROUND_ROBIN", HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN) + ; + py::class_(m, "VDeviceParams") .def(py::init<>()) + // Add device_ids .def_property("device_count", [](const VDeviceParamsWrapper& params) -> uint32_t { return params.orig_params.device_count; @@ -786,6 +796,14 @@ PYBIND11_MODULE(_pyhailort, m) { params.orig_params.device_count = device_count; } ) + .def_property("scheduling_algorithm", + [](const VDeviceParamsWrapper& params) -> uint32_t { + return params.orig_params.scheduling_algorithm; + }, + [](VDeviceParamsWrapper& params, hailo_scheduling_algorithm_t scheduling_algorithm) { + params.orig_params.scheduling_algorithm = scheduling_algorithm; + } + ) .def_property("group_id", [](const VDeviceParamsWrapper& params) -> py::str { return std::string(params.orig_params.group_id); @@ -795,6 +813,14 @@ PYBIND11_MODULE(_pyhailort, m) { params.orig_params.group_id = params.group_id_str.c_str(); } ) + .def_property("multi_process_service", + [](const VDeviceParamsWrapper& params) -> uint32_t { + return params.orig_params.multi_process_service; + }, + [](VDeviceParamsWrapper& params, bool multi_process_service) { + params.orig_params.multi_process_service = multi_process_service; + } + ) .def_static("default", []() { auto orig_params = HailoRTDefaults::get_vdevice_params(); orig_params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_NONE; @@ -808,8 +834,8 @@ PYBIND11_MODULE(_pyhailort, m) { .def_readonly("direction", &hailo_stream_parameters_t::direction) STREAM_PARAMETERS_UNION_PROPERTY(pcie_input_params, hailo_pcie_input_stream_params_t, HAILO_STREAM_INTERFACE_PCIE, HAILO_H2D_STREAM) - STREAM_PARAMETERS_UNION_PROPERTY(core_input_params, hailo_core_input_stream_params_t, - HAILO_STREAM_INTERFACE_CORE, HAILO_H2D_STREAM) + STREAM_PARAMETERS_UNION_PROPERTY(integrated_input_params, hailo_integrated_input_stream_params_t, + HAILO_STREAM_INTERFACE_INTEGRATED, HAILO_H2D_STREAM) STREAM_PARAMETERS_UNION_PROPERTY(eth_input_params, hailo_eth_input_stream_params_t, HAILO_STREAM_INTERFACE_ETH, HAILO_H2D_STREAM) STREAM_PARAMETERS_UNION_PROPERTY(mipi_input_params, hailo_mipi_input_stream_params_t, @@ -818,8 +844,8 @@ PYBIND11_MODULE(_pyhailort, m) { HAILO_STREAM_INTERFACE_PCIE, HAILO_D2H_STREAM) STREAM_PARAMETERS_UNION_PROPERTY(eth_output_params, hailo_eth_output_stream_params_t, HAILO_STREAM_INTERFACE_ETH, HAILO_D2H_STREAM) - STREAM_PARAMETERS_UNION_PROPERTY(core_output_params, hailo_core_output_stream_params_t, - HAILO_STREAM_INTERFACE_CORE, HAILO_D2H_STREAM) + STREAM_PARAMETERS_UNION_PROPERTY(integrated_output_params, hailo_integrated_output_stream_params_t, + HAILO_STREAM_INTERFACE_INTEGRATED, HAILO_D2H_STREAM) ; py::class_(m, "NetworkParameters") @@ -1058,6 +1084,7 @@ PYBIND11_MODULE(_pyhailort, m) { .def_readonly("name", &hailo_stream_info_t::name) .def_readonly("sys_index", &hailo_stream_info_t::index) .def_readonly("data_bytes", &hailo_stream_info_t::hw_data_bytes) + .def_readonly("quant_info", &hailo_stream_info_t::quant_info) .def("__repr__", [](const hailo_stream_info_t &self) { return std::string("StreamInfo(\"") + std::string(self.name) + std::string("\")"); }) diff --git a/hailort/libhailort/bindings/python/src/quantization_api.cpp b/hailort/libhailort/bindings/python/src/quantization_api.cpp index 12658d8..9d6c8cd 100644 --- a/hailort/libhailort/bindings/python/src/quantization_api.cpp +++ b/hailort/libhailort/bindings/python/src/quantization_api.cpp @@ -7,6 +7,8 @@ * @brief Quantization python bindings functions **/ +#include "hailo/quantization.hpp" + #include "quantization_api.hpp" #include "bindings_common.hpp" diff --git a/hailort/libhailort/bindings/python/src/quantization_api.hpp b/hailort/libhailort/bindings/python/src/quantization_api.hpp index 85242d7..f28f628 100644 --- a/hailort/libhailort/bindings/python/src/quantization_api.hpp +++ b/hailort/libhailort/bindings/python/src/quantization_api.hpp @@ -10,12 +10,14 @@ #ifndef _HAILO_QUANTIZATION_API_HPP_ #define _HAILO_QUANTIZATION_API_HPP_ -#include "hailo/hailort.hpp" +#include "hailo/hailort.h" + #include "utils.hpp" #include #include + namespace hailort { diff --git a/hailort/libhailort/bindings/python/src/utils.hpp b/hailort/libhailort/bindings/python/src/utils.hpp index e0498f5..e5356e1 100644 --- a/hailort/libhailort/bindings/python/src/utils.hpp +++ b/hailort/libhailort/bindings/python/src/utils.hpp @@ -40,10 +40,10 @@ class HailoRTStatusException : public HailoRTException { } \ } while (0) -#define VALIDATE_NOT_NULL(__ptr) \ +#define VALIDATE_NOT_NULL(__ptr, __status) \ do { \ if (nullptr == (__ptr)) { \ - throw HailoRTStatusException(std::to_string(HAILO_INVALID_ARGUMENT)); \ + throw HailoRTStatusException(std::to_string(__status)); \ } \ } while (0) diff --git a/hailort/libhailort/bindings/python/src/vdevice_api.hpp b/hailort/libhailort/bindings/python/src/vdevice_api.hpp index 746b72d..02a7ee6 100644 --- a/hailort/libhailort/bindings/python/src/vdevice_api.hpp +++ b/hailort/libhailort/bindings/python/src/vdevice_api.hpp @@ -14,10 +14,16 @@ #include "hailo/hef.hpp" #include "hailo/vdevice.hpp" +#include "hailo/hailort_common.hpp" -#include "utils.hpp" #include "common/logger_macros.hpp" +#ifdef HAILO_SUPPORT_MULTI_PROCESS +#include "service/rpc_client_utils.hpp" +#endif // HAILO_SUPPORT_MULTI_PROCESS + +#include "utils.hpp" + #include #include #include @@ -27,6 +33,7 @@ #include + namespace hailort { @@ -47,6 +54,19 @@ public: return VDeviceWrapper(params.orig_params); } + static VDeviceWrapper create(const VDeviceParamsWrapper ¶ms, const std::vector &device_ids) + { + if (params.orig_params.device_ids != nullptr && (!device_ids.empty())) { + LOGGER__ERROR("VDevice device_ids can be set in params or device_ids argument. Both parameters were passed to the c'tor"); + throw HailoRTStatusException(std::to_string(HAILO_INVALID_OPERATION)); + } + auto modified_params = params; + auto device_ids_vector = HailoRTCommon::to_device_ids_vector(device_ids); + VALIDATE_EXPECTED(device_ids_vector); + modified_params.orig_params.device_ids = device_ids_vector->data(); + return VDeviceWrapper(modified_params.orig_params); + } + static VDeviceWrapper create_from_ids(const std::vector &device_ids) { auto device_ids_vector = HailoRTCommon::to_device_ids_vector(device_ids); @@ -87,8 +107,10 @@ public: VALIDATE_EXPECTED(network_groups); py::list results; + m_net_groups.reserve(m_net_groups.size() + network_groups->size()); for (const auto &network_group : network_groups.value()) { results.append(network_group.get()); + m_net_groups.emplace_back(network_group); } return results; @@ -96,11 +118,43 @@ public: void release() { + m_net_groups.clear(); m_vdevice.reset(); } + void before_fork() + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + if (m_vdevice != nullptr) { + auto status = m_vdevice->before_fork(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + + void after_fork_in_parent() + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + if (m_vdevice != nullptr) { + auto status = m_vdevice->after_fork_in_parent(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + + void after_fork_in_child() + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + if (m_vdevice != nullptr) { + auto status = m_vdevice->after_fork_in_child(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + private: std::unique_ptr m_vdevice; + ConfiguredNetworkGroupVector m_net_groups; }; void VDevice_api_initialize_python_module(py::module &m) @@ -108,10 +162,14 @@ void VDevice_api_initialize_python_module(py::module &m) py::class_(m, "VDevice") .def("create", py::overload_cast(&VDeviceWrapper::create)) .def("create", py::overload_cast(&VDeviceWrapper::create)) + .def("create", py::overload_cast&>(&VDeviceWrapper::create)) .def("create_from_ids", &VDeviceWrapper::create_from_ids) .def("get_physical_devices_ids", &VDeviceWrapper::get_physical_devices_ids) .def("configure", &VDeviceWrapper::configure) .def("release", &VDeviceWrapper::release) + .def("before_fork", &VDeviceWrapper::before_fork) + .def("after_fork_in_parent", &VDeviceWrapper::after_fork_in_parent) + .def("after_fork_in_child", &VDeviceWrapper::after_fork_in_child) ; } diff --git a/hailort/libhailort/bindings/python/src/vstream_api.cpp b/hailort/libhailort/bindings/python/src/vstream_api.cpp index 8d88b84..a1b651b 100644 --- a/hailort/libhailort/bindings/python/src/vstream_api.cpp +++ b/hailort/libhailort/bindings/python/src/vstream_api.cpp @@ -27,6 +27,36 @@ void InputVStreamWrapper::add_to_python_module(py::module &m) MemoryView(const_cast(reinterpret_cast(data.data())), data.nbytes())); VALIDATE_STATUS(status); }) + .def("before_fork", [](InputVStream &self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.before_fork(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + ) + .def("after_fork_in_parent", [](InputVStream &self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.after_fork_in_parent(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + ) + .def("after_fork_in_child", [](InputVStream &self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.after_fork_in_child(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + ) .def("flush", [](InputVStream &self) { hailo_status status = self.flush(); @@ -100,6 +130,36 @@ void InputVStreamsWrapper::clear() VALIDATE_STATUS(status); } +void InputVStreamsWrapper::before_fork() +{ +#ifdef HAILO_SUPPORT_MULTI_PROCESS + for (auto &input_vstream : m_input_vstreams) { + auto status = input_vstream.second->before_fork(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS +} + +void InputVStreamsWrapper::after_fork_in_parent() +{ +#ifdef HAILO_SUPPORT_MULTI_PROCESS + for (auto &input_vstream : m_input_vstreams) { + auto status = input_vstream.second->after_fork_in_parent(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS +} + +void InputVStreamsWrapper::after_fork_in_child() +{ +#ifdef HAILO_SUPPORT_MULTI_PROCESS + for (auto &input_vstream : m_input_vstreams) { + auto status = input_vstream.second->after_fork_in_child(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS +} + void InputVStreamsWrapper::add_to_python_module(py::module &m) { py::class_(m, "InputVStreams") @@ -109,6 +169,9 @@ void InputVStreamsWrapper::add_to_python_module(py::module &m) .def("clear", &InputVStreamsWrapper::clear) .def("__enter__", &InputVStreamsWrapper::enter, py::return_value_policy::reference) .def("__exit__", [&](InputVStreamsWrapper &self, py::args) { self.exit(); }) + .def("before_fork", &InputVStreamsWrapper::before_fork) + .def("after_fork_in_parent", &InputVStreamsWrapper::after_fork_in_parent) + .def("after_fork_in_child", &InputVStreamsWrapper::after_fork_in_child) ; } @@ -151,6 +214,36 @@ void OutputVStreamWrapper::add_to_python_module(py::module &m) return py::array(get_dtype(self), get_shape(self), unmanaged_addr, py::capsule(unmanaged_addr, [](void *p) { delete reinterpret_cast(p); })); }) + .def("before_fork", [](OutputVStream &self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.before_fork(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + ) + .def("after_fork_in_parent", [](OutputVStream &self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.after_fork_in_parent(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + ) + .def("after_fork_in_child", [](OutputVStream &self) + { +#ifdef HAILO_SUPPORT_MULTI_PROCESS + auto status = self.after_fork_in_child(); + VALIDATE_STATUS(status); +#else + (void)self; +#endif // HAILO_SUPPORT_MULTI_PROCESS + } + ) .def_property_readonly("info", [](OutputVStream &self) { return self.get_info(); @@ -213,6 +306,36 @@ void OutputVStreamsWrapper::clear() VALIDATE_STATUS(status); } +void OutputVStreamsWrapper::before_fork() +{ +#ifdef HAILO_SUPPORT_MULTI_PROCESS + for (auto &output_vstream : m_output_vstreams) { + auto status = output_vstream.second->before_fork(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS +} + +void OutputVStreamsWrapper::after_fork_in_parent() +{ +#ifdef HAILO_SUPPORT_MULTI_PROCESS + for (auto &output_vstream : m_output_vstreams) { + auto status = output_vstream.second->after_fork_in_parent(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS +} + +void OutputVStreamsWrapper::after_fork_in_child() +{ +#ifdef HAILO_SUPPORT_MULTI_PROCESS + for (auto &output_vstream : m_output_vstreams) { + auto status = output_vstream.second->after_fork_in_child(); + VALIDATE_STATUS(status); + } +#endif // HAILO_SUPPORT_MULTI_PROCESS +} + void OutputVStreamsWrapper::add_to_python_module(py::module &m) { py::class_(m, "OutputVStreams") @@ -222,6 +345,9 @@ void OutputVStreamsWrapper::add_to_python_module(py::module &m) .def("clear", &OutputVStreamsWrapper::clear) .def("__enter__", &OutputVStreamsWrapper::enter, py::return_value_policy::reference) .def("__exit__", [&](OutputVStreamsWrapper &self, py::args) { self.exit(); }) + .def("before_fork", &OutputVStreamsWrapper::before_fork) + .def("after_fork_in_parent", &OutputVStreamsWrapper::after_fork_in_parent) + .def("after_fork_in_child", &OutputVStreamsWrapper::after_fork_in_child) ; } diff --git a/hailort/libhailort/bindings/python/src/vstream_api.hpp b/hailort/libhailort/bindings/python/src/vstream_api.hpp index 2f38405..83f3bc3 100644 --- a/hailort/libhailort/bindings/python/src/vstream_api.hpp +++ b/hailort/libhailort/bindings/python/src/vstream_api.hpp @@ -41,6 +41,9 @@ public: std::shared_ptr get_input_by_name(const std::string &name); py::dict get_all_inputs(); void clear(); + void before_fork(); + void after_fork_in_parent(); + void after_fork_in_child(); static void add_to_python_module(py::module &m); private: @@ -67,6 +70,9 @@ public: void exit(); py::dict get_all_outputs(); void clear(); + void before_fork(); + void after_fork_in_parent(); + void after_fork_in_child(); static void add_to_python_module(py::module &m); private: diff --git a/hailort/libhailort/examples/README.md b/hailort/libhailort/examples/README.md index 0de5803..04dd107 100644 --- a/hailort/libhailort/examples/README.md +++ b/hailort/libhailort/examples/README.md @@ -26,6 +26,7 @@ The following examples are provided, demonstrating the HailoRT API: - this example uses udp device. - `raw_streams_example` - Basic inference of a shortcut network using raw stream api. - The data is transformed before sent and after received in the same thread sending/receiving using the transformation api. + - `notification_callback_example` - Demonstrates how to work with notification callbacks. - C++ examples: - `vstreams_example` - Basic inference of a shortcut network, same as `vstreams_example` C example, uses HailoRT C++ api. @@ -39,7 +40,7 @@ The following examples are provided, demonstrating the HailoRT API: - `raw_streams_example` - Basic inference of a shortcut network, same as `raw_streams_example` C example, uses HailoRT C++ api. - `multi_process_example` - Demonstrates how to work with HailoRT as a service and using the HailoRT Model Scheduler for network groups switching. Using the script `multi_process_example.sh` one can specify the number of processes to run each hef, see `multi_process_example.sh -h` for more information. - + - `notification_callback_example` - Demonstrates how to work with notification callbacks, same as `notification_callback_example` C example. ## Compiling with CMake Examples are configured and compiled using the following commands: ```sh diff --git a/hailort/libhailort/examples/c/CMakeLists.txt b/hailort/libhailort/examples/c/CMakeLists.txt index 97275f7..46e0ce7 100644 --- a/hailort/libhailort/examples/c/CMakeLists.txt +++ b/hailort/libhailort/examples/c/CMakeLists.txt @@ -9,6 +9,7 @@ add_subdirectory(switch_network_groups_example) add_subdirectory(switch_network_groups_manually_example) add_subdirectory(multi_device_example) add_subdirectory(power_measurement_example) +add_subdirectory(notification_callback_example) add_custom_target(c_hailort_examples) add_dependencies(c_hailort_examples @@ -20,4 +21,5 @@ add_dependencies(c_hailort_examples c_switch_network_groups_example c_switch_network_groups_manually_example c_multi_device_example - c_power_measurement_example) \ No newline at end of file + c_power_measurement_example + c_notification_callback_example) \ No newline at end of file diff --git a/hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt b/hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt index 12d7e71..4933fbd 100644 --- a/hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(data_quantization_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt b/hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt index 2abb3c7..e7523d7 100644 --- a/hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(infer_pipeline_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt b/hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt index f0cd0fc..9729466 100644 --- a/hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(multi_device_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt b/hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt index 0d80fdd..fd39710 100644 --- a/hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(multi_network_vstream_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt b/hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt new file mode 100644 index 0000000..0081620 --- /dev/null +++ b/hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.0.0) + +find_package(Threads REQUIRED) +set(THREADS_PREFER_PTHREAD_FLAG ON) + +find_package(HailoRT 4.13.0 EXACT REQUIRED) + +SET_SOURCE_FILES_PROPERTIES(notification_callback_example.c PROPERTIES LANGUAGE C) + +add_executable(c_notification_callback_example notification_callback_example.c) +target_link_libraries(c_notification_callback_example PRIVATE HailoRT::libhailort Threads::Threads) +target_include_directories(c_notification_callback_example PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../common") + +if(WIN32) + target_compile_options(c_notification_callback_example PRIVATE + /DWIN32_LEAN_AND_MEAN + /DNOMINMAX # NOMINMAX is required in order to play nice with std::min/std::max (otherwise Windows.h defines it's own) + /wd4201 /wd4251 + ) +endif() \ No newline at end of file diff --git a/hailort/libhailort/examples/c/notification_callback_example/notification_callback_example.c b/hailort/libhailort/examples/c/notification_callback_example/notification_callback_example.c new file mode 100644 index 0000000..9c7d15c --- /dev/null +++ b/hailort/libhailort/examples/c/notification_callback_example/notification_callback_example.c @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file notification_callback_example.c + * This example demonstrates the basic usage of notification callbacks. + * The program creates a device and then sets and removes a notification callback on it. + * In this example the notification is HAILO_NOTIFICATION_ID_HEALTH_MONITOR_OVERCURRENT_ALARM and the callback is a simple print function. + **/ + +#include "common.h" +#include "hailo_thread.h" +#include "hailo/hailort.h" + +#define DEVICE_IDS_COUNT (16) + +void sleep_seconds(uint32_t duration_seconds) +{ +#if defined(__unix__) || defined(__QNX__) + sleep(duration_seconds); +#else + Sleep(duration_seconds); +#endif +} + +void callback(hailo_device device, const hailo_notification_t *notification, void *opaque) +{ + hailo_device_id_t device_id = {0}; + hailo_status status = hailo_get_device_id(device, &device_id); + if (HAILO_SUCCESS != status){ + printf("Couldn't get device id\n"); + return; + } + printf("got a notification with notification id %d - Overcurrent Alarm\n", notification->id); + printf("device id: %s\n", device_id.id); + if(NULL == opaque) + printf("User defined data is null\n"); +} + +int main() +{ + hailo_device_id_t device_ids[DEVICE_IDS_COUNT]; + size_t actual_devices_count = DEVICE_IDS_COUNT; + hailo_status status = HAILO_UNINITIALIZED; + hailo_device device = NULL; + + // Scan to find a device + status = hailo_scan_devices(NULL, device_ids, &actual_devices_count); + REQUIRE_SUCCESS(status, l_exit, "Failed to scan devices"); + REQUIRE_ACTION(1 <= actual_devices_count, status = HAILO_INVALID_OPERATION, l_exit, + "Failed to find a connected hailo device."); + + // Create the device + status = hailo_create_device_by_id(&(device_ids[0]), &device); + REQUIRE_SUCCESS(status, l_exit, "Failed to create device"); + + // Set the callback function to the notification id HAILO_NOTIFICATION_ID_HEALTH_MONITOR_OVERCURRENT_ALARM + hailo_notification_callback callback_func = &callback; + status = hailo_set_notification_callback(device, callback_func, HAILO_NOTIFICATION_ID_HEALTH_MONITOR_OVERCURRENT_ALARM, NULL); + REQUIRE_SUCCESS(status, l_release_device, "Failed to set notification callback"); + + // In this part of the program - in case of HAILO_NOTIFICATION_ID_HEALTH_MONITOR_OVERCURRENT_ALARM notification, the callback function will be called. + printf("Notification callback has been set - "); + printf("in case of overcurrent alarm notification, an overcurrent alarm message will be printed\n"); + sleep_seconds(2); + + // Remove the callback notification + status = hailo_remove_notification_callback(device, HAILO_NOTIFICATION_ID_HEALTH_MONITOR_OVERCURRENT_ALARM); + REQUIRE_SUCCESS(status, l_release_device, "Failed to remove notification callback"); + printf("Notification callback has been removed\n"); + +l_release_device: + (void) hailo_release_device(device); +l_exit: + return status; +} diff --git a/hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt b/hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt index 664dfb2..6545a54 100644 --- a/hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(power_measurement_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt b/hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt index 583d00b..8ed9ef7 100644 --- a/hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(raw_streams_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/raw_streams_example/raw_streams_example.c b/hailort/libhailort/examples/c/raw_streams_example/raw_streams_example.c index 41fc88d..04093fd 100644 --- a/hailort/libhailort/examples/c/raw_streams_example/raw_streams_example.c +++ b/hailort/libhailort/examples/c/raw_streams_example/raw_streams_example.c @@ -16,6 +16,7 @@ #define HEF_FILE ("hefs/shortcut_net.hef") #define INFER_FRAME_COUNT (200) #define MAX_EDGE_LAYERS (16) +#define DEVICE_IDS_COUNT (16) typedef struct write_thread_args_t { hailo_input_stream *input_stream; @@ -170,8 +171,8 @@ l_cleanup: int main() { hailo_status status = HAILO_UNINITIALIZED; - hailo_device_id_t device_id = {0}; - size_t actual_devices_count = 1; + hailo_device_id_t device_ids[DEVICE_IDS_COUNT]; + size_t actual_devices_count = DEVICE_IDS_COUNT; hailo_device device = NULL; hailo_hef hef = NULL; hailo_configure_params_t configure_params = {0}; @@ -186,12 +187,12 @@ int main() size_t number_output_streams = 0; size_t index = 0; - status = hailo_scan_devices(NULL, &device_id, &actual_devices_count); + status = hailo_scan_devices(NULL, device_ids, &actual_devices_count); REQUIRE_SUCCESS(status, l_exit, "Failed to scan devices"); - REQUIRE_ACTION(1 == actual_devices_count, status = HAILO_INVALID_OPERATION, l_exit, - "Only 1 device on the system is supported on the example"); + REQUIRE_ACTION(1 <= actual_devices_count, status = HAILO_INVALID_OPERATION, l_exit, + "Failed to find a connected hailo device."); - status = hailo_create_device_by_id(&device_id, &device); + status = hailo_create_device_by_id(&(device_ids[0]), &device); REQUIRE_SUCCESS(status, l_exit, "Failed to create device"); status = hailo_create_hef_file(&hef, HEF_FILE); diff --git a/hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt b/hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt index 3517b2c..7aee572 100644 --- a/hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(switch_network_groups_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt b/hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt index 5af8813..e0768bc 100644 --- a/hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(switch_network_groups_manually_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/c/switch_network_groups_manually_example/switch_network_groups_manually_example.c b/hailort/libhailort/examples/c/switch_network_groups_manually_example/switch_network_groups_manually_example.c index 5d56190..f7872e0 100644 --- a/hailort/libhailort/examples/c/switch_network_groups_manually_example/switch_network_groups_manually_example.c +++ b/hailort/libhailort/examples/c/switch_network_groups_manually_example/switch_network_groups_manually_example.c @@ -57,7 +57,8 @@ thread_return_type input_vstream_thread_func(void *args) for (size_t run_index = 0; run_index < RUN_COUNT; run_index++) { for (size_t hef_index = 0 ; hef_index < HEF_COUNT; hef_index++) { // Wait for hef to be activated to send data - hailo_wait_for_network_group_activation(input_vstream_args->configured_networks[hef_index], HAILO_INFINITE); + status = hailo_wait_for_network_group_activation(input_vstream_args->configured_networks[hef_index], HAILO_INFINITE); + REQUIRE_SUCCESS(status, l_clear_src, "Failed waiting for network group activation"); // Send data on relevant Hef for (uint32_t frame = 0; frame < INFER_FRAME_COUNT; frame++) { @@ -106,9 +107,10 @@ thread_return_type output_vstream_thread_func(void *args) for (size_t run_index = 0; run_index < RUN_COUNT; run_index++) { for (size_t hef_index = 0 ; hef_index < HEF_COUNT; hef_index++) { - // Wait for hef to be activated to send data - hailo_wait_for_network_group_activation(output_vstream_args->configured_networks[hef_index], HAILO_INFINITE); - + // Wait for hef to be activated to recv data + status = hailo_wait_for_network_group_activation(output_vstream_args->configured_networks[hef_index], HAILO_INFINITE); + REQUIRE_SUCCESS(status, l_clear_dst, "Failed waiting for network group activation"); + for (uint32_t i = 0; i < INFER_FRAME_COUNT; i++) { // Read data status = hailo_vstream_read_raw_buffer(output_vstreams[hef_index], diff --git a/hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt b/hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt index ee34cc5..1c32dfc 100644 --- a/hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt +++ b/hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) SET_SOURCE_FILES_PROPERTIES(vstreams_example.c PROPERTIES LANGUAGE C) diff --git a/hailort/libhailort/examples/cpp/CMakeLists.txt b/hailort/libhailort/examples/cpp/CMakeLists.txt index 4dad532..10eea7a 100644 --- a/hailort/libhailort/examples/cpp/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/CMakeLists.txt @@ -3,21 +3,25 @@ cmake_minimum_required(VERSION 3.0.0) add_subdirectory(vstreams_example) add_subdirectory(infer_pipeline_example) add_subdirectory(raw_streams_example) +add_subdirectory(raw_async_streams_example) add_subdirectory(multi_network_vstream_example) add_subdirectory(switch_network_groups_example) add_subdirectory(switch_network_groups_manually_example) add_subdirectory(multi_device_example) add_subdirectory(power_measurement_example) add_subdirectory(multi_process_example) +add_subdirectory(notification_callback_example) add_custom_target(cpp_hailort_examples) add_dependencies(cpp_hailort_examples cpp_vstreams_example cpp_infer_pipeline_example cpp_raw_streams_example + cpp_raw_async_streams_example cpp_multi_network_vstream_example cpp_switch_network_groups_example cpp_switch_network_groups_manually_example cpp_multi_device_example cpp_power_measurement_example - cpp_multi_process_example) \ No newline at end of file + cpp_multi_process_example + cpp_notification_callback_example) \ No newline at end of file diff --git a/hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt index 859bb5a..9f13d63 100644 --- a/hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_infer_pipeline_example infer_pipeline_example.cpp) target_link_libraries(cpp_infer_pipeline_example PRIVATE HailoRT::libhailort) diff --git a/hailort/libhailort/examples/cpp/infer_pipeline_example/infer_pipeline_example.cpp b/hailort/libhailort/examples/cpp/infer_pipeline_example/infer_pipeline_example.cpp index 3327bf8..02d7d13 100644 --- a/hailort/libhailort/examples/cpp/infer_pipeline_example/infer_pipeline_example.cpp +++ b/hailort/libhailort/examples/cpp/infer_pipeline_example/infer_pipeline_example.cpp @@ -13,6 +13,7 @@ #include + #define HEF_FILE ("hefs/shortcut_net.hef") constexpr size_t FRAMES_COUNT = 100; constexpr hailo_format_type_t FORMAT_TYPE = HAILO_FORMAT_TYPE_AUTO; @@ -28,7 +29,7 @@ Expected> configure_network_group(Device return make_unexpected(hef.status()); } - auto configure_params = hef->create_configure_params(HAILO_STREAM_INTERFACE_ETH); + auto configure_params = device.create_configure_params(hef.value()); if (!configure_params) { return make_unexpected(configure_params.status()); } diff --git a/hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt index 1eb6cf8..99fd22e 100644 --- a/hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_multi_device_example multi_device_example.cpp) target_link_libraries(cpp_multi_device_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/multi_device_example/multi_device_example.cpp b/hailort/libhailort/examples/cpp/multi_device_example/multi_device_example.cpp index c0334bc..2d8b9c6 100644 --- a/hailort/libhailort/examples/cpp/multi_device_example/multi_device_example.cpp +++ b/hailort/libhailort/examples/cpp/multi_device_example/multi_device_example.cpp @@ -10,6 +10,8 @@ #include "hailo/hailort.hpp" #include +#include + #define HEF_FILE ("hefs/shortcut_net.hef") constexpr size_t FRAMES_COUNT = 100; @@ -26,7 +28,7 @@ Expected> configure_network_group(VDevic return make_unexpected(hef.status()); } - auto configure_params = hef->create_configure_params(HAILO_STREAM_INTERFACE_PCIE); + auto configure_params = vdevice.create_configure_params(hef.value()); if (!configure_params) { return make_unexpected(configure_params.status()); } diff --git a/hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt index 1c384bc..dadc087 100644 --- a/hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_multi_network_vstream_example multi_network_vstream_example.cpp) target_link_libraries(cpp_multi_network_vstream_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/multi_network_vstream_example/multi_network_vstream_example.cpp b/hailort/libhailort/examples/cpp/multi_network_vstream_example/multi_network_vstream_example.cpp index 493393c..078bba2 100644 --- a/hailort/libhailort/examples/cpp/multi_network_vstream_example/multi_network_vstream_example.cpp +++ b/hailort/libhailort/examples/cpp/multi_network_vstream_example/multi_network_vstream_example.cpp @@ -10,6 +10,8 @@ #include "hailo/hailort.hpp" #include +#include + #define HEF_FILE ("hefs/multi_network_shortcut_net.hef") constexpr size_t INFER_FRAME_COUNT = 100; @@ -27,7 +29,7 @@ using InOutVStreams = std::pair, std::vector> configure_network_group(VDevice &vdevice, Hef &hef, uint16_t batch_size[NET_COUNT]) { - auto configure_params = hef.create_configure_params(HAILO_STREAM_INTERFACE_PCIE); + auto configure_params = vdevice.create_configure_params(hef); if (!configure_params) { std::cerr << "Failed to create configure params" << std::endl; return make_unexpected(configure_params.status()); diff --git a/hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt index c1c5f00..6dc1069 100644 --- a/hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_multi_process_example multi_process_example.cpp) target_link_libraries(cpp_multi_process_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/multi_process_example/multi_process_example.cpp b/hailort/libhailort/examples/cpp/multi_process_example/multi_process_example.cpp index bb5323d..2ff63bd 100644 --- a/hailort/libhailort/examples/cpp/multi_process_example/multi_process_example.cpp +++ b/hailort/libhailort/examples/cpp/multi_process_example/multi_process_example.cpp @@ -13,6 +13,8 @@ #include "hailo/hailort.hpp" #include +#include + constexpr size_t FRAMES_COUNT = 100; constexpr bool QUANTIZED = true; @@ -29,7 +31,7 @@ Expected> configure_network_group(const return make_unexpected(hef.status()); } - auto configure_params = hef->create_configure_params(HAILO_STREAM_INTERFACE_PCIE); + auto configure_params = vdevice.create_configure_params(hef.value()); if (!configure_params) { return make_unexpected(configure_params.status()); } diff --git a/hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt new file mode 100644 index 0000000..8d581c7 --- /dev/null +++ b/hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.0.0) + +find_package(HailoRT 4.13.0 EXACT REQUIRED) + +add_executable(cpp_notification_callback_example notification_callback_example.cpp) +target_link_libraries(cpp_notification_callback_example PRIVATE HailoRT::libhailort) + +if(WIN32) + target_compile_options(cpp_notification_callback_example PRIVATE + /DWIN32_LEAN_AND_MEAN + /DNOMINMAX # NOMINMAX is required in order to play nice with std::min/std::max (otherwise Windows.h defines it's own) + /wd4201 /wd4251 + ) +endif() + +set_target_properties(cpp_notification_callback_example PROPERTIES CXX_STANDARD 14) diff --git a/hailort/libhailort/examples/cpp/notification_callback_example/notification_callback_example.cpp b/hailort/libhailort/examples/cpp/notification_callback_example/notification_callback_example.cpp new file mode 100644 index 0000000..5bfff65 --- /dev/null +++ b/hailort/libhailort/examples/cpp/notification_callback_example/notification_callback_example.cpp @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file notification_callback_example.cpp + * This example demonstrates the basic usage of notification callbacks. + * The program creates a device and then sets and removes a notification callback on it. + * In this example the notification is HAILO_NOTIFICATION_ID_HEALTH_MONITOR_OVERCURRENT_ALARM and the callback is a simple print function. + **/ + +#include "hailo/hailort.hpp" + +#include +#include +#include +#include + + +const std::chrono::seconds SLEEP_DURATION_SECS(2); + +using namespace hailort; + +int main() +{ + auto device_ids = Device::scan(); + if (!device_ids) { + std::cerr << "Failed to scan, status = " << device_ids.status() << std::endl; + return device_ids.status(); + } + if (device_ids->size() < 1){ + std::cerr << "Failed to find a connected hailo device." << std::endl; + return HAILO_INVALID_OPERATION; + } + auto device = Device::create(device_ids->at(0)); + if (!device) { + std::cerr << "Failed to create device " << device.status() << std::endl; + return device.status(); + } + + // Set the callback notification + hailo_status status = device.value()->set_notification_callback( + [] (Device &device, const hailo_notification_t ¬ification, void* opaque) { + std::cout << "got notification with notification id " << notification.id << " - Overcurrent Alarm" << std::endl; + std::cout << "device id: " << device.get_dev_id() << std::endl; + if(nullptr == opaque) + std::cout << "User defined data is null" << std::endl; + }, + HAILO_NOTIFICATION_ID_HEALTH_MONITOR_OVERCURRENT_ALARM, nullptr); + if (HAILO_SUCCESS != status) { + std::cerr << "Setting notification failed " << status << std::endl; + return status; + } + + std::cout << "Notification callback has been set - "; + std::cout << "in case of overcurrent alarm notification, an overcurrent alarm will be printed" << std::endl; + std::this_thread::sleep_for(SLEEP_DURATION_SECS); + + // Remove the callback notification + status = device.value()->remove_notification_callback(HAILO_NOTIFICATION_ID_HEALTH_MONITOR_OVERCURRENT_ALARM); + if (HAILO_SUCCESS != status) { + std::cerr << "Removing notification failed " << status << std::endl; + return status; + } + std::cout << "Notification callback has been removed" << std::endl; + + return HAILO_SUCCESS; +} \ No newline at end of file diff --git a/hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt index e3ec915..0f412ee 100644 --- a/hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0.0) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_power_measurement_example power_measurement_example.cpp) target_link_libraries(cpp_power_measurement_example PRIVATE HailoRT::libhailort) diff --git a/hailort/libhailort/examples/cpp/power_measurement_example/power_measurement_example.cpp b/hailort/libhailort/examples/cpp/power_measurement_example/power_measurement_example.cpp index f3d3d18..431d33c 100644 --- a/hailort/libhailort/examples/cpp/power_measurement_example/power_measurement_example.cpp +++ b/hailort/libhailort/examples/cpp/power_measurement_example/power_measurement_example.cpp @@ -13,6 +13,7 @@ #include #include + #define SAMPLING_PERIOD (HAILO_SAMPLING_PERIOD_1100US) #define AVERAGE_FACTOR (HAILO_AVERAGE_FACTOR_256) #define DVM_OPTION (HAILO_DVM_OPTIONS_AUTO) // For current measurement over EVB - pass DVM explicitly (see hailo_dvm_options_t) diff --git a/hailort/libhailort/examples/cpp/raw_async_streams_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/raw_async_streams_example/CMakeLists.txt new file mode 100644 index 0000000..b453e64 --- /dev/null +++ b/hailort/libhailort/examples/cpp/raw_async_streams_example/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.0.0) + +find_package(Threads REQUIRED) +set(THREADS_PREFER_PTHREAD_FLAG ON) + +find_package(HailoRT 4.13.0 EXACT REQUIRED) + +add_executable(cpp_raw_async_streams_example buffer_pool.cpp raw_async_streams_example.cpp) +target_link_libraries(cpp_raw_async_streams_example PRIVATE HailoRT::libhailort Threads::Threads) + +if(WIN32) + target_compile_options(cpp_raw_async_streams_example PRIVATE + /DWIN32_LEAN_AND_MEAN + /DNOMINMAX # NOMINMAX is required in order to play nice with std::min/std::max (otherwise Windows.h defines it's own) + /wd4201 /wd4251 + ) +endif() + +set_target_properties(cpp_raw_async_streams_example PROPERTIES CXX_STANDARD 14) \ No newline at end of file diff --git a/hailort/libhailort/examples/cpp/raw_async_streams_example/buffer_pool.cpp b/hailort/libhailort/examples/cpp/raw_async_streams_example/buffer_pool.cpp new file mode 100644 index 0000000..706fbe8 --- /dev/null +++ b/hailort/libhailort/examples/cpp/raw_async_streams_example/buffer_pool.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file buffer_pool.cpp + * @brief Implementation of vdma buffer pool + **/ + +#include "buffer_pool.hpp" +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +Expected BufferPool::create(size_t num_buffers, size_t buffer_size, + hailo_vdma_buffer_direction_flags_t data_direction_flags, Device &device) +{ + std::queue> queue; + for (auto i = 0; i < num_buffers; i++) { + auto mapped_buffer = DmaMappedBuffer::create(buffer_size, data_direction_flags, device); + if (!mapped_buffer) { + return make_unexpected(mapped_buffer.status()); + } + + auto mapped_buffer_ptr = std::make_shared(mapped_buffer.release()); + if (nullptr == mapped_buffer_ptr) { + return make_unexpected(HAILO_OUT_OF_HOST_MEMORY); + } + + queue.push(mapped_buffer_ptr); + } + + auto result = std::make_shared(num_buffers, std::move(queue)); + if (nullptr == result) { + return make_unexpected(HAILO_OUT_OF_HOST_MEMORY); + } + + return result; +} + +BufferPool::BufferPool(size_t max_size, std::queue> &&queue) : + m_max_size(max_size), + m_mutex(), + m_cv(), + m_queue(queue) +{} + +BufferPool::~BufferPool() +{ + m_cv.notify_all(); +} + +std::shared_ptr BufferPool::dequeue() +{ + std::unique_lock lock(m_mutex); + m_cv.wait(lock, [this] { return m_queue.size() > 0; }); + auto buffer = m_queue.front(); + m_queue.pop(); + + return buffer; +} +void BufferPool::enqueue(std::shared_ptr buffer) +{ + { + std::unique_lock lock(m_mutex); + m_cv.wait(lock, [this] { return m_max_size > m_queue.size(); }); + m_queue.push(buffer); + } + + m_cv.notify_one(); +} + +void BufferPool::wait_for_pending_buffers() +{ + std::unique_lock lock(m_mutex); + m_cv.wait(lock, [this] { return m_max_size == m_queue.size(); }); +} diff --git a/hailort/libhailort/examples/cpp/raw_async_streams_example/buffer_pool.hpp b/hailort/libhailort/examples/cpp/raw_async_streams_example/buffer_pool.hpp new file mode 100644 index 0000000..4ff5f63 --- /dev/null +++ b/hailort/libhailort/examples/cpp/raw_async_streams_example/buffer_pool.hpp @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file buffer_pool.hpp + * @brief Pool of vdma mapped buffers, allowing FIFO queue access to buffers + **/ + +#ifndef _HAILO_BUFFER_POOL_HPP_ +#define _HAILO_BUFFER_POOL_HPP_ + +#include "hailo/hailort.hpp" +#include "hailo/expected.hpp" + +#include +#include +#include +#include + + +using namespace hailort; + +class BufferPool; +using BufferPoolPtr = std::shared_ptr; + +class BufferPool final +{ +public: + static Expected create(size_t num_buffers, size_t buffer_size, + hailo_vdma_buffer_direction_flags_t data_direction_flags, Device &device); + BufferPool(size_t max_size, std::queue> &&queue); + BufferPool(BufferPool &&) = delete; + BufferPool(const BufferPool &) = delete; + BufferPool &operator=(BufferPool &&) = delete; + BufferPool &operator=(const BufferPool &) = delete; + ~BufferPool(); + + std::shared_ptr dequeue(); + void enqueue(std::shared_ptr buffer); + void wait_for_pending_buffers(); + +private: + const size_t m_max_size; + std::mutex m_mutex; + std::condition_variable m_cv; + std::queue> m_queue; +}; + +#endif /* _HAILO_BUFFER_POOL_HPP_ */ diff --git a/hailort/libhailort/examples/cpp/raw_async_streams_example/raw_async_streams_example.cpp b/hailort/libhailort/examples/cpp/raw_async_streams_example/raw_async_streams_example.cpp new file mode 100644 index 0000000..fa0f1b4 --- /dev/null +++ b/hailort/libhailort/examples/cpp/raw_async_streams_example/raw_async_streams_example.cpp @@ -0,0 +1,146 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file raw_async_streams_example + * This example demonstrates using low level async streams over c++ + **/ + +#include "hailo/hailort.hpp" +#include "buffer_pool.hpp" + +#include +#include + + +constexpr size_t FRAMES_COUNT = 10000; +constexpr size_t BUFFER_POOL_SIZE = 10; +constexpr auto TIMEOUT = std::chrono::milliseconds(1000); + +using namespace hailort; + +Expected> configure_network_group(Device &device, const std::string &hef_path) +{ + auto hef = Hef::create(hef_path); + if (!hef) { + return make_unexpected(hef.status()); + } + + auto configure_params = device.create_configure_params(hef.value()); + if (!configure_params) { + return make_unexpected(configure_params.status()); + } + + // change stream_params here + for (auto &ng_name_params_pair : *configure_params) { + for (auto &stream_params_name_pair : ng_name_params_pair.second.stream_params_by_name) { + stream_params_name_pair.second.flags = HAILO_STREAM_FLAGS_ASYNC; + } + } + + auto network_groups = device.configure(hef.value(), configure_params.value()); + if (!network_groups) { + return make_unexpected(network_groups.status()); + } + + if (1 != network_groups->size()) { + std::cerr << "Invalid amount of network groups" << std::endl; + return make_unexpected(HAILO_INTERNAL_FAILURE); + } + + return std::move(network_groups->at(0)); +} + +void read_all(OutputStream &output, BufferPoolPtr buffer_pool, size_t frames_to_read, hailo_status &status) +{ + for (size_t i = 0; i < frames_to_read; i++) { + status = output.wait_for_ready(output.get_frame_size(), TIMEOUT); + if (HAILO_SUCCESS != status) { + return; + } + status = output.read_async(buffer_pool->dequeue(), + [buffer_pool](std::shared_ptr buffer, const hailo_async_transfer_completion_info_t &, void *) { + buffer_pool->enqueue(buffer); + }); + if (HAILO_SUCCESS != status) { + return; + } + } +} + +void write_all(InputStream &input, BufferPoolPtr buffer_pool, size_t frames_to_write, hailo_status &status) +{ + for (size_t i = 0; i < frames_to_write; i++) { + status = input.wait_for_ready(input.get_frame_size(), TIMEOUT); + if (HAILO_SUCCESS != status) { + return; + } + status = input.write_async(buffer_pool->dequeue(), + [buffer_pool](std::shared_ptr buffer, const hailo_async_transfer_completion_info_t &, void *) { + buffer_pool->enqueue(buffer); + }); + if (HAILO_SUCCESS != status) { + return; + } + } +} + +int main() +{ + auto device = Device::create(); + if (!device) { + std::cerr << "Failed create device " << device.status() << std::endl; + return device.status(); + } + + static const auto HEF_FILE = "hefs/shortcut_net.hef"; + auto network_group = configure_network_group(*device.value(), HEF_FILE); + if (!network_group) { + std::cerr << "Failed to configure network group" << HEF_FILE << std::endl; + return network_group.status(); + } + + auto activated_network_group = network_group.value()->activate(); + if (!activated_network_group) { + std::cerr << "Failed to activate network group " << activated_network_group.status() << std::endl; + return activated_network_group.status(); + } + + // Assume one input and output + auto output = network_group->get()->get_output_streams()[0]; + auto input = network_group->get()->get_input_streams()[0]; + + auto output_buffer_pool = BufferPool::create(BUFFER_POOL_SIZE, output.get().get_frame_size(), HAILO_VDMA_BUFFER_DIRECTION_FLAGS_D2H, *device.value()); + if (!output_buffer_pool) { + std::cerr << "Failed to create output buffer pool" << std::endl; + return output_buffer_pool.status(); + } + hailo_status output_status = HAILO_UNINITIALIZED; + auto output_thread = std::make_unique(read_all, output, output_buffer_pool.value(), FRAMES_COUNT, std::ref(output_status)); + + auto input_buffer_pool = BufferPool::create(BUFFER_POOL_SIZE, input.get().get_frame_size(), HAILO_VDMA_BUFFER_DIRECTION_FLAGS_H2D, *device.value()); + if (!input_buffer_pool) { + std::cerr << "Failed to create input buffer pool" << std::endl; + return input_buffer_pool.status(); + } + hailo_status input_status = HAILO_UNINITIALIZED; + auto input_thread = std::make_unique(write_all, input, input_buffer_pool.value(), FRAMES_COUNT, std::ref(input_status)); + + // Join threads + input_thread->join(); + output_thread->join(); + if (HAILO_SUCCESS != input_status) { + return input_status; + } + if (HAILO_SUCCESS != output_status) { + return output_status; + } + + // The read/write threads have completed but the transfers issued by them haven't necessarily completed. + // We'll wait for the output buffer queue to fill back up, since the callback we registered enqueues buffers + // back to the pool + we issued the same number of reads as writes + output_buffer_pool.value()->wait_for_pending_buffers(); + + return HAILO_SUCCESS; +} diff --git a/hailort/libhailort/examples/cpp/raw_streams_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/raw_streams_example/CMakeLists.txt index 16ab196..4184783 100644 --- a/hailort/libhailort/examples/cpp/raw_streams_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/raw_streams_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_raw_streams_example raw_streams_example.cpp) target_link_libraries(cpp_raw_streams_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/raw_streams_example/raw_streams_example.cpp b/hailort/libhailort/examples/cpp/raw_streams_example/raw_streams_example.cpp index 76fe911..a73fd7c 100644 --- a/hailort/libhailort/examples/cpp/raw_streams_example/raw_streams_example.cpp +++ b/hailort/libhailort/examples/cpp/raw_streams_example/raw_streams_example.cpp @@ -10,6 +10,8 @@ #include "hailo/hailort.hpp" #include +#include + #define HEF_FILE ("hefs/shortcut_net.hef") constexpr size_t FRAMES_COUNT = 100; @@ -26,12 +28,7 @@ Expected> configure_network_group(Device return make_unexpected(hef.status()); } - auto stream_interface = device.get_default_streams_interface(); - if (!stream_interface) { - return make_unexpected(stream_interface.status()); - } - - auto configure_params = hef->create_configure_params(stream_interface.value()); + auto configure_params = device.create_configure_params(hef.value()); if (!configure_params) { return make_unexpected(configure_params.status()); } @@ -148,13 +145,18 @@ hailo_status infer(InputStreamRefVector &input_streams, OutputStreamRefVector &o int main() { - /* - For simplicity, not passing `device_id` - This function will fail in case more than one device is present. - See `hailort::Device::scan_devices` and `hailort::Device::create` functions documentation. - */ - auto device = Device::create(); + auto device_ids = Device::scan(); + if (!device_ids) { + std::cerr << "Failed to scan, status = " << device_ids.status() << std::endl; + return device_ids.status(); + } + if (device_ids->size() < 1){ + std::cerr << "Failed to find a connected hailo device." << std::endl; + return HAILO_INVALID_OPERATION; + } + auto device = Device::create(device_ids->at(0)); if (!device) { - std::cerr << "Failed create device " << device.status() << std::endl; + std::cerr << "Failed to create device " << device.status() << std::endl; return device.status(); } diff --git a/hailort/libhailort/examples/cpp/switch_network_groups_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/switch_network_groups_example/CMakeLists.txt index 3327f19..a0e9c7e 100644 --- a/hailort/libhailort/examples/cpp/switch_network_groups_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/switch_network_groups_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_switch_network_groups_example switch_network_groups_example.cpp) target_link_libraries(cpp_switch_network_groups_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/switch_network_groups_example/switch_network_groups_example.cpp b/hailort/libhailort/examples/cpp/switch_network_groups_example/switch_network_groups_example.cpp index 6c65417..e7c4d9f 100644 --- a/hailort/libhailort/examples/cpp/switch_network_groups_example/switch_network_groups_example.cpp +++ b/hailort/libhailort/examples/cpp/switch_network_groups_example/switch_network_groups_example.cpp @@ -13,6 +13,8 @@ #include #include +#include + constexpr bool QUANTIZED = true; constexpr hailo_format_type_t FORMAT_TYPE = HAILO_FORMAT_TYPE_AUTO; diff --git a/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/CMakeLists.txt index b98ea57..9c6114e 100644 --- a/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_switch_network_groups_manually_example switch_network_groups_manually_example.cpp) target_link_libraries(cpp_switch_network_groups_manually_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/switch_network_groups_manually_example.cpp b/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/switch_network_groups_manually_example.cpp index 26d890d..ddbe2f8 100644 --- a/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/switch_network_groups_manually_example.cpp +++ b/hailort/libhailort/examples/cpp/switch_network_groups_manually_example/switch_network_groups_manually_example.cpp @@ -14,6 +14,8 @@ #include #include +#include + constexpr bool QUANTIZED = true; constexpr hailo_format_type_t FORMAT_TYPE = HAILO_FORMAT_TYPE_AUTO; diff --git a/hailort/libhailort/examples/cpp/vstreams_example/CMakeLists.txt b/hailort/libhailort/examples/cpp/vstreams_example/CMakeLists.txt index 218f0b2..522ea6a 100644 --- a/hailort/libhailort/examples/cpp/vstreams_example/CMakeLists.txt +++ b/hailort/libhailort/examples/cpp/vstreams_example/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0) find_package(Threads REQUIRED) set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(HailoRT 4.12.1 EXACT REQUIRED) +find_package(HailoRT 4.13.0 EXACT REQUIRED) add_executable(cpp_vstreams_example vstreams_example.cpp) target_link_libraries(cpp_vstreams_example PRIVATE HailoRT::libhailort Threads::Threads) diff --git a/hailort/libhailort/examples/cpp/vstreams_example/vstreams_example.cpp b/hailort/libhailort/examples/cpp/vstreams_example/vstreams_example.cpp index 9e93cac..097d8a1 100644 --- a/hailort/libhailort/examples/cpp/vstreams_example/vstreams_example.cpp +++ b/hailort/libhailort/examples/cpp/vstreams_example/vstreams_example.cpp @@ -10,6 +10,8 @@ #include "hailo/hailort.hpp" #include +#include + #define HEF_FILE ("hefs/shortcut_net.hef") constexpr size_t FRAMES_COUNT = 100; @@ -26,7 +28,7 @@ Expected> configure_network_group(VDevic return make_unexpected(hef.status()); } - auto configure_params = hef->create_configure_params(HAILO_STREAM_INTERFACE_PCIE); + auto configure_params = vdevice.create_configure_params(hef.value()); if (!configure_params) { return make_unexpected(configure_params.status()); } diff --git a/hailort/libhailort/hef.proto b/hailort/libhailort/hef.proto index 9334b33..3407238 100644 --- a/hailort/libhailort/hef.proto +++ b/hailort/libhailort/hef.proto @@ -38,6 +38,9 @@ enum ProtoHEFExtensionType { KO_RUN_ASAP = 12; HAILO_NET_FLOW = 13; HAILO_NET_FLOW_YOLO_NMS = 14; + HAILO_NET_FLOW_YOLOX_NMS = 15; + HAILO_NET_FLOW_SSD_NMS = 16; + HAILO_NET_FLOW_IOU_NMS = 17; UNUSED = 0XFFFF; } @@ -76,7 +79,7 @@ enum ProtoHEFHwArch { PROTO__HW_ARCH__SAGE_A0 = 100; PROTO__HW_ARCH__SAGE_B0 = 101; PROTO__HW_ARCH__PAPRIKA_B0 = 102; - PROTO__HW_ARCH__MERCURY = 103; + PROTO__HW_ARCH__HAILO15H = 103; PROTO__HW_ARCH__GINGER = 104; PROTO__HW_ARCH__LAVENDER = 105; } @@ -146,6 +149,43 @@ message ProtoHEFYoloNmsOp { repeated ProtoHEFYoloBboxDecoder bbox_decoders = 4; }; +message ProtoHEFSSDBboxDecoder { + // List of Height coordinates (given as fraction of input size), defining each box dimensions around the anchor coordinates + repeated float h = 1; + + // List of Width coordinates (given as fraction of input size), defining each box dimensions around the anchor coordinates + repeated float w = 2; + + // Index of the pad connected to the encoded layer in the decoder (reg layer) + uint32 reg_pad_index = 3; + + // Index of the pad connected to the classes scores layer in the decoder (cls layer) + uint32 cls_pad_index = 4; +}; + +message ProtoHEFSSDNmsOp { + // Input image dimensions + double image_height = 1; + double image_width = 2; + + // Values used for compensation of rescales done in the training phase (derived from faster_rcnn architecture). This param rescales anchors centers. + uint32 centers_scale_factor = 3; + + // Values used for compensation of rescales done in the training phase (derived from faster_rcnn architecture). This param rescales anchors dimensions. + uint32 bbox_dimensions_scale_factor = 4; + + // Regression layer input order into bbox decoder + uint32 ty = 5; + uint32 tx = 6; + uint32 th = 7; + uint32 tw = 8; + + // List of bbox decoders (anchors) for the NMS layer. Each model has its own number of boxes per anchor + repeated ProtoHEFSSDBboxDecoder bbox_decoders = 9; +}; + +message ProtoHEFIOUNmsOp {}; + message ProtoHEFNmsOp { // NMS score threshold double nms_score_th = 1; @@ -167,7 +207,10 @@ message ProtoHEFNmsOp { // Additional information needed for specific NMS types oneof nms_op { - ProtoHEFYoloNmsOp yolo_nms_op = 7; + ProtoHEFYoloNmsOp yolo_nms_op = 7; // YOLOv5 post process + ProtoHEFYoloNmsOp yolox_nms_op = 8; // YOLO-X post process (ignores bbox decoder coordinations) + ProtoHEFSSDNmsOp ssd_nms_op = 9; // SSD post process + ProtoHEFIOUNmsOp iou_op = 10; // IoU only } }; @@ -221,7 +264,7 @@ message ProtoHEFPad { // Pad's unique index uint32 index = 1; - // Pad's name, can be empty of meaningful + // Pad's name, can be empty or meaningful string name = 2; // Additional information describing the data going through this pad's interface @@ -246,7 +289,7 @@ message ProtoHEFOp { // Op type for a subgraph that is running on Hailo's core ProtoHEFCoreOp core_op = 4; - // Op type for NMS post-processing + // Op type for NMS post-process ProtoHEFNmsOp nms_op = 5; } }; diff --git a/hailort/libhailort/include/hailo/buffer.hpp b/hailort/libhailort/include/hailo/buffer.hpp index 734386a..311185e 100644 --- a/hailort/libhailort/include/hailo/buffer.hpp +++ b/hailort/libhailort/include/hailo/buffer.hpp @@ -18,6 +18,7 @@ #include #include + namespace hailort { @@ -113,7 +114,7 @@ public: // Note: If this->size() is less than sizeof(T), then part of the data pointed to by the returned pointer // will be outside of the buffer's bounds. template::value, int> = 0> - T* as_pointer() + T* as_pointer() const { assert(m_size >= sizeof(T)); return reinterpret_cast(m_data.get()); diff --git a/hailort/libhailort/include/hailo/device.hpp b/hailort/libhailort/include/hailo/device.hpp index 8896a44..904b37e 100644 --- a/hailort/libhailort/include/hailo/device.hpp +++ b/hailort/libhailort/include/hailo/device.hpp @@ -21,6 +21,7 @@ #include #include + namespace hailort { @@ -43,7 +44,7 @@ public: enum class Type { PCIE = 0, ETH, - CORE + INTEGRATED }; /** @@ -169,6 +170,25 @@ public: */ static Expected get_device_type(const std::string &device_id); + /** + * Create the default configure params from an hef. + * + * @param[in] hef A reference to an Hef object to create configure params by + * @return Upon success, returns Expected of a NetworkGroupsParamsMap (map of string and ConfiguredNetworkParams). + * Otherwise, returns Unexpected of ::hailo_status error. + */ + Expected create_configure_params(Hef &hef) const; + + /** + * Create the default configure params from an hef. + * + * @param[in] hef A reference to an Hef object to create configure params by + * @param[in] network_group_name Name of network_group to make configure params for. + * @return Upon success, returns Expected of a NetworkGroupsParamsMap (map of string and ConfiguredNetworkParams). + * Otherwise, returns Unexpected of ::hailo_status error. + */ + Expected create_configure_params(Hef &hef, const std::string &network_group_name) const; + /** * Configure the device from an hef. * diff --git a/hailort/libhailort/include/hailo/dma_mapped_buffer.hpp b/hailort/libhailort/include/hailo/dma_mapped_buffer.hpp new file mode 100644 index 0000000..f25ac37 --- /dev/null +++ b/hailort/libhailort/include/hailo/dma_mapped_buffer.hpp @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file dma_mapped_buffer.hpp + * @brief The mapped buffer that is continuous in virtual memory, but not on physical memory. + * We map the buffer to the IOMMU. + * + * The buffer can be used only with the help of a descriptors list that contains pointers to a physical + * continuous "dma pages". + * + * There are 2 options to allocated the buffer: + * 1. User mode allocation - the user mode calls `malloc` or `mmap` to allocate the buffer, then + * using HailoRTDriver we map the driver to the IOMMU (and pin the pages to avoid pagigs). + * This is the default option + * 2. Kernel mode allocation - on some systems, the user mode doesn't allocate the memory in a "dma-able" address, + * so we need to allocate the pages in driver. + **/ + +#ifndef _HAILO_DMA_MAPPED_BUFFER_HPP_ +#define _HAILO_DMA_MAPPED_BUFFER_HPP_ + +#include "hailo/expected.hpp" +#include "hailo/device.hpp" + + +namespace hailort { + +// Forward deceleration across namespaces +namespace vdma { + class DescriptorList; + class MappedBufferFactory; + class BufferedChannel; +} + +// ******************************************** NOTE ******************************************** // +// Async Stream API and DmaMappedBuffer are currently not supported and are for internal use only // +// ********************************************************************************************** // +class HAILORTAPI DmaMappedBuffer final +{ +public: + static Expected create(size_t size, + hailo_vdma_buffer_direction_flags_t data_direction_flags, Device &device); + // TODO: doc that the addr needs to be on a new page and aligned to 64B (HRT-9559) + // probably best just to call mmap + static Expected create_from_user_address(void *user_address, size_t size, + hailo_vdma_buffer_direction_flags_t data_direction_flags, Device &device); + + DmaMappedBuffer(const DmaMappedBuffer &other) = delete; + DmaMappedBuffer &operator=(const DmaMappedBuffer &other) = delete; + DmaMappedBuffer(DmaMappedBuffer &&other) noexcept; + DmaMappedBuffer &operator=(DmaMappedBuffer &&other) = delete; + ~DmaMappedBuffer(); + + void *user_address(); + size_t size() const; + hailo_status synchronize(); + +private: + static Expected create(void *user_address, size_t size, + hailo_vdma_buffer_direction_flags_t data_direction_flags, Device &device); + + // Need access to pimpl + friend class vdma::DescriptorList; + friend class vdma::MappedBufferFactory; + friend class vdma::BufferedChannel; + + class Impl; + explicit DmaMappedBuffer(std::unique_ptr pimpl); + std::unique_ptr pimpl; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_DMA_MAPPED_BUFFER_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/include/hailo/event.hpp b/hailort/libhailort/include/hailo/event.hpp index 81190d3..e46517b 100644 --- a/hailort/libhailort/include/hailo/event.hpp +++ b/hailort/libhailort/include/hailo/event.hpp @@ -15,7 +15,6 @@ #include #include -#include #include #if defined(__GNUC__) #include @@ -25,6 +24,7 @@ #include #include + // Forward declare neosmart::neosmart_event_t_ namespace neosmart { struct neosmart_event_t_; diff --git a/hailort/libhailort/include/hailo/expected.hpp b/hailort/libhailort/include/hailo/expected.hpp index c77c2a0..6c766f7 100644 --- a/hailort/libhailort/include/hailo/expected.hpp +++ b/hailort/libhailort/include/hailo/expected.hpp @@ -162,10 +162,12 @@ #define _HAILO_EXPECTED_HPP_ #include "hailo/hailort.h" + #include #include #include + namespace hailort { @@ -451,6 +453,12 @@ public: return value(); } + const T& operator*() const& + { + assert(has_value()); + return value(); + } + /** * Checks whether the object contains a value. */ diff --git a/hailort/libhailort/include/hailo/hailort.h b/hailort/libhailort/include/hailo/hailort.h index a66cb12..2ac0702 100644 --- a/hailort/libhailort/include/hailo/hailort.h +++ b/hailort/libhailort/include/hailo/hailort.h @@ -16,12 +16,13 @@ extern "C" { #endif +#include "platform.h" + #include #include #include #include -#include "platform.h" /** @defgroup group_defines HailoRT API definitions * @{ @@ -68,6 +69,10 @@ extern "C" { #define HAILO_UNIQUE_VDEVICE_GROUP_ID ("UNIQUE") #define HAILO_DEFAULT_VDEVICE_GROUP_ID HAILO_UNIQUE_VDEVICE_GROUP_ID +#define HAILO_SCHEDULER_PRIORITY_NORMAL (16) +#define HAILO_SCHEDULER_PRIORITY_MAX (31) +#define HAILO_SCHEDULER_PRIORITY_MIN (0) + typedef float float32_t; typedef double float64_t; typedef uint16_t nms_bbox_counter_t; @@ -110,7 +115,7 @@ typedef uint16_t nms_bbox_counter_t; HAILO_STATUS__X(33, HAILO_ATR_TABLES_CONF_VALIDATION_FAIL /*!< Validating address translation tables failure, for FW control use */)\ HAILO_STATUS__X(34, HAILO_CONTROL_EVENT_CREATE_FAIL /*!< Creating control event failure */)\ HAILO_STATUS__X(35, HAILO_READ_EVENT_FAIL /*!< Reading event failure */)\ - HAILO_STATUS__X(36, HAILO_PCIE_DRIVER_FAIL /*!< PCIE driver failure */)\ + HAILO_STATUS__X(36, HAILO_DRIVER_FAIL /*!< Driver failure */)\ HAILO_STATUS__X(37, HAILO_INVALID_FIRMWARE_MAGIC /*!< Invalid FW magic */)\ HAILO_STATUS__X(38, HAILO_INVALID_FIRMWARE_CODE_SIZE /*!< Invalid FW code size */)\ HAILO_STATUS__X(39, HAILO_INVALID_KEY_CERTIFICATE_SIZE /*!< Invalid key certificate size */)\ @@ -358,7 +363,7 @@ typedef struct { typedef enum { HAILO_DEVICE_TYPE_PCIE, HAILO_DEVICE_TYPE_ETH, - HAILO_DEVICE_TYPE_CORE, + HAILO_DEVICE_TYPE_INTEGRATED, /** Max enum value to maintain ABI Integrity */ HAILO_DEVICE_TYPE_MAX_ENUM = HAILO_MAX_ENUM @@ -402,8 +407,7 @@ typedef enum hailo_device_architecture_e { HAILO_ARCH_HAILO8_A0 = 0, HAILO_ARCH_HAILO8, HAILO_ARCH_HAILO8L, - HAILO_ARCH_MERCURY_CA, - HAILO_ARCH_MERCURY_VPU, + HAILO_ARCH_HAILO15, /** Max enum value to maintain ABI Integrity */ HAILO_ARCH_MAX_ENUM = HAILO_MAX_ENUM @@ -685,6 +689,19 @@ typedef enum { */ HAILO_FORMAT_ORDER_RGB4 = 17, + /** + * YUV format, encoding 8 pixels in 96 bits + * [Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7, U0, U1, V0, V1] represents + * [Y0, U0, V0,], [Y1, U0, V0], [Y2, U0, V0], [Y3, U0, V0], [Y4, U1, V1], [Y5, U1, V1], [Y6, U1, V1], [Y7, U1, V1] + */ + HAILO_FORMAT_ORDER_I420 = 18, + + /** + * Internal implementation for HAILO_FORMAT_ORDER_I420 format + * [Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7, U0, U1, V0, V1] is represented by [Y0, Y1, Y2, Y3, U0, V0, Y4, Y5, Y6, Y7, U1, V1] + */ + HAILO_FORMAT_ORDER_HAILO_YYYYUV = 19, + /** Max enum value to maintain ABI Integrity */ HAILO_FORMAT_ORDER_MAX_ENUM = HAILO_MAX_ENUM } hailo_format_order_t; @@ -750,9 +767,32 @@ typedef enum { HAILO_H2D_STREAM = 0, HAILO_D2H_STREAM = 1, + /** Max enum value to maintain ABI Integrity */ HAILO_STREAM_DIRECTION_MAX_ENUM = HAILO_MAX_ENUM } hailo_stream_direction_t; +// ******************************************** NOTE ******************************************** // +// Async Stream API and DmaMappedBuffer are currently not supported and are for internal use only // +// ********************************************************************************************** // +/** Stream flags */ +typedef enum { + HAILO_STREAM_FLAGS_NONE = 0, /*!< No flags */ + HAILO_STREAM_FLAGS_ASYNC = 1 << 0, /*!< Async stream */ + + /** Max enum value to maintain ABI Integrity */ + HAILO_STREAM_FLAGS_MAX_ENUM = HAILO_MAX_ENUM +} hailo_stream_flags_t; + +/** Hailo vdma buffer direction */ +typedef enum { + HAILO_VDMA_BUFFER_DIRECTION_FLAGS_NONE = 0, + HAILO_VDMA_BUFFER_DIRECTION_FLAGS_H2D = 1 << 0, + HAILO_VDMA_BUFFER_DIRECTION_FLAGS_D2H = 1 << 1, + + /** Max enum value to maintain ABI Integrity */ + HAILO_VDMA_BUFFER_DIRECTION_FLAGS_MAX_ENUM = HAILO_MAX_ENUM +} hailo_vdma_buffer_direction_flags_t; + /** Input or output data transform parameters */ typedef struct { hailo_stream_transform_mode_t transform_mode; @@ -1020,18 +1060,18 @@ typedef struct { /** Core input stream (host to device) parameters */ typedef struct { EMPTY_STRUCT_PLACEHOLDER -} hailo_core_input_stream_params_t; +} hailo_integrated_input_stream_params_t; /** Core output stream (device to host) parameters */ typedef struct { EMPTY_STRUCT_PLACEHOLDER -} hailo_core_output_stream_params_t; +} hailo_integrated_output_stream_params_t; typedef enum { HAILO_STREAM_INTERFACE_PCIE = 0, HAILO_STREAM_INTERFACE_ETH, HAILO_STREAM_INTERFACE_MIPI, - HAILO_STREAM_INTERFACE_CORE, + HAILO_STREAM_INTERFACE_INTEGRATED, /** Max enum value to maintain ABI Integrity */ HAILO_STREAM_INTERFACE_MAX_ENUM = HAILO_MAX_ENUM @@ -1041,13 +1081,14 @@ typedef enum { typedef struct { hailo_stream_interface_t stream_interface; hailo_stream_direction_t direction; + hailo_stream_flags_t flags; union { hailo_pcie_input_stream_params_t pcie_input_params; - hailo_core_input_stream_params_t core_input_params; + hailo_integrated_input_stream_params_t integrated_input_params; hailo_eth_input_stream_params_t eth_input_params; hailo_mipi_input_stream_params_t mipi_input_params; hailo_pcie_output_stream_params_t pcie_output_params; - hailo_core_output_stream_params_t core_output_params; + hailo_integrated_output_stream_params_t integrated_output_params; hailo_eth_output_stream_params_t eth_output_params; }; } hailo_stream_parameters_t; @@ -1165,6 +1206,14 @@ typedef struct { } hailo_bbox_float32_t; #pragma pack(pop) +typedef struct { + /** + * - HAILO_SUCCESS when transfer is complete + * - HAILO_STREAM_NOT_ACTIVATED due to stream deactivation + */ + hailo_status status; +} hailo_async_transfer_completion_info_t; + /** * Input or output stream information. In case of multiple inputs or outputs, each one has * its own stream. @@ -2622,6 +2671,23 @@ HAILORTAPI hailo_status hailo_set_scheduler_timeout(hailo_configured_network_gro HAILORTAPI hailo_status hailo_set_scheduler_threshold(hailo_configured_network_group configured_network_group, uint32_t threshold, const char *network_name); +/** + * Sets the priority of the network. + * When the network group scheduler will choose the next network, networks with higher priority will be prioritized in the selection. + * bigger number represent higher priority. + * + * @param[in] configured_network_group NetworkGroup for which to set the scheduler priority. + * @param[in] priority Priority as a number between HAILO_SCHEDULER_PRIORITY_MIN - HAILO_SCHEDULER_PRIORITY_MAX. + * @param[in] network_name Network name for which to set the priority. + * If NULL is passed, the priority will be set for all the networks in the network group. + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * @note Using this function is only allowed when scheduling_algorithm is not ::HAILO_SCHEDULING_ALGORITHM_NONE. + * @note The default priority is HAILO_SCHEDULER_PRIORITY_NORMAL. + * @note Currently, setting the priority for a specific network is not supported. + */ +HAILORTAPI hailo_status hailo_set_scheduler_priority(hailo_configured_network_group configured_network_group, + uint8_t priority, const char *network_name); + /** @} */ // end of group_network_group_functions /** @defgroup group_stream_functions Stream functions diff --git a/hailort/libhailort/include/hailo/hailort.hpp b/hailort/libhailort/include/hailo/hailort.hpp index 6cf365b..d429a7d 100644 --- a/hailort/libhailort/include/hailo/hailort.hpp +++ b/hailort/libhailort/include/hailo/hailort.hpp @@ -13,6 +13,7 @@ #define _HAILORT_HPP_ #include "hailo/hailort.h" +#include "hailo/hailort_common.hpp" #include "hailo/hef.hpp" #include "hailo/device.hpp" #include "hailo/vdevice.hpp" @@ -24,9 +25,10 @@ #include "hailo/expected.hpp" #include "hailo/buffer.hpp" #include "hailo/event.hpp" -#include "hailo/hailort_common.hpp" #include "hailo/runtime_statistics.hpp" #include "hailo/network_rate_calculator.hpp" #include "hailo/quantization.hpp" +#include "hailo/dma_mapped_buffer.hpp" +#include "hailo/hailort_defaults.hpp" #endif /* _HAILORT_HPP_ */ diff --git a/hailort/libhailort/include/hailo/hailort_common.hpp b/hailort/libhailort/include/hailo/hailort_common.hpp index 8b6f96f..d9ab7cf 100644 --- a/hailort/libhailort/include/hailo/hailort_common.hpp +++ b/hailort/libhailort/include/hailo/hailort_common.hpp @@ -12,11 +12,13 @@ #include "hailo/hailort.h" #include "hailo/expected.hpp" + #include #include #include #include + namespace hailort { @@ -157,6 +159,29 @@ public: } } + /** + * Gets a string reprenestation of the given device architecture. + * + * @param[in] arch A ::hailo_device_architecture_t object. + * @return The string representation of the device architecture. + */ + static std::string get_device_arch_str(const hailo_device_architecture_t &arch) + { + switch (arch) + { + case HAILO_ARCH_HAILO8_A0: + return "HAILO8_A0"; + case HAILO_ARCH_HAILO8: + return "HAILO8"; + case HAILO_ARCH_HAILO8L: + return "HAILO8L"; + case HAILO_ARCH_HAILO15: + return "HAILO15"; + default: + return "UNKNOWN ARCHITECTURE"; + } + } + /** * Gets a string reprenestation of the given format order. * @@ -201,6 +226,10 @@ public: return "YYVU"; case HAILO_FORMAT_ORDER_RGB4: return "RGB4"; + case HAILO_FORMAT_ORDER_I420: + return "I420"; + case HAILO_FORMAT_ORDER_HAILO_YYYYUV: + return "YYYYUV"; default: return "Nan"; } @@ -318,7 +347,7 @@ public: static constexpr bool is_vdma_stream_interface(hailo_stream_interface_t stream_interface) { - return (HAILO_STREAM_INTERFACE_PCIE == stream_interface) || (HAILO_STREAM_INTERFACE_CORE == stream_interface); + return (HAILO_STREAM_INTERFACE_PCIE == stream_interface) || (HAILO_STREAM_INTERFACE_INTEGRATED == stream_interface); } static Expected to_device_id(const std::string &device_id); @@ -379,6 +408,11 @@ inline constexpr hailo_pipeline_elem_stats_flags_t& operator|=(hailo_pipeline_el return a; } +inline bool is_bit_set(uint32_t num, uint8_t i) +{ + return (1 == ((num >> i) & 1)); +} + } /* namespace hailort */ #endif /* _HAILO_HAILORT_COMMON_HPP_ */ diff --git a/hailort/libhailort/include/hailo/hailort_defaults.hpp b/hailort/libhailort/include/hailo/hailort_defaults.hpp new file mode 100644 index 0000000..9577eec --- /dev/null +++ b/hailort/libhailort/include/hailo/hailort_defaults.hpp @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file hailort_defaults.hpp + * @brief + **/ + +#ifndef _HAILO_HAILORT_DEFAULTS_HPP_ +#define _HAILO_HAILORT_DEFAULTS_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" +#include "hailo/network_group.hpp" + + +namespace hailort +{ + +#define HAILO_DEFAULT_NETWORK_NAME_QUALIFIER (std::string("/")) + + +class HAILORTAPI HailoRTDefaults final +{ +public: + HailoRTDefaults() = delete; + + static Expected get_device_format_order(uint32_t compiler_format_order); + static hailo_format_order_t get_default_host_format_order(const hailo_format_t &device_format); + + static hailo_format_t expand_auto_format(const hailo_format_t &host_format, const hailo_format_t &hw_format); + static hailo_format_t get_user_buffer_format(); + static hailo_format_t get_user_buffer_format(bool quantized, hailo_format_type_t format_type); + + static hailo_transform_params_t get_transform_params(bool quantized, hailo_format_type_t format_type); + static hailo_transform_params_t get_transform_params(const hailo_stream_info_t &stream_info); + static hailo_transform_params_t get_transform_params(); + + static hailo_vstream_params_t get_vstreams_params(); + static hailo_vstream_params_t get_vstreams_params(bool quantized, hailo_format_type_t format_type); + + static Expected get_stream_parameters(hailo_stream_interface_t interface, + hailo_stream_direction_t direction); + + static ConfigureNetworkParams get_configure_params(uint16_t batch_size = HAILO_DEFAULT_BATCH_SIZE, + hailo_power_mode_t power_mode = HAILO_POWER_MODE_PERFORMANCE); + static hailo_network_parameters_t get_network_parameters(uint16_t batch_size = HAILO_DEFAULT_BATCH_SIZE); + static std::string get_network_name(const std::string &net_group_name); + static hailo_activate_network_group_params_t get_active_network_group_params(); + + static hailo_vdevice_params_t get_vdevice_params(); + +private: + static struct sockaddr_in get_sockaddr(); + static hailo_eth_input_stream_params_t get_eth_input_stream_params(); + static hailo_eth_output_stream_params_t get_eth_output_stream_params(); + static hailo_pcie_input_stream_params_t get_pcie_input_stream_params(); + static hailo_pcie_output_stream_params_t get_pcie_output_stream_params(); + static hailo_integrated_input_stream_params_t get_integrated_input_stream_params(); + static hailo_integrated_output_stream_params_t get_integrated_output_stream_params(); + static hailo_mipi_input_stream_params_t get_mipi_input_stream_params(); +}; + +} /* namespace hailort */ + +#endif /* _HAILO_HAILORT_DEFAULTS_HPP_ */ diff --git a/hailort/libhailort/include/hailo/hef.hpp b/hailort/libhailort/include/hailo/hef.hpp index 0aa6f13..ad16c39 100644 --- a/hailort/libhailort/include/hailo/hef.hpp +++ b/hailort/libhailort/include/hailo/hef.hpp @@ -18,6 +18,7 @@ #include #include + namespace hailort { @@ -223,6 +224,23 @@ public: */ Expected get_bottleneck_fps(const std::string &net_group_name=""); + /** + * Get device Architecture HEF was compiled for. + * + * @return Upon success, returns Expected containing the device architecture the HEF was compiled for. + * Otherwise, returns Unexpected of ::hailo_status error. + */ + Expected get_hef_device_arch(); + + /** + * Get string of device architecture HEF was compiled for. + * + * @param[in] arch hailo_device_architecture_t representing the device architecture of the HEF + * @return Upon success, returns string representing the device architecture the HEF was compiled for. + * Otherwise, returns Unexpected of ::hailo_status error. + */ + static Expected device_arch_to_string(const hailo_device_architecture_t arch); + /** * Gets all stream names under the given vstream name * @@ -403,7 +421,7 @@ public: * the function returns the output virtual stream params of the given network. * If NULL is passed, the function returns the output virtual stream params of * all the networks of the first network group. - * @param[in] quantized Whether the data fed into the chip is already quantized. True means + * @param[in] quantized Whether the data returned from the chip is already quantized. True means * the data is already quantized. False means it's HailoRT's responsibility * to quantize (scale) the data. * @param[in] format_type The default format type for all output virtual streams. @@ -434,6 +452,8 @@ public: */ std::string hash() const; + Expected get_hef_description(bool stream_infos, bool vstream_infos); + ~Hef(); Hef(Hef &&); Hef &operator=(Hef &&); @@ -448,6 +468,7 @@ private: friend class OutputStream; friend class PyhailortInternal; friend class ConfiguredNetworkGroupBase; + friend class CoreOp; friend class VDeviceBase; #ifdef HAILO_SUPPORT_MULTI_PROCESS diff --git a/hailort/libhailort/include/hailo/inference_pipeline.hpp b/hailort/libhailort/include/hailo/inference_pipeline.hpp index 556d65b..c4a3254 100644 --- a/hailort/libhailort/include/hailo/inference_pipeline.hpp +++ b/hailort/libhailort/include/hailo/inference_pipeline.hpp @@ -12,8 +12,6 @@ #include "hailo/vstream.hpp" -#include -#include namespace hailort { @@ -90,12 +88,14 @@ public: m_inputs(std::move(other.m_inputs)), m_outputs(std::move(other.m_outputs)), m_is_multi_context(std::move(other.m_is_multi_context)), + m_is_scheduled(std::move(other.m_is_scheduled)), m_network_name_to_input_count(std::move(other.m_network_name_to_input_count)), m_network_name_to_output_count(std::move(other.m_network_name_to_output_count)), m_batch_size(std::move(other.m_batch_size)) {}; private: - InferVStreams(std::vector &&inputs, std::vector &&outputs, bool is_multi_context, uint16_t batch_size); + InferVStreams(std::vector &&inputs, std::vector &&outputs, bool is_multi_context, + bool is_scheduled, uint16_t batch_size); hailo_status verify_network_inputs_and_outputs(const std::map& inputs_name_mem_view_map, const std::map& outputs_name_mem_view_map); hailo_status verify_memory_view_size(const std::map& inputs_name_mem_view_map, @@ -105,7 +105,8 @@ private: std::vector m_inputs; std::vector m_outputs; - bool m_is_multi_context; + const bool m_is_multi_context; + const bool m_is_scheduled; std::map m_network_name_to_input_count; std::map m_network_name_to_output_count; uint16_t m_batch_size; diff --git a/hailort/libhailort/include/hailo/network_group.hpp b/hailort/libhailort/include/hailo/network_group.hpp index 8daba2f..ae9ae8e 100644 --- a/hailort/libhailort/include/hailo/network_group.hpp +++ b/hailort/libhailort/include/hailo/network_group.hpp @@ -17,8 +17,7 @@ #include #include #include -#include -#include + namespace hailort { @@ -67,7 +66,7 @@ public: virtual const std::string &get_network_group_name() const = 0; virtual Expected get_intermediate_buffer(const IntermediateBufferKey &key) = 0; - + virtual hailo_status set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) = 0; /** @@ -241,7 +240,7 @@ public: /** * Creates output virtual stream params. * - * @param[in] quantized Whether the data fed into the chip is already quantized. True means + * @param[in] quantized Whether the data returned from the chip is already quantized. True means * the data is already quantized. False means it's HailoRT's responsibility * to quantize (scale) the data. * @param[in] format_type The default format type for all output virtual streams. @@ -259,7 +258,7 @@ public: /** * Creates output virtual stream params. The groups are splitted with respect to their low-level streams. * - * @param[in] quantized Whether the data fed into the chip is already quantized. True means + * @param[in] quantized Whether the data returned from the chip is already quantized. True means * the data is already quantized. False means it's HailoRT's responsibility * to quantize (scale) the data. * @param[in] format_type The default format type for all output virtual streams. @@ -319,6 +318,11 @@ public: */ virtual Expected> get_all_vstream_infos(const std::string &network_name="") const = 0; + /** + * @returns whether the network group is managed by the model scheduler. + */ + virtual bool is_scheduled() const = 0; + /** * Sets the maximum time period that may pass before getting run time from the scheduler, * even without reaching the minimum required send requests (e.g. threshold - see set_scheduler_threshold()), @@ -350,6 +354,21 @@ public: */ virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name="") = 0; + /** + * Sets the priority of the network. + * When the network group scheduler will choose the next network, networks with higher priority will be prioritized in the selection. + * bigger number represent higher priority. + * + * @param[in] priority Priority as a number between HAILO_SCHEDULER_PRIORITY_MIN - HAILO_SCHEDULER_PRIORITY_MAX. + * @param[in] network_name Network name for which to set the Priority. + * If not passed, the priority will be set for all the networks in the network group. + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * @note Using this function is only allowed when scheduling_algorithm is not ::HAILO_SCHEDULING_ALGORITHM_NONE. + * @note The default priority is HAILO_SCHEDULER_PRIORITY_NORMAL. + * @note Currently, setting the priority for a specific network is not supported. + */ + virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name="") = 0; + /** * @return Is the network group multi-context or not. */ @@ -366,6 +385,10 @@ public: virtual Expected> create_input_vstreams(const std::map &inputs_params) = 0; virtual Expected> create_output_vstreams(const std::map &outputs_params) = 0; + virtual hailo_status before_fork() { return HAILO_SUCCESS; } + virtual hailo_status after_fork_in_parent() { return HAILO_SUCCESS; } + virtual hailo_status after_fork_in_child() { return HAILO_SUCCESS; } + protected: ConfiguredNetworkGroup() = default; diff --git a/hailort/libhailort/include/hailo/quantization.hpp b/hailort/libhailort/include/hailo/quantization.hpp index 7c2ca5f..d48b008 100644 --- a/hailort/libhailort/include/hailo/quantization.hpp +++ b/hailort/libhailort/include/hailo/quantization.hpp @@ -12,9 +12,11 @@ #include "hailo/hailort.h" #include "hailo/hailort_common.hpp" + #include #include + namespace hailort { @@ -171,14 +173,6 @@ public: return (T)((number - quant_info.qp_zp) * quant_info.qp_scale); } -private: - template - static inline Q quantize_input(T number, hailo_quant_info_t quant_info) - { - float32_t clipped_number = clip((float32_t)number, quant_info.limvals_min, quant_info.limvals_max); - return (Q)rintf((clipped_number / quant_info.qp_scale) + quant_info.qp_zp); - } - static inline float32_t clip(float32_t n, float32_t limval_min, float32_t limval_max) { if (n >= limval_max) { @@ -191,6 +185,14 @@ private: return n; } } + +private: + template + static inline Q quantize_input(T number, hailo_quant_info_t quant_info) + { + float32_t clipped_number = clip((float32_t)number, quant_info.limvals_min, quant_info.limvals_max); + return (Q)rintf((clipped_number / quant_info.qp_scale) + quant_info.qp_zp); + } }; } /* namespace hailort */ diff --git a/hailort/libhailort/include/hailo/runtime_statistics.hpp b/hailort/libhailort/include/hailo/runtime_statistics.hpp index b73c8c7..60a5b29 100644 --- a/hailort/libhailort/include/hailo/runtime_statistics.hpp +++ b/hailort/libhailort/include/hailo/runtime_statistics.hpp @@ -16,6 +16,7 @@ #include #include + namespace hailort { @@ -126,9 +127,10 @@ public: * Add a new measurement to the Accumulator, updating the statistics measured. * * @param data The measurement to be added. + * @param samples_count The weight of the measurement to be considered in average calculations. * @note Implementations of this interface are to update the statistics in constant time */ - virtual void add_data_point(T data) = 0; + virtual void add_data_point(T data, uint32_t samples_count = 1) = 0; /** * Gets the current statistics of the data added to the Accumulator, clearing the statistics afterwards. diff --git a/hailort/libhailort/include/hailo/stream.hpp b/hailort/libhailort/include/hailo/stream.hpp index a2ebe5f..2c5ee19 100644 --- a/hailort/libhailort/include/hailo/stream.hpp +++ b/hailort/libhailort/include/hailo/stream.hpp @@ -15,15 +15,22 @@ #include "hailo/event.hpp" #include -#include #include #include +#include + namespace hailort { // Forward declaration struct LayerInfo; +class DmaMappedBuffer; + +using TransferDoneCallback = std::function buffer, + const hailo_async_transfer_completion_info_t &status, + void *opaque)>; + /*! Input (host to device) stream representation */ class HAILORTAPI InputStream @@ -76,10 +83,11 @@ public: /** * @returns a pointer for network group activated event. */ - virtual EventPtr &get_network_group_activated_event() = 0; + EventPtr &get_network_group_activated_event() + DEPRECATED("'InputStream::get_network_group_activated_event' is deprecated."); /** - * @returns whether the stream is managed by a network group scheduler. + * @returns whether the stream is managed by the model scheduler. */ virtual bool is_scheduled() = 0; @@ -93,6 +101,13 @@ public: */ virtual hailo_status write(const MemoryView &buffer); + // ******************************************** NOTE ******************************************** // + // Async Stream API and DmaMappedBuffer are currently not supported and are for internal use only // + // ********************************************************************************************** // + virtual hailo_status wait_for_ready(size_t transfer_size, std::chrono::milliseconds timeout); // Internal use only + virtual hailo_status write_async(std::shared_ptr buffer, const TransferDoneCallback &user_callback, + void *opaque = nullptr); // Internal use only + /** * @returns A ::hailo_stream_info_t object containing the stream's info. */ @@ -122,14 +137,16 @@ public: */ virtual std::string to_string() const; + // get_network_group_activated_event is same as this function + virtual EventPtr &get_core_op_activated_event() = 0; protected: InputStream() = default; - InputStream(InputStream &&) = default; + InputStream(InputStream &&) = delete; // Note: Implement sync_write_all_raw_buffer_no_transform_impl for the actual stream interaction in sub classes virtual hailo_status sync_write_all_raw_buffer_no_transform_impl(void *buffer, size_t offset, size_t size) = 0; - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) = 0; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) = 0; virtual hailo_status deactivate_stream() = 0; virtual Expected sync_write_raw_buffer(const MemoryView &buffer) = 0; @@ -140,6 +157,7 @@ protected: private: friend class HefConfigurator; friend class ConfiguredNetworkGroupBase; + friend class CoreOp; }; /*! Output (device to host) stream representation */ @@ -186,13 +204,14 @@ public: /** * @returns a pointer for network group activated event. */ - virtual EventPtr &get_network_group_activated_event() = 0; + EventPtr &get_network_group_activated_event() + DEPRECATED("'OutputStream::get_network_group_activated_event' is deprecated."); /** - * @returns whether the stream is managed by a network group scheduler. + * @returns whether the stream is managed by the model scheduler. */ virtual bool is_scheduled() = 0; - + /** * @returns the stream's info. */ @@ -237,11 +256,20 @@ public: */ virtual hailo_status read(MemoryView buffer); + // ******************************************** NOTE ******************************************** // + // Async Stream API and DmaMappedBuffer are currently not supported and are for internal use only // + // ********************************************************************************************** // + virtual hailo_status wait_for_ready(size_t transfer_size, std::chrono::milliseconds timeout); // Internal use only + virtual hailo_status read_async(std::shared_ptr buffer, const TransferDoneCallback &user_callback, + void *opaque = nullptr); // Internal use only + + // get_network_group_activated_event is same as this function + virtual EventPtr &get_core_op_activated_event() = 0; protected: OutputStream() = default; - OutputStream(OutputStream&&); + OutputStream(OutputStream&&) = delete; - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) = 0; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) = 0; virtual hailo_status deactivate_stream() = 0; virtual hailo_status read_all(MemoryView &buffer) = 0; @@ -260,6 +288,7 @@ private: friend class ConfiguredNetworkGroupBase; friend class HwReadElement; friend class OutputDemuxer; + friend class CoreOp; }; } /* namespace hailort */ diff --git a/hailort/libhailort/include/hailo/transform.hpp b/hailort/libhailort/include/hailo/transform.hpp index 54abd0d..883dbca 100644 --- a/hailort/libhailort/include/hailo/transform.hpp +++ b/hailort/libhailort/include/hailo/transform.hpp @@ -20,6 +20,7 @@ #include #include + namespace hailort { diff --git a/hailort/libhailort/include/hailo/vdevice.hpp b/hailort/libhailort/include/hailo/vdevice.hpp index 9657374..9dd592a 100644 --- a/hailort/libhailort/include/hailo/vdevice.hpp +++ b/hailort/libhailort/include/hailo/vdevice.hpp @@ -16,11 +16,10 @@ #include "hailo/network_group.hpp" #include "hailo/device.hpp" + namespace hailort { -#define HAILO_ENABLE_MULTI_DEVICE_SCHEDULER "HAILO_ENABLE_MULTI_DEVICE_SCHEDULER" - /*! Represents a bundle of physical devices. */ class HAILORTAPI VDevice { @@ -89,6 +88,29 @@ public: */ virtual Expected get_default_streams_interface() const = 0; + /** + * Create the default configure params from an hef. + * + * @param[in] hef A reference to an Hef object to create configure params by + * @return Upon success, returns Expected of a NetworkGroupsParamsMap (map of string and ConfiguredNetworkParams). + * Otherwise, returns Unexpected of ::hailo_status error. + */ + Expected create_configure_params(Hef &hef) const; + + /** + * Create the default configure params from an hef. + * + * @param[in] hef A reference to an Hef object to create configure params by + * @param[in] network_group_name Name of network_group to make configure params for. + * @return Upon success, returns Expected of a NetworkGroupsParamsMap (map of string and ConfiguredNetworkParams). + * Otherwise, returns Unexpected of ::hailo_status error. + */ + Expected create_configure_params(Hef &hef, const std::string &network_group_name) const; + + virtual hailo_status before_fork() { return HAILO_SUCCESS; } + virtual hailo_status after_fork_in_parent() { return HAILO_SUCCESS; } + virtual hailo_status after_fork_in_child() { return HAILO_SUCCESS; } + virtual ~VDevice() = default; VDevice(const VDevice &) = delete; VDevice &operator=(const VDevice &) = delete; diff --git a/hailort/libhailort/include/hailo/vstream.hpp b/hailort/libhailort/include/hailo/vstream.hpp index 040e554..faf2bf2 100644 --- a/hailort/libhailort/include/hailo/vstream.hpp +++ b/hailort/libhailort/include/hailo/vstream.hpp @@ -10,8 +10,6 @@ #ifndef _HAILO_VSTREAM_HPP_ #define _HAILO_VSTREAM_HPP_ -#include "hailo/transform.hpp" -#include "hailo/stream.hpp" #include "hailo/network_group.hpp" #include "hailo/runtime_statistics.hpp" @@ -29,7 +27,7 @@ public: static Expected create(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::shared_ptr pipeline_exit, std::vector> &&pipeline, - std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr network_group_activated_event, + std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator); InputVStream(InputVStream &&other) noexcept = default; InputVStream &operator=(InputVStream &&other) noexcept = default; @@ -158,6 +156,10 @@ public: */ const std::vector> &get_pipeline() const; + hailo_status before_fork(); + hailo_status after_fork_in_parent(); + hailo_status after_fork_in_child(); + protected: explicit InputVStream(std::shared_ptr vstream); std::string get_pipeline_description() const; @@ -178,7 +180,7 @@ public: const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, - EventPtr network_group_activated_event, AccumulatorPtr pipeline_latency_accumulator); + EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator); OutputVStream(OutputVStream &&other) noexcept = default; OutputVStream &operator=(OutputVStream &&other) noexcept = default; virtual ~OutputVStream() = default; @@ -299,6 +301,10 @@ public: */ const std::vector> &get_pipeline() const; + hailo_status before_fork(); + hailo_status after_fork_in_parent(); + hailo_status after_fork_in_child(); + protected: explicit OutputVStream(std::shared_ptr vstream); std::string get_pipeline_description() const; @@ -310,7 +316,7 @@ protected: std::shared_ptr m_vstream; friend class VStreamsBuilderUtils; - friend class VDeviceNetworkGroup; + friend class VDeviceCoreOp; }; /*! Contains the virtual streams creation functions */ diff --git a/hailort/libhailort/scheduler_mon.proto b/hailort/libhailort/scheduler_mon.proto index 01155f8..8a9656f 100644 --- a/hailort/libhailort/scheduler_mon.proto +++ b/hailort/libhailort/scheduler_mon.proto @@ -5,7 +5,7 @@ option optimize_for = LITE_RUNTIME; message ProtoMonInfo { string network_name = 1; double fps = 2; - double active_time = 3; + double utilization = 3; } enum ProtoMonStreamDirection { @@ -13,6 +13,12 @@ enum ProtoMonStreamDirection { PROTO__STREAM_DIRECTION__DEVICE_TO_HOST = 1; } +message ProtoMonDeviceInfo { + string device_id = 1; + double utilization = 2; + string device_arch = 3; +} + message ProtoMonStreamFramesInfo { string stream_name = 1; ProtoMonStreamDirection stream_direction = 2; @@ -29,4 +35,5 @@ message ProtoMon { string pid = 1; repeated ProtoMonInfo networks_infos = 2; repeated ProtoMonNetworkFrames net_frames_infos = 3; + repeated ProtoMonDeviceInfo device_infos = 4; } \ No newline at end of file diff --git a/hailort/libhailort/src/CMakeLists.txt b/hailort/libhailort/src/CMakeLists.txt index f42ec10..6dc846a 100644 --- a/hailort/libhailort/src/CMakeLists.txt +++ b/hailort/libhailort/src/CMakeLists.txt @@ -15,91 +15,32 @@ FUNCTION(relative_to_absolute_paths output) SET(${output} "${listVar}" PARENT_SCOPE) ENDFUNCTION(relative_to_absolute_paths) -add_subdirectory(os) -add_subdirectory(net_flow) - set(HAILORT_CPP_SOURCES - device.cpp - device_internal.cpp - control.cpp - stream.cpp - stream_internal.cpp - transform.cpp - buffer.cpp - network_rate_calculator.cpp - hailort_logger.cpp hailort.cpp - hailort_common.cpp - sensor_config_utils.cpp - pipeline.cpp - pipeline_multiplexer.cpp - - eth_device.cpp - eth_stream.cpp - udp.cpp - - hef.cpp - network_group_metadata.cpp - - context_switch/context_switch_actions.cpp - context_switch/hcp_config_network_group.cpp - context_switch/hcp_config_activated_network_group.cpp - context_switch/vdma_config_network_group.cpp - context_switch/vdevice_network_group.cpp - context_switch/vdma_config_activated_network_group.cpp - context_switch/network_group.cpp - context_switch/resource_manager.cpp - context_switch/resource_manager_builder.cpp - context_switch/context_switch_buffer_builder.cpp - - channel_allocator.cpp - inter_context_buffer.cpp - ddr_channels_pair.cpp - config_buffer.cpp - d2h_events_parser.cpp - mipi_stream.cpp - - vdma_channel.cpp - vdma_descriptor_list.cpp - vdma_device.cpp - vdma_stream.cpp - - vdma/vdma_mapped_buffer_impl.cpp - vdma/mapped_buffer.cpp - vdma/sg_buffer.cpp - vdma/continuous_buffer.cpp - vdma/vdma_buffer.cpp - - pcie_device.cpp - pcie_stream.cpp - - core_device.cpp - core_stream.cpp - - vdevice.cpp - vdevice_stream.cpp - vdevice_stream_multiplexer_wrapper.cpp - multi_device_scheduled_stream.cpp - - control_protocol.cpp + hailort_defaults.cpp +) - vstream.cpp - inference_pipeline.cpp +add_subdirectory(utils) +add_subdirectory(os) +add_subdirectory(device_common) +add_subdirectory(vdevice) +add_subdirectory(transform) +add_subdirectory(stream_common) +add_subdirectory(eth) +add_subdirectory(vdma) +add_subdirectory(mipi) +add_subdirectory(hef) +add_subdirectory(network_group) +add_subdirectory(core_op) +add_subdirectory(net_flow) - network_group_scheduler.cpp - scheduled_network_group.cpp - scheduler_oracle.cpp -) +set(HAILORT_CPP_SOURCES "${HAILORT_CPP_SOURCES}" "${HAILORT_OPS_CPP_SOURCES}") if(HAILO_BUILD_SERVICE) - set(HAILORT_CPP_SOURCES "${HAILORT_CPP_SOURCES}" hailort_rpc_client.cpp network_group_client.cpp) -endif() -if(HAILO_BUILD_PROFILER) - set(HAILORT_CPP_SOURCES "${HAILORT_CPP_SOURCES}" tracer.cpp) + add_subdirectory(service) endif() - set(common_dir "${PROJECT_SOURCE_DIR}/common/src") set(COMMON_C_SOURCES ${common_dir}/firmware_status.c @@ -119,6 +60,7 @@ set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} CACHE INTERNAL "Absolute paths of set(HAILORT_CPP_OS_SOURCES ${HAILORT_CPP_OS_SOURCES} CACHE INTERNAL "Absolute paths of os-related source files") set(COMMON_C_SOURCES ${COMMON_C_SOURCES} CACHE INTERNAL "Absolute paths of common source files") set(HAILORT_SRCS_ABS ${HAILORT_CPP_SOURCES} ${HAILORT_CPP_OS_SOURCES} ${HAILORT_COMMON_CPP_SOURCES} ${COMMON_C_SOURCES} CACHE INTERNAL "All absolute paths of hailort's source files") +set(HAILORT_OPS_CPP_SOURCES ${HAILORT_OPS_CPP_SOURCES} PARENT_SCOPE) SET_SOURCE_FILES_PROPERTIES(${C_SOURCES} PROPERTIES LANGUAGE CXX) add_library(libhailort SHARED ${HAILORT_SRCS_ABS}) @@ -159,12 +101,12 @@ set(HAILORT_PUBLIC_HEADERS ${HAILORT_INC_DIR}/hailo/hailort.h ${HAILORT_INC_DIR}/hailo/platform.h + ${HAILORT_INC_DIR}/hailo/hailort.hpp ${HAILORT_INC_DIR}/hailo/buffer.hpp ${HAILORT_INC_DIR}/hailo/device.hpp ${HAILORT_INC_DIR}/hailo/event.hpp ${HAILORT_INC_DIR}/hailo/expected.hpp ${HAILORT_INC_DIR}/hailo/hailort_common.hpp - ${HAILORT_INC_DIR}/hailo/hailort.hpp ${HAILORT_INC_DIR}/hailo/hef.hpp ${HAILORT_INC_DIR}/hailo/network_group.hpp ${HAILORT_INC_DIR}/hailo/stream.hpp @@ -175,6 +117,8 @@ set(HAILORT_PUBLIC_HEADERS ${HAILORT_INC_DIR}/hailo/network_rate_calculator.hpp ${HAILORT_INC_DIR}/hailo/vdevice.hpp ${HAILORT_INC_DIR}/hailo/quantization.hpp + ${HAILORT_INC_DIR}/hailo/dma_mapped_buffer.hpp + ${HAILORT_INC_DIR}/hailo/hailort_defaults.hpp ) set_target_properties(libhailort PROPERTIES diff --git a/hailort/libhailort/src/context_switch/active_network_group_holder.hpp b/hailort/libhailort/src/context_switch/active_network_group_holder.hpp deleted file mode 100644 index c6edb5c..0000000 --- a/hailort/libhailort/src/context_switch/active_network_group_holder.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file active_network_group_holder.hpp - * @brief place_holder stored in ConfigManager indicating which ConfiguredNetworkGroup is currently active - * - **/ - -#ifndef _HAILO_CONTEXT_SWITCH_ACTIVE_NETWORK_GROUP_HOLDER_HPP_ -#define _HAILO_CONTEXT_SWITCH_ACTIVE_NETWORK_GROUP_HOLDER_HPP_ - -// TODO: cant we just have ActiveNetworkGroup ref under device? - -#include "hailo/hailort.h" -#include "common/utils.hpp" - -namespace hailort -{ - -template -class ActiveNetworkGroupHolder final -{ - public: - ActiveNetworkGroupHolder() : m_net_group(nullptr) {} - - ExpectedRef get() - { - CHECK_NOT_NULL_AS_EXPECTED(m_net_group, HAILO_INVALID_OPERATION); - return std::ref(*m_net_group); - } - void set(T &net_group) - { - assert(!is_any_active()); - m_net_group = &net_group; - } - - bool is_any_active() { return nullptr != m_net_group; } - - void clear() { m_net_group = nullptr; } - - ActiveNetworkGroupHolder(ActiveNetworkGroupHolder&) = delete; - ActiveNetworkGroupHolder& operator=(ActiveNetworkGroupHolder&) = delete; - ActiveNetworkGroupHolder& operator=(ActiveNetworkGroupHolder&&) = delete; - ActiveNetworkGroupHolder(ActiveNetworkGroupHolder&&) = default; - private: - T *m_net_group; -}; - -} /* namespace hailort */ - -#endif //_HAILO_CONTEXT_SWITCH_ACTIVE_NETWORK_GROUP_HOLDER_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/context_switch/hcp_config_activated_network_group.cpp b/hailort/libhailort/src/context_switch/hcp_config_activated_network_group.cpp deleted file mode 100644 index 6b85a7e..0000000 --- a/hailort/libhailort/src/context_switch/hcp_config_activated_network_group.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file hcp_config_activated_network_group.cpp - * @brief HcpConfigActivatedNetworkGroup implementation - **/ - -#include "context_switch/single_context/hcp_config_activated_network_group.hpp" -#include "control.hpp" - -namespace hailort -{ - -Expected HcpConfigActivatedNetworkGroup::create(Device &device, std::vector &config, - const std::string &network_group_name, - const hailo_activate_network_group_params_t &network_group_params, - std::map> &input_streams, - std::map> &output_streams, - ActiveNetGroupHolder &active_net_group_holder, - hailo_power_mode_t power_mode, EventPtr network_group_activated_event, - ConfiguredNetworkGroupBase &network_group) -{ - CHECK(!active_net_group_holder.is_any_active(), make_unexpected(HAILO_INVALID_OPERATION), - "network group is currently active. You must deactivate before activating another network_group"); - - // Close older dataflows - auto status = Control::close_all_streams(device); - CHECK_SUCCESS_AS_EXPECTED(status); - - // Reset nn_core before writing configurations - status = device.reset(HAILO_RESET_DEVICE_MODE_NN_CORE); - CHECK_SUCCESS_AS_EXPECTED(status); - - for (auto &m : config) { - status = device.write_memory(m.address, MemoryView(m.data)); - CHECK_SUCCESS_AS_EXPECTED(status); - } - - HcpConfigActivatedNetworkGroup object(device, active_net_group_holder, network_group_name, network_group_params, input_streams, output_streams, - power_mode, std::move(network_group_activated_event), network_group, status); - CHECK_SUCCESS_AS_EXPECTED(status); - return object; -} - -HcpConfigActivatedNetworkGroup::HcpConfigActivatedNetworkGroup( - Device &device, - ActiveNetGroupHolder &active_net_group_holder, - const std::string &network_group_name, - const hailo_activate_network_group_params_t &network_group_params, - std::map> &input_streams, - std::map> &output_streams, - hailo_power_mode_t power_mode, - EventPtr &&network_group_activated_event, - ConfiguredNetworkGroupBase &network_group, hailo_status &status) : - ActivatedNetworkGroupBase(network_group_params, input_streams, output_streams, - std::move(network_group_activated_event), status), - m_active_net_group_holder(active_net_group_holder), - m_is_active(true), - m_power_mode(power_mode), - m_device(device), - m_network_group_name(network_group_name) -{ - // Validate ActivatedNetworkGroup status - if (HAILO_SUCCESS != status) { - return; - } - status = network_group.activate_impl(CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to activate network group"); - return; - } -} - -HcpConfigActivatedNetworkGroup::~HcpConfigActivatedNetworkGroup() -{ - if (!m_is_active) { - return; - } - - auto expected_config_network_ref = m_active_net_group_holder.get(); - if (!expected_config_network_ref.has_value()) { - LOGGER__ERROR("Error getting configured network group"); - return; - } - const auto &config_network_group = expected_config_network_ref.value(); - - const auto status = config_network_group.get().deactivate_impl(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to deactivate network group"); - } -} - -const std::string &HcpConfigActivatedNetworkGroup::get_network_group_name() const -{ - return m_network_group_name; -} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/context_switch/hcp_config_network_group.cpp b/hailort/libhailort/src/context_switch/hcp_config_network_group.cpp deleted file mode 100644 index e6521f2..0000000 --- a/hailort/libhailort/src/context_switch/hcp_config_network_group.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include "context_switch/single_context/hcp_config_network_group.hpp" -#include "network_group_internal.hpp" -#include "control.hpp" - -#define OUTPUT_CHANNEL_INDEX_OFFSET (16) - - -namespace hailort -{ - -HcpConfigNetworkGroup::HcpConfigNetworkGroup(Device &device, ActiveNetGroupHolder &active_net_group_holder, - std::vector &&config, const ConfigureNetworkParams &config_params, NetworkGroupMetadata &&network_group_metadata, - hailo_status &status, std::vector> &&net_flow_ops) - : ConfiguredNetworkGroupBase(config_params, network_group_metadata, std::move(net_flow_ops), status), - m_config(std::move(config)), m_active_net_group_holder(active_net_group_holder), m_device(device) -{} - -Expected> HcpConfigNetworkGroup::create_activated_network_group( - const hailo_activate_network_group_params_t &network_group_params, uint16_t /* dynamic_batch_size */) -{ - auto start_time = std::chrono::steady_clock::now(); - - auto activated_net_group = HcpConfigActivatedNetworkGroup::create(m_device, m_config, name(), network_group_params, - m_input_streams, m_output_streams, m_active_net_group_holder, m_config_params.power_mode, - m_network_group_activated_event, (*this)); - CHECK_EXPECTED(activated_net_group); - - std::unique_ptr activated_net_group_ptr = make_unique_nothrow(activated_net_group.release()); - CHECK_AS_EXPECTED(nullptr != activated_net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); - - auto elapsed_time_ms = std::chrono::duration(std::chrono::steady_clock::now() - start_time).count(); - LOGGER__INFO("Activating {} took {} milliseconds. Note that the function is asynchronous and thus the network is not fully activated yet.", name(), elapsed_time_ms); - - return activated_net_group_ptr; -} - -Expected HcpConfigNetworkGroup::get_default_streams_interface() -{ - return m_device.get_default_streams_interface(); -} - -hailo_status HcpConfigNetworkGroup::set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) -{ - (void) timeout; - (void) network_name; - return HAILO_INVALID_OPERATION; -} - -hailo_status HcpConfigNetworkGroup::set_scheduler_threshold(uint32_t threshold, const std::string &network_name) -{ - (void) threshold; - (void) network_name; - return HAILO_INVALID_OPERATION; -} - -Expected> HcpConfigNetworkGroup::get_latency_meters() -{ - /* hcp does not support latnecy. return empty map */ - LatencyMetersMap empty_map; - return make_shared_nothrow(empty_map); -} - -Expected> HcpConfigNetworkGroup::get_boundary_vdma_channel_by_stream_name( - const std::string &stream_name) -{ - LOGGER__ERROR("get_boundary_vdma_channel_by_stream_name function for stream name {} is not supported in hcp config manager", - stream_name); - return make_unexpected(HAILO_INVALID_OPERATION); -} - -hailo_status HcpConfigNetworkGroup::activate_impl(uint16_t dynamic_batch_size) -{ - m_active_net_group_holder.set(*this); - - auto status = activate_low_level_streams(dynamic_batch_size); - CHECK_SUCCESS(status, "Failed activating low level streams"); - - status = m_network_group_activated_event->signal(); - CHECK_SUCCESS(status, "Failed to signal network activation event"); - - return HAILO_SUCCESS; -} -hailo_status HcpConfigNetworkGroup::deactivate_impl() -{ - auto expected_config_network_ref = m_active_net_group_holder.get(); - CHECK(expected_config_network_ref.has_value(), HAILO_INTERNAL_FAILURE, "Error getting configured network group"); - - const auto &config_network_group = expected_config_network_ref.value(); - // Make sure the network group we are deactivating is this object - CHECK(this == std::addressof(config_network_group.get()), HAILO_INTERNAL_FAILURE, - "Trying to deactivate different network goup"); - - m_active_net_group_holder.clear(); - - if (!m_network_group_activated_event) { - return HAILO_SUCCESS; - } - - m_network_group_activated_event->reset(); - - for (auto &name_pair : m_input_streams) { - const auto status = name_pair.second->flush(); - CHECK_SUCCESS(status, "Failed to flush input stream {}", name_pair.first); - } - - auto status = deactivate_low_level_streams(); - CHECK_SUCCESS(status, "Failed deactivating low level streams"); - - return HAILO_SUCCESS; -} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/context_switch/multi_context/vdma_config_activated_network_group.hpp b/hailort/libhailort/src/context_switch/multi_context/vdma_config_activated_network_group.hpp deleted file mode 100644 index 2c698e0..0000000 --- a/hailort/libhailort/src/context_switch/multi_context/vdma_config_activated_network_group.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file vdma_config_activated_network_group.hpp - * @brief TODO: Represent activated network_group from HEF - **/ - -#ifndef _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_ACTIVATED_NETWORK_GROUP_HPP_ -#define _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_ACTIVATED_NETWORK_GROUP_HPP_ - -#include "hailo/expected.hpp" -#include "vdma_channel.hpp" -#include "pcie_stream.hpp" -#include "context_switch/active_network_group_holder.hpp" -#include "context_switch/network_group_internal.hpp" -#include "context_switch/multi_context/resource_manager.hpp" - -#include -#include -#include - -namespace hailort -{ - -class VdmaConfigActivatedNetworkGroup : public ActivatedNetworkGroupBase -{ -public: - - static Expected create( - ActiveNetGroupHolder &active_net_group_holder, - const std::string &network_group_name, - std::shared_ptr resources_manager, - const hailo_activate_network_group_params_t &network_group_params, - uint16_t dynamic_batch_size, - std::map> &input_streams, - std::map> &output_streams, - EventPtr network_group_activated_event, - AccumulatorPtr deactivation_time_accumulator, - ConfiguredNetworkGroupBase &network_group); - - virtual ~VdmaConfigActivatedNetworkGroup(); - - VdmaConfigActivatedNetworkGroup(const VdmaConfigActivatedNetworkGroup &other) = delete; - VdmaConfigActivatedNetworkGroup &operator=(const VdmaConfigActivatedNetworkGroup &other) = delete; - VdmaConfigActivatedNetworkGroup &operator=(VdmaConfigActivatedNetworkGroup &&other) = delete; - VdmaConfigActivatedNetworkGroup(VdmaConfigActivatedNetworkGroup &&other) noexcept; - - virtual const std::string &get_network_group_name() const override; - virtual Expected get_intermediate_buffer(const IntermediateBufferKey &key) override; - virtual hailo_status set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) override; - -private: - VdmaConfigActivatedNetworkGroup( - const std::string &network_group_name, - const hailo_activate_network_group_params_t &network_group_params, - uint16_t dynamic_batch_size, - std::map> &input_streams, - std::map> &output_streams, - std::shared_ptr &&resources_manager, - ActiveNetGroupHolder &active_net_group_holder, - EventPtr &&network_group_activated_event, - AccumulatorPtr deactivation_time_accumulator, - ConfiguredNetworkGroupBase &network_group, hailo_status &status); - - std::string m_network_group_name; - bool m_should_reset_network_group; - ActiveNetGroupHolder &m_active_net_group_holder; - std::shared_ptr m_resources_manager; - AccumulatorPtr m_deactivation_time_accumulator; - bool m_keep_nn_config_during_reset; -}; - -} /* namespace hailort */ - -#endif /* _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_ACTIVATED_NETWORK_GROUP_HPP_ */ diff --git a/hailort/libhailort/src/context_switch/multi_context/vdma_config_manager.hpp b/hailort/libhailort/src/context_switch/multi_context/vdma_config_manager.hpp deleted file mode 100644 index f9fff52..0000000 --- a/hailort/libhailort/src/context_switch/multi_context/vdma_config_manager.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file vdma_config_manager.hpp - * @brief Manager of HEF parsing and vdma-configured network groups resources for Pcie devices (both single and multi context) - * - **/ - -#ifndef HAILO_VDMA_CONFIG_MANAGER_HPP_ -#define HAILO_VDMA_CONFIG_MANAGER_HPP_ - -#include "context_switch/multi_context/vdma_config_network_group.hpp" -#include "hailo/hailort.h" -#include "common/utils.hpp" - - -namespace hailort -{ - -class VdmaConfigManager final -{ -public: - VdmaConfigManager() = delete; - - static hailo_status switch_network_group(std::shared_ptr current_active_ng, - std::shared_ptr next_ng, const uint16_t batch_size) - { - auto status = HAILO_UNINITIALIZED; - // If current_active_ng is nullptr - we are activating first network group - if (nullptr != current_active_ng) { - status = current_active_ng->deactivate_impl(); - CHECK_SUCCESS(status, "Failed deactivating current network group"); - - // TODO: MSW-762 - In mercury we need to reset after deactivate in case of mercury - this will be fixed and the - // If will be removed when we make the nn_manager responsible to reset the nn-core - // And if switching to nullptr (which is final deactivate - we must also reset state machine) - if (Device::Type::CORE == current_active_ng->get_resources_manager()->get_device().get_type() || - (nullptr == next_ng)) { - status = current_active_ng->get_resources_manager()->reset_state_machine(false); - CHECK_SUCCESS(status, "Failed to reset state machine in switch network group"); - } - } - - // If next_ng is nullptr we are deactivating last network group - if (nullptr != next_ng) { - status = next_ng->activate_impl(batch_size); - CHECK_SUCCESS(status, "Failed activating network group"); - } - - return HAILO_SUCCESS; - } -}; - -} /* namespace hailort */ - -#endif /* HAILO_VDMA_CONFIG_MANAGER_HPP_ */ diff --git a/hailort/libhailort/src/context_switch/multi_context/vdma_config_network_group.hpp b/hailort/libhailort/src/context_switch/multi_context/vdma_config_network_group.hpp deleted file mode 100644 index 494a08f..0000000 --- a/hailort/libhailort/src/context_switch/multi_context/vdma_config_network_group.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file vdma_config_network_group.hpp - * @brief Represent network_group from HEF file that can be activated - * - * This network_group can be used for both single or multi context network_groups but for PCIE only - **/ - -#ifndef _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_NETWORK_GROUP_HPP_ -#define _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_NETWORK_GROUP_HPP_ - -#include "hailo/hailort.h" -#include "common/utils.hpp" -#include "control_protocol.h" -#include "hailort_defaults.hpp" -#include "vdma_channel.hpp" -#include "context_switch/network_group_internal.hpp" -#include "context_switch/multi_context/resource_manager.hpp" -#include "context_switch/multi_context/vdma_config_activated_network_group.hpp" -#include "context_switch/active_network_group_holder.hpp" - -#include -#include -#include -#include - -namespace hailort -{ - - -class VdmaConfigNetworkGroup : public ConfiguredNetworkGroupBase -{ -public: - static Expected create(ActiveNetGroupHolder &active_net_group_holder, - const ConfigureNetworkParams &config_params, - std::shared_ptr resources_managers, const std::string &hef_hash, - std::shared_ptr network_group_metadata, - std::vector> &&net_flow_ops); - - std::shared_ptr &get_resources_manager() - { - return m_resources_manager; - } - - // Functions to activate and deactivate network group for scheduler - dont create ActivatedNetworkGroup objects - virtual hailo_status activate_impl(uint16_t dynamic_batch_size) override; - virtual hailo_status deactivate_impl() override; - - virtual Expected> create_activated_network_group( - const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size) override; - - virtual Expected get_default_streams_interface() override; - - virtual Expected> get_latency_meters() override; - virtual Expected> get_boundary_vdma_channel_by_stream_name( - const std::string &stream_name) override; - - virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) override; - virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) override; - - virtual ~VdmaConfigNetworkGroup() = default; - VdmaConfigNetworkGroup(const VdmaConfigNetworkGroup &other) = delete; - VdmaConfigNetworkGroup &operator=(const VdmaConfigNetworkGroup &other) = delete; - VdmaConfigNetworkGroup &operator=(VdmaConfigNetworkGroup &&other) = delete; - VdmaConfigNetworkGroup(VdmaConfigNetworkGroup &&other) noexcept : ConfiguredNetworkGroupBase(std::move(other)), - m_active_net_group_holder(other.m_active_net_group_holder), - m_resources_manager(std::move(other.m_resources_manager)), - m_hef_hash(std::move(other.m_hef_hash)) - {} - - bool equals(const Hef &hef, const std::string &network_group_name) { - return (network_group_name == name()) && (hef.hash() == m_hef_hash); - } - -private: - VdmaConfigNetworkGroup(ActiveNetGroupHolder &active_net_group_holder, - const ConfigureNetworkParams &config_params, - std::shared_ptr &&resources_manager, const std::string &hef_hash, - const NetworkGroupMetadata &network_group_metadata, hailo_status &status, - std::vector> &&net_flow_ops); - - ActiveNetGroupHolder &m_active_net_group_holder; - std::shared_ptr m_resources_manager; - std::string m_hef_hash; - - friend class VDeviceNetworkGroupWrapper; -}; - -} /* namespace hailort */ - -#endif /* _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_NETWORK_GROUP_HPP_ */ diff --git a/hailort/libhailort/src/context_switch/single_context/hcp_config_activated_network_group.hpp b/hailort/libhailort/src/context_switch/single_context/hcp_config_activated_network_group.hpp deleted file mode 100644 index 2b058e3..0000000 --- a/hailort/libhailort/src/context_switch/single_context/hcp_config_activated_network_group.hpp +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file hcp_config_activated_network_group.hpp - * @brief Represent activated network_group from HEF. - * - * This network_group can be used for control-cofigured network_groups only (for etherent or pcie) - **/ - -#ifndef _HAILO_CONTEXT_SWITCH_HCP_CONFIG_ACTIVATED_NETWORK_GROUP_HPP_ -#define _HAILO_CONTEXT_SWITCH_HCP_CONFIG_ACTIVATED_NETWORK_GROUP_HPP_ - -#include "hailo/device.hpp" -#include "common/utils.hpp" -#include "context_switch/network_group_internal.hpp" -#include "context_switch/active_network_group_holder.hpp" - -#include -#include - -namespace hailort -{ - -struct WriteMemoryInfo -{ - uint32_t address; - Buffer data; -}; - -class HcpConfigActivatedNetworkGroup : public ActivatedNetworkGroupBase -{ - public: - static Expected create(Device &device, std::vector &config, - const std::string &network_group_name, - const hailo_activate_network_group_params_t &network_group_params, - std::map> &input_streams, - std::map> &output_streams, - ActiveNetGroupHolder &active_net_group_holder, - hailo_power_mode_t power_mode, EventPtr network_group_activated_event, - ConfiguredNetworkGroupBase &network_group); - - virtual ~HcpConfigActivatedNetworkGroup(); - HcpConfigActivatedNetworkGroup(const HcpConfigActivatedNetworkGroup &) = delete; - HcpConfigActivatedNetworkGroup &operator=(const HcpConfigActivatedNetworkGroup &) = delete; - HcpConfigActivatedNetworkGroup &operator=(HcpConfigActivatedNetworkGroup &&) = delete; - HcpConfigActivatedNetworkGroup(HcpConfigActivatedNetworkGroup &&other) noexcept : - ActivatedNetworkGroupBase(std::move(other)), m_active_net_group_holder(other.m_active_net_group_holder), - m_is_active(std::exchange(other.m_is_active, false)), m_power_mode(other.m_power_mode), - m_device(other.m_device), m_network_group_name(std::move(other.m_network_group_name)) {}; - - virtual const std::string &get_network_group_name() const override; - - virtual Expected get_intermediate_buffer(const IntermediateBufferKey &/*key*/) override - { - LOGGER__ERROR("get_intermediate_buffer() is not supported on single_context network_groups"); - return make_unexpected(HAILO_INVALID_OPERATION); - } - - virtual hailo_status set_keep_nn_config_during_reset(const bool /* keep_nn_config_during_reset */) override - { - LOGGER__ERROR("set_keep_nn_config_during_reset() is not supported on single_context network_groups"); - return HAILO_INVALID_OPERATION; - } - - private: - HcpConfigActivatedNetworkGroup(Device &device, ActiveNetGroupHolder &active_net_group_holder, - const std::string &network_group_name, - const hailo_activate_network_group_params_t &network_group_params, - std::map> &input_streams, - std::map> &output_streams, - hailo_power_mode_t power_mode, EventPtr &&network_group_activated_event, - ConfiguredNetworkGroupBase &network_group, hailo_status &status); - - ActiveNetGroupHolder &m_active_net_group_holder; - bool m_is_active; - hailo_power_mode_t m_power_mode; - Device &m_device; - std::string m_network_group_name; -}; - -} /* namespace hailort */ - -#endif /* _HAILO_CONTEXT_SWITCH_HCP_CONFIG_ACTIVATED_NETWORK_GROUP_HPP_ */ diff --git a/hailort/libhailort/src/context_switch/single_context/hcp_config_network_group.hpp b/hailort/libhailort/src/context_switch/single_context/hcp_config_network_group.hpp deleted file mode 100644 index 7645735..0000000 --- a/hailort/libhailort/src/context_switch/single_context/hcp_config_network_group.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file hcp_config_network_group.hpp - * @brief Represent network_group from HEF file that can be activated - * - * This network_group can be used for control-configured network_groups (for etherent or pcie) - **/ - -#ifndef _HAILO_CONTEXT_SWITCH_HCP_CONFIG_NETWORK_GROUP_HPP_ -#define _HAILO_CONTEXT_SWITCH_HCP_CONFIG_NETWORK_GROUP_HPP_ - -#include "hailo/device.hpp" -#include "common/utils.hpp" -#include "context_switch/network_group_internal.hpp" -#include "context_switch/active_network_group_holder.hpp" -#include "hailort_defaults.hpp" -#include "context_switch/single_context/hcp_config_activated_network_group.hpp" - -#include -#include - -namespace hailort -{ - -class HcpConfigNetworkGroup : public ConfiguredNetworkGroupBase -{ -public: - HcpConfigNetworkGroup( - Device &device, ActiveNetGroupHolder &active_net_group_holder, std::vector &&config, - const ConfigureNetworkParams &config_params, NetworkGroupMetadata &&network_group_metadata, hailo_status &status, - std::vector> &&net_flow_ops); - - virtual Expected> create_activated_network_group( - const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size) override; - virtual Expected get_default_streams_interface() override; - - virtual Expected> get_latency_meters() override; - virtual Expected> get_boundary_vdma_channel_by_stream_name( - const std::string &stream_name) override; - virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) override; - virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) override; - - virtual hailo_status activate_impl(uint16_t dynamic_batch_size) override; - virtual hailo_status deactivate_impl() override; - - virtual ~HcpConfigNetworkGroup() = default; - HcpConfigNetworkGroup(const HcpConfigNetworkGroup &other) = delete; - HcpConfigNetworkGroup &operator=(const HcpConfigNetworkGroup &other) = delete; - HcpConfigNetworkGroup &operator=(HcpConfigNetworkGroup &&other) = delete; - HcpConfigNetworkGroup(HcpConfigNetworkGroup &&other) noexcept : ConfiguredNetworkGroupBase(std::move(other)), - m_config(std::move(other.m_config)), m_active_net_group_holder(other.m_active_net_group_holder), - m_device(other.m_device) {} - -private: - std::vector m_config; - ActiveNetGroupHolder &m_active_net_group_holder; - Device &m_device; -}; - -} /* namespace hailort */ - -#endif /* _HAILO_CONTEXT_SWITCH_HCP_CONFIG_NETWORK_GROUP_HPP_ */ diff --git a/hailort/libhailort/src/context_switch/vdevice_network_group.cpp b/hailort/libhailort/src/context_switch/vdevice_network_group.cpp deleted file mode 100644 index 11ff46c..0000000 --- a/hailort/libhailort/src/context_switch/vdevice_network_group.cpp +++ /dev/null @@ -1,368 +0,0 @@ -/** - * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file vdevice_network_group.cpp - * @brief: Network Group Wrapper - **/ - -#include "vdevice_network_group.hpp" -#include "vdevice_stream.hpp" -#include "vdevice_stream_multiplexer_wrapper.hpp" -#include "vstream_internal.hpp" -#include "tracer_macros.hpp" - -namespace hailort -{ - -Expected> VDeviceActivatedNetworkGroup::create( - std::vector> configured_network_groups, - std::map> &input_streams, - std::map> &output_streams, - const hailo_activate_network_group_params_t &network_group_params, - EventPtr network_group_activated_event, uint16_t dynamic_batch_size, - AccumulatorPtr deactivation_time_accumulator) -{ - auto status = HAILO_UNINITIALIZED; - std::vector> activated_network_groups; - activated_network_groups.reserve(configured_network_groups.size()); - for (auto cng : configured_network_groups) { - auto ang = cng->create_activated_network_group(network_group_params, dynamic_batch_size); - CHECK_EXPECTED(ang); - activated_network_groups.emplace_back(ang.release()); - } - auto ang = VDeviceActivatedNetworkGroup(std::move(activated_network_groups), input_streams, output_streams, - network_group_params, network_group_activated_event, deactivation_time_accumulator, status); - - CHECK_SUCCESS_AS_EXPECTED(status); - std::unique_ptr activated_net_group_ptr = - make_unique_nothrow(std::move(ang)); - CHECK_AS_EXPECTED(nullptr != activated_net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); - - status = network_group_activated_event->signal(); - CHECK_SUCCESS_AS_EXPECTED(status, "Failed to signal network activation event"); - - return activated_net_group_ptr; -} - -const std::string &VDeviceActivatedNetworkGroup::get_network_group_name() const -{ - // network_group_name is same across all NGs - return m_activated_network_groups[0]->get_network_group_name(); -} - -Expected VDeviceActivatedNetworkGroup::get_intermediate_buffer(const IntermediateBufferKey &key) -{ - CHECK_AS_EXPECTED(1 == m_activated_network_groups.size(), HAILO_INVALID_OPERATION, "getting intermediate buffer is supported only over single device"); - return m_activated_network_groups[0]->get_intermediate_buffer(key); -} - -hailo_status VDeviceActivatedNetworkGroup::set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) -{ - for (auto &activated_network_group : m_activated_network_groups) { - auto status = activated_network_group->set_keep_nn_config_during_reset(keep_nn_config_during_reset); - CHECK_SUCCESS(status); - } - return HAILO_SUCCESS; -} - -VDeviceActivatedNetworkGroup::VDeviceActivatedNetworkGroup(std::vector> &&activated_network_groups, - std::map> &input_streams, std::map> &output_streams, - const hailo_activate_network_group_params_t &network_group_params, EventPtr network_group_activated_event, AccumulatorPtr deactivation_time_accumulator, hailo_status &status) - : ActivatedNetworkGroupBase(network_group_params, input_streams, output_streams, std::move(network_group_activated_event), status), - m_activated_network_groups(std::move(activated_network_groups)), m_should_reset_network_group(true), m_deactivation_time_accumulator(deactivation_time_accumulator) -{ -} - -VDeviceActivatedNetworkGroup::VDeviceActivatedNetworkGroup(VDeviceActivatedNetworkGroup &&other) noexcept : - ActivatedNetworkGroupBase(std::move(other)), - m_activated_network_groups(std::move(other.m_activated_network_groups)), - m_should_reset_network_group(std::exchange(other.m_should_reset_network_group, false)), - m_deactivation_time_accumulator(std::move(other.m_deactivation_time_accumulator)) -{ -} - - -Expected> VDeviceNetworkGroup::create(std::vector> configured_network_group, - NetworkGroupSchedulerWeakPtr network_group_scheduler) -{ - auto status = HAILO_UNINITIALIZED; - std::vector> vdma_config_ngs; - vdma_config_ngs.reserve(configured_network_group.size()); - for (auto &network_group : configured_network_group) { - auto vdma_ng = std::dynamic_pointer_cast(network_group); - assert(nullptr != vdma_ng); - vdma_config_ngs.push_back(vdma_ng); - } - - auto net_flow_ops_copy = vdma_config_ngs[0]->m_net_flow_ops; - - VDeviceNetworkGroup object(std::move(vdma_config_ngs), network_group_scheduler, std::move(net_flow_ops_copy), status); - CHECK_SUCCESS_AS_EXPECTED(status); - - auto obj_ptr = make_shared_nothrow(std::move(object)); - CHECK_NOT_NULL_AS_EXPECTED(obj_ptr, HAILO_OUT_OF_HOST_MEMORY); - - return obj_ptr; -} - -Expected> VDeviceNetworkGroup::duplicate(std::shared_ptr other) -{ - auto status = HAILO_UNINITIALIZED; - auto net_flow_ops_copy = other->m_net_flow_ops; - - VDeviceNetworkGroup object(other->m_configured_network_groups, other->m_network_group_scheduler, std::move(net_flow_ops_copy), status); - CHECK_SUCCESS_AS_EXPECTED(status); - - auto obj_ptr = make_shared_nothrow(std::move(object)); - CHECK_NOT_NULL_AS_EXPECTED(obj_ptr, HAILO_OUT_OF_HOST_MEMORY); - - return obj_ptr; -} - - -VDeviceNetworkGroup::VDeviceNetworkGroup(std::vector> configured_network_group, - NetworkGroupSchedulerWeakPtr network_group_scheduler, std::vector> &&net_flow_ops, hailo_status &status) : - ConfiguredNetworkGroupBase(configured_network_group[0]->m_config_params, - configured_network_group[0]->m_network_group_metadata, std::move(net_flow_ops), status), - m_configured_network_groups(std::move(configured_network_group)), - m_network_group_scheduler(network_group_scheduler), - m_scheduler_handle(INVALID_NETWORK_GROUP_HANDLE), - m_multiplexer_handle(0), - m_multiplexer() -{} - -Expected VDeviceNetworkGroup::get_default_streams_interface() -{ - auto first_streams_interface = m_configured_network_groups[0]->get_default_streams_interface(); - CHECK_EXPECTED(first_streams_interface); -#ifndef NDEBUG - // Check that all physical devices has the same interface - for (auto &network_group : m_configured_network_groups) { - auto iface = network_group->get_default_streams_interface(); - CHECK_EXPECTED(iface); - CHECK_AS_EXPECTED(iface.value() == first_streams_interface.value(), HAILO_INTERNAL_FAILURE, - "Not all default stream interfaces are the same"); - } -#endif - return first_streams_interface; -} - -hailo_status VDeviceNetworkGroup::create_vdevice_streams_from_config_params(std::shared_ptr multiplexer, scheduler_ng_handle_t scheduler_handle) -{ - // TODO - HRT-6931 - raise error on this case - if (((m_config_params.latency & HAILO_LATENCY_MEASURE) == HAILO_LATENCY_MEASURE) && (1 < m_configured_network_groups.size())) { - LOGGER__WARNING("Latency measurement is not supported on more than 1 physical device."); - } - - m_multiplexer = multiplexer; - - for (const auto &stream_parameters_pair : m_config_params.stream_params_by_name) { - switch (stream_parameters_pair.second.direction) { - case HAILO_H2D_STREAM: - { - auto status = create_input_vdevice_stream_from_config_params(stream_parameters_pair.second, - stream_parameters_pair.first, multiplexer, scheduler_handle); - CHECK_SUCCESS(status); - } - break; - case HAILO_D2H_STREAM: - { - auto status = create_output_vdevice_stream_from_config_params(stream_parameters_pair.second, - stream_parameters_pair.first, multiplexer, scheduler_handle); - CHECK_SUCCESS(status); - } - break; - default: - LOGGER__ERROR("stream name {} direction is invalid.", stream_parameters_pair.first); - return HAILO_INVALID_ARGUMENT; - } - } - - for (const auto &input_stream : m_input_streams) { - auto expected_queue_size = static_cast(*input_stream.second).get_buffer_frames_size(); - CHECK_EXPECTED_AS_STATUS(expected_queue_size); - TRACE(CreateNetworkGroupInputStreamsTrace, "", name(), input_stream.first, (uint32_t)expected_queue_size.value()); - } - for (const auto &output_stream : m_output_streams) { - if (static_cast(*output_stream.second).get_layer_info().format.order == hailo_format_order_t::HAILO_FORMAT_ORDER_HAILO_NMS) { - continue; - } - auto expected_queue_size = static_cast(*output_stream.second).get_buffer_frames_size(); - CHECK_EXPECTED_AS_STATUS(expected_queue_size); - TRACE(CreateNetworkGroupOutputStreamsTrace, "", name(), output_stream.first, (uint32_t)expected_queue_size.value()); - } - - auto status = m_multiplexer->add_network_group_instance(m_multiplexer_handle, *this); - CHECK_SUCCESS(status); - - return HAILO_SUCCESS; -} - -hailo_status VDeviceNetworkGroup::create_input_vdevice_stream_from_config_params(const hailo_stream_parameters_t &stream_params, - const std::string &stream_name, std::shared_ptr multiplexer, scheduler_ng_handle_t scheduler_handle) -{ - auto edge_layer = get_layer_info(stream_name); - CHECK_EXPECTED_AS_STATUS(edge_layer); - - CHECK(HailoRTCommon::is_vdma_stream_interface(stream_params.stream_interface), HAILO_INVALID_OPERATION, - "Stream {} not supported on VDevice usage. {} has {} interface.", stream_name, stream_params.stream_interface); - - std::vector> low_level_streams; - low_level_streams.reserve(m_configured_network_groups.size()); - for (auto &net_group : m_configured_network_groups) { - auto stream = net_group->get_input_stream_by_name(stream_name); - CHECK(stream, HAILO_INTERNAL_FAILURE); - low_level_streams.emplace_back(dynamic_cast(stream.release().get())); - } - auto input_stream = InputVDeviceBaseStream::create(std::move(low_level_streams), edge_layer.value(), - scheduler_handle, m_network_group_activated_event, m_network_group_scheduler); - CHECK_EXPECTED_AS_STATUS(input_stream); - auto input_stream_wrapper = VDeviceInputStreamMultiplexerWrapper::create(input_stream.release(), edge_layer->network_name, multiplexer, scheduler_handle); - CHECK_EXPECTED_AS_STATUS(input_stream_wrapper); - m_input_streams.insert(make_pair(stream_name, input_stream_wrapper.release())); - - return HAILO_SUCCESS; -} - -hailo_status VDeviceNetworkGroup::create_output_vdevice_stream_from_config_params(const hailo_stream_parameters_t &stream_params, - const std::string &stream_name, std::shared_ptr multiplexer, scheduler_ng_handle_t scheduler_handle) -{ - auto edge_layer = get_layer_info(stream_name); - CHECK_EXPECTED_AS_STATUS(edge_layer); - - CHECK(HailoRTCommon::is_vdma_stream_interface(stream_params.stream_interface), HAILO_INVALID_OPERATION, - "Stream {} not supported on VDevice usage. {} has {} interface.", stream_name, stream_params.stream_interface); - - std::vector> low_level_streams; - low_level_streams.reserve(m_configured_network_groups.size()); - for (auto &net_group : m_configured_network_groups) { - auto stream = net_group->get_output_stream_by_name(stream_name); - CHECK(stream, HAILO_INTERNAL_FAILURE); - low_level_streams.emplace_back(dynamic_cast(stream.release().get())); - } - auto output_stream = OutputVDeviceBaseStream::create(std::move(low_level_streams), edge_layer.value(), - scheduler_handle, m_network_group_activated_event, m_network_group_scheduler); - CHECK_EXPECTED_AS_STATUS(output_stream); - auto output_stream_wrapper = VDeviceOutputStreamMultiplexerWrapper::create(output_stream.release(), edge_layer->network_name, multiplexer, scheduler_handle); - CHECK_EXPECTED_AS_STATUS(output_stream_wrapper); - m_output_streams.insert(make_pair(stream_name, output_stream_wrapper.release())); - - return HAILO_SUCCESS; -} - -hailo_status VDeviceNetworkGroup::create_vdevice_streams_from_duplicate(std::shared_ptr other) -{ - // TODO - HRT-6931 - raise error on this case - if (((m_config_params.latency & HAILO_LATENCY_MEASURE) == HAILO_LATENCY_MEASURE) && (1 < m_configured_network_groups.size())) { - LOGGER__WARNING("Latency measurement is not supported on more than 1 physical device."); - } - - m_multiplexer = other->m_multiplexer; - m_multiplexer_handle = other->multiplexer_duplicates_count() + 1; - - for (auto &name_stream_pair : other->m_input_streams) { - auto input_stream = static_cast(name_stream_pair.second.get()); - auto copy = input_stream->clone(m_multiplexer_handle); - CHECK_EXPECTED_AS_STATUS(copy); - - m_input_streams.insert(make_pair(name_stream_pair.first, copy.release())); - } - - for (auto &name_stream_pair : other->m_output_streams) { - auto output_stream = static_cast(name_stream_pair.second.get()); - auto copy = output_stream->clone(m_multiplexer_handle); - CHECK_EXPECTED_AS_STATUS(copy); - - m_output_streams.insert(make_pair(name_stream_pair.first, copy.release())); - } - - auto status = other->m_multiplexer->add_network_group_instance(m_multiplexer_handle, *this); - CHECK_SUCCESS(status); - - return HAILO_SUCCESS; -} - -void VDeviceNetworkGroup::set_network_group_handle(scheduler_ng_handle_t handle) -{ - m_scheduler_handle = handle; -} - -scheduler_ng_handle_t VDeviceNetworkGroup::network_group_handle() const -{ - return m_scheduler_handle; -} - -hailo_status VDeviceNetworkGroup::set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) -{ - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INVALID_OPERATION, - "Cannot set scheduler timeout for network group {}, as it is configured on a vdevice which does not have scheduling enabled", name()); - if (network_name != HailoRTDefaults::get_network_name(name())) { - CHECK(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Setting scheduler timeout for a specific network is currently not supported"); - } - auto status = network_group_scheduler->set_timeout(m_scheduler_handle, timeout, network_name); - CHECK_SUCCESS(status); - return HAILO_SUCCESS; -} - -hailo_status VDeviceNetworkGroup::set_scheduler_threshold(uint32_t threshold, const std::string &network_name) -{ - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INVALID_OPERATION, - "Cannot set scheduler threshold for network group {}, as it is configured on a vdevice which does not have scheduling enabled", name()); - if (network_name != HailoRTDefaults::get_network_name(name())) { - CHECK(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Setting scheduler threshold for a specific network is currently not supported"); - } - auto status = network_group_scheduler->set_threshold(m_scheduler_handle, threshold, network_name); - CHECK_SUCCESS(status); - return HAILO_SUCCESS; -} - -Expected> VDeviceNetworkGroup::get_latency_meters() -{ - return m_configured_network_groups[0]->get_latency_meters(); -} - -Expected> VDeviceNetworkGroup::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) -{ - if (1 < m_configured_network_groups.size()) { - LOGGER__ERROR("get_boundary_vdma_channel_by_stream_name function is not supported on more than 1 physical device."); - return make_unexpected(HAILO_INVALID_OPERATION); - } - - return m_configured_network_groups[0]->get_boundary_vdma_channel_by_stream_name(stream_name); -} - -Expected> VDeviceNetworkGroup::create_output_vstreams(const std::map &outputs_params) -{ - auto expected = ConfiguredNetworkGroupBase::create_output_vstreams(outputs_params); - CHECK_EXPECTED(expected); - - if (nullptr == m_multiplexer) { - return expected.release(); - } - - m_multiplexer->set_output_vstreams_names(m_multiplexer_handle, expected.value()); - - for (auto &vstream : expected.value()) { - static_cast(*vstream.m_vstream).set_on_vstream_cant_read_callback([this, name = vstream.name()] () { - m_multiplexer->set_can_output_vstream_read(m_multiplexer_handle, name, false); - }); - static_cast(*vstream.m_vstream).set_on_vstream_can_read_callback([this, name = vstream.name()] () { - m_multiplexer->set_can_output_vstream_read(m_multiplexer_handle, name, true); - }); - } - - return expected.release(); -} - -Expected> VDeviceNetworkGroup::get_network_group_by_device_index(uint32_t device_index) -{ - CHECK_AS_EXPECTED(device_index - -namespace hailort -{ - -class VDeviceActivatedNetworkGroup : public ActivatedNetworkGroupBase -{ -public: - static Expected> create(std::vector> configured_network_groups, - std::map> &input_streams, - std::map> &output_streams, - const hailo_activate_network_group_params_t &network_group_params, EventPtr network_group_activated_event, - uint16_t dynamic_batch_size, AccumulatorPtr deactivation_time_accumulator); - - virtual ~VDeviceActivatedNetworkGroup() - { - if (!m_should_reset_network_group) { - return; - } - const auto start_time = std::chrono::steady_clock::now(); - - m_network_group_activated_event->reset(); - m_activated_network_groups.clear(); - - const auto elapsed_time_ms = std::chrono::duration( - std::chrono::steady_clock::now() - start_time).count(); - LOGGER__INFO("Deactivating took {} ms", elapsed_time_ms); - m_deactivation_time_accumulator->add_data_point(elapsed_time_ms); - } - - VDeviceActivatedNetworkGroup(const VDeviceActivatedNetworkGroup &other) = delete; - VDeviceActivatedNetworkGroup &operator=(const VDeviceActivatedNetworkGroup &other) = delete; - VDeviceActivatedNetworkGroup &operator=(VDeviceActivatedNetworkGroup &&other) = delete; - VDeviceActivatedNetworkGroup(VDeviceActivatedNetworkGroup &&other) noexcept; - - virtual const std::string &get_network_group_name() const override; - virtual Expected get_intermediate_buffer(const IntermediateBufferKey &key) override; - virtual hailo_status set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) override; - -private: - VDeviceActivatedNetworkGroup( - std::vector> &&activated_network_groups, - std::map> &input_streams, - std::map> &output_streams, - const hailo_activate_network_group_params_t &network_group_params, EventPtr network_group_activated_event, - AccumulatorPtr deactivation_time_accumulator, hailo_status &status); - - std::vector> m_activated_network_groups; - bool m_should_reset_network_group; - AccumulatorPtr m_deactivation_time_accumulator; -}; - -class VDeviceNetworkGroup : public ConfiguredNetworkGroupBase -{ -public: - // TODO (HRT-8751): remove duplicate members from this class or from vdma_config_network _group - static Expected> create(std::vector> configured_network_group, - NetworkGroupSchedulerWeakPtr network_group_scheduler); - - static Expected> duplicate(std::shared_ptr other); - - virtual ~VDeviceNetworkGroup() = default; - VDeviceNetworkGroup(const VDeviceNetworkGroup &other) = delete; - VDeviceNetworkGroup &operator=(const VDeviceNetworkGroup &other) = delete; - VDeviceNetworkGroup &operator=(VDeviceNetworkGroup &&other) = delete; - VDeviceNetworkGroup(VDeviceNetworkGroup &&other) = default; - - // Function from vdma network group - hailo_status create_vdevice_streams_from_config_params(std::shared_ptr multiplexer, - scheduler_ng_handle_t scheduler_handle); - hailo_status create_input_vdevice_stream_from_config_params( - const hailo_stream_parameters_t &stream_params, const std::string &stream_name, - std::shared_ptr multiplexer, scheduler_ng_handle_t scheduler_handle); - hailo_status create_output_vdevice_stream_from_config_params( - const hailo_stream_parameters_t &stream_params, const std::string &stream_name, - std::shared_ptr multiplexer, scheduler_ng_handle_t scheduler_handle); - - hailo_status create_vdevice_streams_from_duplicate(std::shared_ptr other); - - bool equals(const Hef &hef, const std::string &network_group_name) - { - return m_configured_network_groups[0]->equals(hef, network_group_name); - } - - uint32_t multiplexer_duplicates_count() - { - assert(m_multiplexer->instances_count() > 0); - return static_cast(m_multiplexer->instances_count() - 1); - } - - virtual Expected get_default_streams_interface() override; - - virtual Expected> get_latency_meters() override; - virtual Expected> get_boundary_vdma_channel_by_stream_name( - const std::string &stream_name) override; - - void set_network_group_handle(scheduler_ng_handle_t handle); - scheduler_ng_handle_t network_group_handle() const; - virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) override; - virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) override; - - virtual Expected> create_output_vstreams(const std::map &outputs_params) override; - - virtual hailo_status wait_for_activation(const std::chrono::milliseconds &timeout) override - { - CHECK(!m_network_group_scheduler.lock(), HAILO_INVALID_OPERATION, - "Waiting for network group activation is not allowed when the network group scheduler is active!"); - - return m_network_group_activated_event->wait(timeout); - } - - virtual hailo_status activate_impl(uint16_t /*dynamic_batch_size*/) override - { - return HAILO_INTERNAL_FAILURE; - } - - virtual hailo_status deactivate_impl() override - { - return HAILO_INTERNAL_FAILURE; - } - - virtual Expected> create_activated_network_group( - const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size) override - { - auto start_time = std::chrono::steady_clock::now(); - - CHECK_AS_EXPECTED(!m_network_group_scheduler.lock(), HAILO_INVALID_OPERATION, - "Manually activating a network group is not allowed when the network group scheduler is active!"); - - auto res = VDeviceActivatedNetworkGroup::create(m_configured_network_groups, m_input_streams, m_output_streams, - network_group_params, m_network_group_activated_event, dynamic_batch_size, m_deactivation_time_accumulator); - const auto elapsed_time_ms = std::chrono::duration( - std::chrono::steady_clock::now() - start_time).count(); - CHECK_EXPECTED(res); - - LOGGER__INFO("Activating {} on VDevice took {} milliseconds. Note that the function is asynchronous and" - " thus the network is not fully activated yet.", name(), elapsed_time_ms); - m_activation_time_accumulator->add_data_point(elapsed_time_ms); - - return res; - } - - Expected> get_network_group_by_device_index(uint32_t device_index); - -private: - VDeviceNetworkGroup(std::vector> configured_network_group, - NetworkGroupSchedulerWeakPtr network_group_scheduler, std::vector> &&net_flow_ops, - hailo_status &status); - - std::vector> m_configured_network_groups; - NetworkGroupSchedulerWeakPtr m_network_group_scheduler; - scheduler_ng_handle_t m_scheduler_handle; - multiplexer_ng_handle_t m_multiplexer_handle; - std::shared_ptr m_multiplexer; -}; - -} - -#endif /* _HAILO_VDEVICE_NETWORK_GROUP_WRAPPER_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/context_switch/vdma_config_activated_network_group.cpp b/hailort/libhailort/src/context_switch/vdma_config_activated_network_group.cpp deleted file mode 100644 index a7e8a6e..0000000 --- a/hailort/libhailort/src/context_switch/vdma_config_activated_network_group.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file multi_context_activatedN_network_group.cpp - * @brief VdmaConfigActivatedNetworkGroup implementation - **/ - -#include "context_switch/multi_context/vdma_config_activated_network_group.hpp" -#include "control.hpp" -#include - -namespace hailort -{ - -Expected VdmaConfigActivatedNetworkGroup::create( - ActiveNetGroupHolder &active_net_group_holder, - const std::string &network_group_name, - std::shared_ptr resources_manager, - const hailo_activate_network_group_params_t &network_group_params, - uint16_t dynamic_batch_size, - std::map> &input_streams, - std::map> &output_streams, - EventPtr network_group_activated_event, - AccumulatorPtr deactivation_time_accumulator, - ConfiguredNetworkGroupBase &network_group) -{ - CHECK(!active_net_group_holder.is_any_active(), make_unexpected(HAILO_INVALID_OPERATION), - "network group is currently active. You must deactivate before activating another network_group"); - - CHECK_ARG_NOT_NULL_AS_EXPECTED(deactivation_time_accumulator); - - auto status = HAILO_UNINITIALIZED; - VdmaConfigActivatedNetworkGroup object(network_group_name, network_group_params, dynamic_batch_size, input_streams, output_streams, - std::move(resources_manager), active_net_group_holder, std::move(network_group_activated_event), - deactivation_time_accumulator, network_group, status); - CHECK_SUCCESS_AS_EXPECTED(status); - - return object; -} - -VdmaConfigActivatedNetworkGroup::VdmaConfigActivatedNetworkGroup( - const std::string &network_group_name, - const hailo_activate_network_group_params_t &network_group_params, - uint16_t dynamic_batch_size, - std::map> &input_streams, - std::map> &output_streams, - std::shared_ptr &&resources_manager, - ActiveNetGroupHolder &active_net_group_holder, - EventPtr &&network_group_activated_event, - AccumulatorPtr deactivation_time_accumulator, - ConfiguredNetworkGroupBase &network_group, - hailo_status &status) : - ActivatedNetworkGroupBase(network_group_params, input_streams, output_streams, - std::move(network_group_activated_event), status), - m_network_group_name(network_group_name), - m_should_reset_network_group(true), - m_active_net_group_holder(active_net_group_holder), - m_resources_manager(std::move(resources_manager)), - m_deactivation_time_accumulator(deactivation_time_accumulator), - m_keep_nn_config_during_reset(false) -{ - // Validate ActivatedNetworkGroup status - if (HAILO_SUCCESS != status) { - return; - } - - // We know network_group is a VdmaConfigNetworkGroup - status = network_group.activate_impl(dynamic_batch_size); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Error activating network group"); - return; - } -} - -VdmaConfigActivatedNetworkGroup::VdmaConfigActivatedNetworkGroup(VdmaConfigActivatedNetworkGroup &&other) noexcept : - ActivatedNetworkGroupBase(std::move(other)), - m_network_group_name(std::move(other.m_network_group_name)), - m_should_reset_network_group(std::exchange(other.m_should_reset_network_group, false)), - m_active_net_group_holder(other.m_active_net_group_holder), - m_resources_manager(std::move(other.m_resources_manager)), - m_deactivation_time_accumulator(std::move(other.m_deactivation_time_accumulator)), - m_keep_nn_config_during_reset(std::move(other.m_keep_nn_config_during_reset)) -{} - -VdmaConfigActivatedNetworkGroup::~VdmaConfigActivatedNetworkGroup() -{ - if (!m_should_reset_network_group) { - return; - } - - auto status = HAILO_UNINITIALIZED; - const auto start_time = std::chrono::steady_clock::now(); - - auto config_network_group_ref = m_active_net_group_holder.get(); - if (!config_network_group_ref.has_value()) { - LOGGER__ERROR("Error getting configured network group"); - return; - } - - auto vdma_config_network_group = config_network_group_ref.value(); - - status = vdma_config_network_group.get().deactivate_impl(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed deactivating network group"); - } - - status = m_resources_manager->reset_state_machine(m_keep_nn_config_during_reset); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to reset context switch with status {}", status); - } - - const auto elapsed_time_ms = std::chrono::duration( - std::chrono::steady_clock::now() - start_time).count(); - LOGGER__INFO("Deactivating took {} ms", elapsed_time_ms); - m_deactivation_time_accumulator->add_data_point(elapsed_time_ms); -} - -const std::string &VdmaConfigActivatedNetworkGroup::get_network_group_name() const -{ - return m_network_group_name; -} - -Expected VdmaConfigActivatedNetworkGroup::get_intermediate_buffer(const IntermediateBufferKey &key) -{ - return m_resources_manager->read_intermediate_buffer(key); -} - -hailo_status VdmaConfigActivatedNetworkGroup::set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) -{ - m_keep_nn_config_during_reset = keep_nn_config_during_reset; - return HAILO_SUCCESS; -} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/context_switch/vdma_config_network_group.cpp b/hailort/libhailort/src/context_switch/vdma_config_network_group.cpp deleted file mode 100644 index 42f500d..0000000 --- a/hailort/libhailort/src/context_switch/vdma_config_network_group.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include "tracer_macros.hpp" -#include "context_switch/multi_context/vdma_config_network_group.hpp" -#include "network_group_internal.hpp" -#include "eth_stream.hpp" -#include "pcie_stream.hpp" -#include "mipi_stream.hpp" -#include "vstream_internal.hpp" - -namespace hailort -{ - -Expected VdmaConfigNetworkGroup::create(ActiveNetGroupHolder &active_net_group_holder, - const ConfigureNetworkParams &config_params, - std::shared_ptr resources_manager, const std::string &hef_hash, - std::shared_ptr network_group_metadata, - std::vector> &&net_flow_ops) -{ - auto status = HAILO_UNINITIALIZED; - - VdmaConfigNetworkGroup object(active_net_group_holder, config_params, - std::move(resources_manager), hef_hash, *network_group_metadata, status, std::move(net_flow_ops)); - CHECK_SUCCESS_AS_EXPECTED(status); - - return object; -} - -VdmaConfigNetworkGroup::VdmaConfigNetworkGroup(ActiveNetGroupHolder &active_net_group_holder, - const ConfigureNetworkParams &config_params, - std::shared_ptr &&resources_manager, const std::string &hef_hash, - const NetworkGroupMetadata &network_group_metadata, hailo_status &status, - std::vector> &&net_flow_ops) : - ConfiguredNetworkGroupBase(config_params, - network_group_metadata, std::move(net_flow_ops), status), - m_active_net_group_holder(active_net_group_holder), - m_resources_manager(std::move(resources_manager)), - m_hef_hash(hef_hash) -{} - -hailo_status VdmaConfigNetworkGroup::activate_impl(uint16_t dynamic_batch_size) -{ - auto status = HAILO_UNINITIALIZED; - - // Check that no network is currently activated - CHECK(!m_active_net_group_holder.is_any_active(), HAILO_INTERNAL_FAILURE, - "Cant activate network because a network is already activated"); - - m_active_net_group_holder.set(*this); - - status = m_resources_manager->register_fw_managed_vdma_channels(); - CHECK_SUCCESS(status, "Failed to start fw managed vdma channels."); - - status = m_resources_manager->set_inter_context_channels_dynamic_batch_size(dynamic_batch_size); - CHECK_SUCCESS(status, "Failed to set inter-context channels dynamic batch size."); - - status = m_resources_manager->enable_state_machine(dynamic_batch_size); - CHECK_SUCCESS(status, "Failed to activate state-machine"); - - status = activate_low_level_streams(dynamic_batch_size); - CHECK_SUCCESS(status, "Failed to activate low level streams"); - - status = m_network_group_activated_event->signal(); - CHECK_SUCCESS(status, "Failed to signal network activation event"); - - return HAILO_SUCCESS; -} - -hailo_status VdmaConfigNetworkGroup::deactivate_impl() -{ - auto status = HAILO_UNINITIALIZED; - - // Check that network is currently activated - CHECK(m_active_net_group_holder.is_any_active(), HAILO_INTERNAL_FAILURE, - "Cant Deactivate network because no network is already activated"); - - // Make sure the network group we are deactivating is this object - auto config_network_group_ref = m_active_net_group_holder.get().value(); - CHECK(this == std::addressof(config_network_group_ref.get()), HAILO_INTERNAL_FAILURE, - "Trying to deactivate different network goup"); - - m_active_net_group_holder.clear(); - - m_network_group_activated_event->reset(); - - status = deactivate_low_level_streams(); - CHECK_SUCCESS(status, "Failed to deactivate low level streams"); - - status = m_resources_manager->unregister_fw_managed_vdma_channels(); - CHECK_SUCCESS(status, "Failed to stop fw managed vdma channels"); - - return HAILO_SUCCESS; -} - -Expected> VdmaConfigNetworkGroup::create_activated_network_group( - const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size) -{ - auto start_time = std::chrono::steady_clock::now(); - auto activated_net_group = VdmaConfigActivatedNetworkGroup::create( - m_active_net_group_holder, name(), m_resources_manager, network_group_params, dynamic_batch_size, - m_input_streams, m_output_streams, m_network_group_activated_event, m_deactivation_time_accumulator, (*this)); - const auto elapsed_time_ms = std::chrono::duration( - std::chrono::steady_clock::now() - start_time).count(); - CHECK_EXPECTED(activated_net_group); - - LOGGER__INFO("Activating {} took {} milliseconds. Note that the function is asynchronous and" - " thus the network is not fully activated yet.", name(), elapsed_time_ms); - m_activation_time_accumulator->add_data_point(elapsed_time_ms); - - std::unique_ptr activated_net_group_ptr = - make_unique_nothrow(activated_net_group.release()); - CHECK_AS_EXPECTED(nullptr != activated_net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); - - return activated_net_group_ptr; -} - -Expected VdmaConfigNetworkGroup::get_default_streams_interface() -{ - return m_resources_manager->get_default_streams_interface(); -} - -hailo_status VdmaConfigNetworkGroup::set_scheduler_timeout(const std::chrono::milliseconds &/*timeout*/, const std::string &/*network_name*/) -{ - LOGGER__ERROR("Setting scheduler's timeout is only allowed when working with VDevice and scheduler enabled"); - return HAILO_INVALID_OPERATION; -} - -hailo_status VdmaConfigNetworkGroup::set_scheduler_threshold(uint32_t /*threshold*/, const std::string &/*network_name*/) -{ - LOGGER__ERROR("Setting scheduler's threshold is only allowed when working with VDevice and scheduler enabled"); - return HAILO_INVALID_OPERATION; -} - -Expected> VdmaConfigNetworkGroup::get_latency_meters() -{ - auto latency_meters = m_resources_manager->get_latency_meters(); - return make_shared_nothrow(latency_meters); -} - -Expected> VdmaConfigNetworkGroup::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) -{ - return m_resources_manager->get_boundary_vdma_channel_by_stream_name(stream_name); -} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/core_device.cpp b/hailort/libhailort/src/core_device.cpp deleted file mode 100644 index 12d8ff7..0000000 --- a/hailort/libhailort/src/core_device.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "core_device.hpp" -#include "hailo/hailort.h" -#include "hailo/expected.hpp" -#include "common/logger_macros.hpp" -#include "context_switch/multi_context/vdma_config_manager.hpp" -#include "md5.h" - -#include - -static const std::string CORE_DRIVER_PATH = "/dev/hailo_core"; - -namespace hailort -{ - -bool CoreDevice::is_loaded() -{ -#if defined(_MSC_VER) - // windows is not supported for core driver - return false; -#else - return (access(CORE_DRIVER_PATH.c_str(), F_OK) == 0); -#endif // defined(_MSC_VER) -} - -Expected> CoreDevice::create() -{ - hailo_status status = HAILO_UNINITIALIZED; - - auto driver = HailoRTDriver::create(CORE_DRIVER_PATH); - CHECK_EXPECTED(driver, "Failed to initialize HailoRTDriver"); - - auto device = std::unique_ptr(new (std::nothrow) CoreDevice(driver.release(), status, DEVICE_ID)); - CHECK_AS_EXPECTED((nullptr != device), HAILO_OUT_OF_HOST_MEMORY); - CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating CoreDevice"); - - return device; -} - - -CoreDevice::CoreDevice(HailoRTDriver &&driver, hailo_status &status, const std::string &device_id) : - VdmaDevice::VdmaDevice(std::move(driver), Device::Type::CORE, device_id) -{ - status = update_fw_state(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("update_fw_state() failed with status {}", status); - return; - } - - status = HAILO_SUCCESS; -} - -Expected CoreDevice::get_architecture() const { - return Expected(m_device_architecture); -} - -hailo_status CoreDevice::reset_impl(CONTROL_PROTOCOL__reset_type_t reset_type) -{ - if (CONTROL_PROTOCOL__RESET_TYPE__NN_CORE == reset_type) { - return m_driver.reset_nn_core(); - } - - LOGGER__ERROR("Can't reset CoreDevice, please use linux reboot"); - return HAILO_NOT_IMPLEMENTED; -} - -Expected CoreDevice::read_log(MemoryView &buffer, hailo_cpu_id_t cpu_id) -{ - if (hailo_cpu_id_t::HAILO_CPU_ID_0 == cpu_id) { - LOGGER__ERROR("Read FW log is supported only on core CPU"); - return make_unexpected(HAILO_INVALID_ARGUMENT); - } - - return VdmaDevice::read_log(buffer, cpu_id); -} - -} /* namespace hailort */ \ No newline at end of file diff --git a/hailort/libhailort/src/core_op/CMakeLists.txt b/hailort/libhailort/src/core_op/CMakeLists.txt new file mode 100644 index 0000000..d70f0ee --- /dev/null +++ b/hailort/libhailort/src/core_op/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/core_op.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/resource_manager.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/resource_manager_builder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/config_buffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/inter_context_buffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/ddr_channels_pair.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/channel_allocator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/resource_manager/context_switch_buffer_builder.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} ${HAILORT_OPS_CPP_SOURCES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/core_op/active_core_op_holder.hpp b/hailort/libhailort/src/core_op/active_core_op_holder.hpp new file mode 100644 index 0000000..c87e0b0 --- /dev/null +++ b/hailort/libhailort/src/core_op/active_core_op_holder.hpp @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file active_core_op_holder.hpp + * @brief place_holder stored in ConfigManager indicating which CoreOp is currently active + * + **/ + +#ifndef _HAILO_CONTEXT_SWITCH_ACTIVE_CORE_OP_HOLDER_HPP_ +#define _HAILO_CONTEXT_SWITCH_ACTIVE_CORE_OP_HOLDER_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "common/utils.hpp" + +#include "core_op/core_op.hpp" + + +namespace hailort +{ + +class CoreOp; + +class ActiveCoreOpHolder final +{ + public: + ActiveCoreOpHolder() : m_core_op(nullptr) {} + + ExpectedRef get() + { + CHECK_NOT_NULL_AS_EXPECTED(m_core_op, HAILO_INVALID_OPERATION); + return std::ref(*m_core_op); + } + void set(CoreOp &core_op) + { + assert(!is_any_active()); + m_core_op = &core_op; + } + + bool is_any_active() { return nullptr != m_core_op; } + + void clear() { m_core_op = nullptr; } + + ActiveCoreOpHolder(ActiveCoreOpHolder&) = delete; + ActiveCoreOpHolder& operator=(ActiveCoreOpHolder&) = delete; + ActiveCoreOpHolder& operator=(ActiveCoreOpHolder&&) = delete; + ActiveCoreOpHolder(ActiveCoreOpHolder&&) = default; + private: + CoreOp *m_core_op; +}; + +} /* namespace hailort */ + +#endif //_HAILO_CONTEXT_SWITCH_ACTIVE_CORE_OP_HOLDER_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/context_switch/network_group.cpp b/hailort/libhailort/src/core_op/core_op.cpp similarity index 50% rename from hailort/libhailort/src/context_switch/network_group.cpp rename to hailort/libhailort/src/core_op/core_op.cpp index 53f94c5..0fb260e 100644 --- a/hailort/libhailort/src/context_switch/network_group.cpp +++ b/hailort/libhailort/src/core_op/core_op.cpp @@ -1,36 +1,37 @@ /** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ +**/ /** - * @file network_group.cpp - * @brief: Configured Network Group and Activated Network Group + * @file core_op.cpp + * @brief Core-Op module implementation **/ +#include "hailo/network_group.hpp" #include "hailo/transform.hpp" -#include "hailo/vstream.hpp" -#include "network_group_internal.hpp" -#include "hef_internal.hpp" +#include "hailo/hailort_defaults.hpp" + #include "common/utils.hpp" -#include "hailort_defaults.hpp" -#include "eth_stream.hpp" -#include "pcie_stream.hpp" -#include "core_stream.hpp" -#include "mipi_stream.hpp" -#include "control.hpp" #include "common/runtime_statistics_internal.hpp" -#include "vstream_internal.hpp" -#include "context_switch/multi_context/resource_manager.hpp" + +#include "core_op/core_op.hpp" +#include "core_op/resource_manager/resource_manager.hpp" +#include "hef/hef_internal.hpp" +#include "eth/eth_stream.hpp" +#include "vdma/vdma_stream_base.hpp" +#include "mipi/mipi_stream.hpp" +#include "device_common/control_protocol.hpp" + namespace hailort { -ActivatedNetworkGroupBase::ActivatedNetworkGroupBase(const hailo_activate_network_group_params_t &network_group_params, +ActivatedCoreOp::ActivatedCoreOp(const hailo_activate_network_group_params_t &network_group_params, std::map> &input_streams, std::map> &output_streams, - EventPtr &&network_group_activated_event, hailo_status &status) : + EventPtr &&core_op_activated_event, hailo_status &status) : m_network_group_params(network_group_params), - m_network_group_activated_event(std::move(network_group_activated_event)), + m_core_op_activated_event(std::move(core_op_activated_event)), m_input_streams(input_streams), m_output_streams(output_streams) { @@ -41,7 +42,7 @@ ActivatedNetworkGroupBase::ActivatedNetworkGroupBase(const hailo_activate_networ } } -uint32_t ActivatedNetworkGroupBase::get_invalid_frames_count() +uint32_t ActivatedCoreOp::get_invalid_frames_count() { uint32_t total_invalid_frames_count = 0; for (auto& name_stream_pair : m_output_streams) { @@ -51,22 +52,50 @@ uint32_t ActivatedNetworkGroupBase::get_invalid_frames_count() } // TODO: Implement function (HRT-3174) -hailo_status ActivatedNetworkGroupBase::validate_network_group_params( +hailo_status ActivatedCoreOp::validate_network_group_params( const hailo_activate_network_group_params_t &/*network_group_params*/) { return HAILO_SUCCESS; } -Expected> ConfiguredNetworkGroup::activate() +CoreOp::CoreOp( + const ConfigureNetworkParams &config_params, std::shared_ptr metadata, hailo_status &status) : + m_config_params(config_params), + m_min_configured_batch_size(get_smallest_configured_batch_size(config_params)), + m_activation_time_accumulator(), + m_deactivation_time_accumulator(), + m_metadata(metadata) { - const auto network_group_params = HailoRTDefaults::get_network_group_params(); - return activate(network_group_params); + auto event = Event::create_shared(Event::State::not_signalled); + if (nullptr == event) { + LOGGER__ERROR("Failed to create activation event"); + status = HAILO_INTERNAL_FAILURE; + return; + } + m_core_op_activated_event = std::move(std::move(event)); + + m_activation_time_accumulator = make_shared_nothrow>("activation_time"); + if (nullptr == m_activation_time_accumulator) { + LOGGER__ERROR("Failed to create activation time accumulator"); + status = HAILO_OUT_OF_HOST_MEMORY; + return; + }; + + m_deactivation_time_accumulator = make_shared_nothrow>("deactivation_time"); + if (nullptr == m_deactivation_time_accumulator) { + LOGGER__ERROR("Failed to create deactivation time accumulator"); + status = HAILO_OUT_OF_HOST_MEMORY; + return; + }; + + status = HAILO_SUCCESS; } -Expected> ConfiguredNetworkGroupBase::activate( - const hailo_activate_network_group_params_t &network_group_params) +Expected> CoreOp::activate(const hailo_activate_network_group_params_t &network_group_params) { - return create_activated_network_group(network_group_params, CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE); + static const auto RESET_PENDING_STREAM_TRANSFERS = false; + return create_activated_network_group(network_group_params, CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE, + RESET_PENDING_STREAM_TRANSFERS); } Expected get_latency(LatencyMeterPtr &latency_meter, bool clear) @@ -80,7 +109,7 @@ Expected get_latency(LatencyMeterPtr &latency_meter, b } /* Network group base functions */ -Expected ConfiguredNetworkGroupBase::get_latency_measurement(const std::string &network_name) +Expected CoreOp::get_latency_measurement(const std::string &network_name) { bool clear = ((m_config_params.latency & HAILO_LATENCY_CLEAR_AFTER_GET) == HAILO_LATENCY_CLEAR_AFTER_GET); LatencyMeasurementResult result = {}; @@ -121,13 +150,13 @@ Expected ConfiguredNetworkGroupBase::get_latency_measu return result; } -Expected ConfiguredNetworkGroupBase::get_output_streams_from_vstream_names( +Expected CoreOp::get_output_streams_from_vstream_names( const std::map &outputs_params) { OutputStreamWithParamsVector results; std::unordered_map outputs_edges_params; for (auto &name_params_pair : outputs_params) { - auto stream_names = m_network_group_metadata.get_stream_names_from_vstream_name(name_params_pair.first); + auto stream_names = m_metadata->get_stream_names_from_vstream_name(name_params_pair.first); CHECK_EXPECTED(stream_names); for (auto &stream_name : stream_names.value()) { @@ -152,7 +181,7 @@ Expected ConfiguredNetworkGroupBase::get_output_st // This function adds to results the OutputStreams that correspond to the edges in outputs_edges_params. // If an edge name appears in outputs_edges_params then all of its predecessors must appear in outputs_edges_params as well, Otherwise, an error is returned. // We use the set seen_edges in order to mark the edges already evaluated by one of its' predecessor. -hailo_status ConfiguredNetworkGroupBase::add_mux_streams_by_edges_names(OutputStreamWithParamsVector &results, +hailo_status CoreOp::add_mux_streams_by_edges_names(OutputStreamWithParamsVector &results, const std::unordered_map &outputs_edges_params) { std::unordered_set seen_edges; @@ -185,9 +214,9 @@ hailo_status ConfiguredNetworkGroupBase::add_mux_streams_by_edges_names(OutputSt return HAILO_SUCCESS; } -Expected ConfiguredNetworkGroupBase::get_output_streams_by_vstream_name(const std::string &name) +Expected CoreOp::get_output_streams_by_vstream_name(const std::string &name) { - auto stream_names = m_network_group_metadata.get_stream_names_from_vstream_name(name); + auto stream_names = m_metadata->get_stream_names_from_vstream_name(name); CHECK_EXPECTED(stream_names); OutputStreamPtrVector output_streams; @@ -200,9 +229,9 @@ Expected ConfiguredNetworkGroupBase::get_output_streams_b return output_streams; } -Expected ConfiguredNetworkGroupBase::get_layer_info(const std::string &stream_name) +Expected CoreOp::get_layer_info(const std::string &stream_name) { - for (auto layer_info : m_network_group_metadata.get_all_layer_infos()) { + for (auto layer_info : m_metadata->get_all_layer_infos()) { if (layer_info.name == stream_name) { return layer_info; } @@ -211,43 +240,7 @@ Expected ConfiguredNetworkGroupBase::get_layer_info(const std::string return make_unexpected(HAILO_NOT_FOUND); } -ConfiguredNetworkGroupBase::ConfiguredNetworkGroupBase( - const ConfigureNetworkParams &config_params, const NetworkGroupMetadata &network_group_metadata, - std::vector> &&net_flow_ops, - hailo_status &status) : - m_config_params(config_params), - m_min_configured_batch_size(get_smallest_configured_batch_size(config_params)), - m_network_group_metadata(network_group_metadata), - m_activation_time_accumulator(), - m_deactivation_time_accumulator(), - m_net_flow_ops(std::move(net_flow_ops)) -{ - auto event = Event::create_shared(Event::State::not_signalled); - if (nullptr == event) { - LOGGER__ERROR("Failed to create activation event"); - status = HAILO_INTERNAL_FAILURE; - return; - } - m_network_group_activated_event = std::move(std::move(event)); - - m_activation_time_accumulator = make_shared_nothrow>("activation_time"); - if (nullptr == m_activation_time_accumulator) { - LOGGER__ERROR("Failed to create activation time accumulator"); - status = HAILO_OUT_OF_HOST_MEMORY; - return; - }; - - m_deactivation_time_accumulator = make_shared_nothrow>("deactivation_time"); - if (nullptr == m_deactivation_time_accumulator) { - LOGGER__ERROR("Failed to create deactivation time accumulator"); - status = HAILO_OUT_OF_HOST_MEMORY; - return; - }; - - status = HAILO_SUCCESS; -} - -uint16_t ConfiguredNetworkGroupBase::get_smallest_configured_batch_size(const ConfigureNetworkParams &config_params) +uint16_t CoreOp::get_smallest_configured_batch_size(const ConfigureNetworkParams &config_params) { // There are two possible situations: // 1) All networks in the network group have the same configured (and hence smallest) batch_size => @@ -270,36 +263,32 @@ uint16_t ConfiguredNetworkGroupBase::get_smallest_configured_batch_size(const Co return (UINT16_MAX == min_batch_size) ? DEFAULT_ACTUAL_BATCH_SIZE : min_batch_size; } -Expected> ConfiguredNetworkGroupBase::activate_with_batch(uint16_t dynamic_batch_size) -{ - return create_activated_network_group(HailoRTDefaults::get_network_group_params(), dynamic_batch_size); -} - -const std::string &ConfiguredNetworkGroupBase::get_network_group_name() const +Expected> CoreOp::activate_with_batch(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) { - return m_network_group_metadata.network_group_name(); + return create_activated_network_group(HailoRTDefaults::get_active_network_group_params(), dynamic_batch_size, + resume_pending_stream_transfers); } -const std::string &ConfiguredNetworkGroupBase::name() const +const std::string &CoreOp::name() const { - return m_network_group_metadata.network_group_name(); + return m_metadata->core_op_name(); } -hailo_status ConfiguredNetworkGroupBase::activate_low_level_streams(uint16_t dynamic_batch_size) +hailo_status CoreOp::activate_low_level_streams(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) { for (auto &name_pair : m_input_streams) { - auto status = name_pair.second->activate_stream(dynamic_batch_size); + auto status = name_pair.second->activate_stream(dynamic_batch_size, resume_pending_stream_transfers); CHECK_SUCCESS(status); } for (auto &name_pair : m_output_streams) { - auto status = name_pair.second->activate_stream(dynamic_batch_size); + auto status = name_pair.second->activate_stream(dynamic_batch_size, resume_pending_stream_transfers); CHECK_SUCCESS(status); } return HAILO_SUCCESS; } -hailo_status ConfiguredNetworkGroupBase::deactivate_low_level_streams() +hailo_status CoreOp::deactivate_low_level_streams() { // Best effort auto status = HAILO_SUCCESS; @@ -322,9 +311,19 @@ hailo_status ConfiguredNetworkGroupBase::deactivate_low_level_streams() return status; } -Expected ConfiguredNetworkGroupBase::get_stream_batch_size(const std::string &stream_name) +Expected> CoreOp::get_vstream_names_from_stream_name(const std::string &stream_name) +{ + return m_metadata->get_vstream_names_from_stream_name(stream_name); +} + +const SupportedFeatures &CoreOp::get_supported_features() { - for (const auto &layer_info : m_network_group_metadata.get_all_layer_infos()) { + return m_metadata->supported_features(); +} + +Expected CoreOp::get_stream_batch_size(const std::string &stream_name) +{ + for (const auto &layer_info : m_metadata->get_all_layer_infos()) { if (layer_info.name == stream_name) { for (auto const &network_params_pair : m_config_params.network_params_by_name) { if (network_params_pair.first == layer_info.network_name) { @@ -338,21 +337,21 @@ Expected ConfiguredNetworkGroupBase::get_stream_batch_size(const std:: return make_unexpected(HAILO_NOT_FOUND); } -bool ConfiguredNetworkGroupBase::is_multi_context() const +bool CoreOp::is_multi_context() const { - return m_network_group_metadata.supported_features().multi_context; + return m_metadata->supported_features().multi_context; } -const ConfigureNetworkParams ConfiguredNetworkGroupBase::get_config_params() const +const ConfigureNetworkParams CoreOp::get_config_params() const { return m_config_params; } -hailo_status ConfiguredNetworkGroupBase::create_input_stream_from_config_params(Device &device, +hailo_status CoreOp::create_input_stream_from_config_params(Device &device, const hailo_stream_parameters_t &stream_params, const std::string &stream_name) { - auto edge_layer = get_layer_info(stream_name); - CHECK_EXPECTED_AS_STATUS(edge_layer); + auto layer_info = get_layer_info(stream_name); + CHECK_EXPECTED_AS_STATUS(layer_info); CHECK(device.is_stream_interface_supported(stream_params.stream_interface), HAILO_INVALID_OPERATION, "Device does not supports the given stream interface streams. Please update input_stream_params for stream {}.", @@ -360,64 +359,60 @@ hailo_status ConfiguredNetworkGroupBase::create_input_stream_from_config_params( switch (stream_params.stream_interface) { case HAILO_STREAM_INTERFACE_PCIE: - { - auto batch_size_exp = get_stream_batch_size(stream_name); - CHECK_EXPECTED_AS_STATUS(batch_size_exp); - const auto stream_index = edge_layer->stream_index; - auto vdma_channel_ptr = get_boundary_vdma_channel_by_stream_name(stream_name); - CHECK_EXPECTED_AS_STATUS(vdma_channel_ptr, "Failed to get vdma channel for output stream {}", stream_index); - - auto input_stream = PcieInputStream::create(device, vdma_channel_ptr.release(), - edge_layer.value(), batch_size_exp.value(), m_network_group_activated_event); - CHECK_EXPECTED_AS_STATUS(input_stream); - m_input_streams.insert(make_pair(stream_name, input_stream.release())); - } - break; - case HAILO_STREAM_INTERFACE_CORE: - { - auto batch_size_exp = get_stream_batch_size(stream_name); - CHECK_EXPECTED_AS_STATUS(batch_size_exp); - const auto stream_index = edge_layer->stream_index; - auto vdma_channel_ptr = get_boundary_vdma_channel_by_stream_name(stream_name); - CHECK_EXPECTED_AS_STATUS(vdma_channel_ptr, "Failed to get vdma channel for output stream {}", stream_index); - - auto input_stream = CoreInputStream::create(device, vdma_channel_ptr.release(), - edge_layer.value(), batch_size_exp.value(), m_network_group_activated_event); - CHECK_EXPECTED_AS_STATUS(input_stream); - m_input_streams.insert(make_pair(stream_name, input_stream.release())); - } - break; + // Fallthrough + case HAILO_STREAM_INTERFACE_INTEGRATED: + return create_vdma_input_stream(device, stream_name, layer_info.value(), stream_params); + case HAILO_STREAM_INTERFACE_ETH: { auto input_stream = EthernetInputStream::create(device, - edge_layer.value(), stream_params.eth_input_params, m_network_group_activated_event); + layer_info.value(), stream_params.eth_input_params, m_core_op_activated_event); CHECK_EXPECTED_AS_STATUS(input_stream); m_input_streams.insert(make_pair(stream_name, input_stream.release())); + return HAILO_SUCCESS; } - break; + case HAILO_STREAM_INTERFACE_MIPI: { auto input_stream = MipiInputStream::create(device, - edge_layer.value(), stream_params.mipi_input_params, m_network_group_activated_event); + layer_info.value(), stream_params.mipi_input_params, m_core_op_activated_event); CHECK_EXPECTED_AS_STATUS(input_stream); m_input_streams.insert(make_pair(stream_name, input_stream.release())); + return HAILO_SUCCESS; } - break; + default: - { - LOGGER__ERROR("{} interface is not supported.", stream_params.stream_interface); - return HAILO_NOT_IMPLEMENTED; - } + LOGGER__ERROR("{} interface is not supported.", stream_params.stream_interface); + return HAILO_NOT_IMPLEMENTED; } +} + +hailo_status CoreOp::create_vdma_input_stream(Device &device, const std::string &stream_name, + const LayerInfo &layer_info, const hailo_stream_parameters_t &stream_params) +{ + // Make sure the downcast is safe + CHECK((Device::Type::INTEGRATED == device.get_type()) || (Device::Type::PCIE == device.get_type()), + HAILO_INTERNAL_FAILURE, "Invalid device type"); + VdmaDevice *vdma_device = reinterpret_cast(&device); + + auto batch_size_exp = get_stream_batch_size(stream_name); + CHECK_EXPECTED_AS_STATUS(batch_size_exp); + auto vdma_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(stream_name); + CHECK_EXPECTED_AS_STATUS(vdma_channel_ptr_exp, "Failed to get vdma channel for output stream {}", stream_name); + + auto input_stream = VdmaInputStreamBase::create(stream_params.stream_interface, *vdma_device, vdma_channel_ptr_exp.value(), + layer_info, stream_params, batch_size_exp.value(), m_core_op_activated_event); + CHECK_EXPECTED_AS_STATUS(input_stream); + m_input_streams.insert(make_pair(stream_name, input_stream.release())); return HAILO_SUCCESS; } -hailo_status ConfiguredNetworkGroupBase::create_output_stream_from_config_params(Device &device, +hailo_status CoreOp::create_output_stream_from_config_params(Device &device, const hailo_stream_parameters_t &stream_params, const std::string &stream_name) { - auto edge_layer = get_layer_info(stream_name); - CHECK_EXPECTED_AS_STATUS(edge_layer); + auto layer_info = get_layer_info(stream_name); + CHECK_EXPECTED_AS_STATUS(layer_info); CHECK(device.is_stream_interface_supported(stream_params.stream_interface), HAILO_INVALID_OPERATION, "Device does not supports the given stream interface streams. Please update input_stream_params for stream {}.", @@ -425,53 +420,48 @@ hailo_status ConfiguredNetworkGroupBase::create_output_stream_from_config_params switch (stream_params.stream_interface) { case HAILO_STREAM_INTERFACE_PCIE: - { - auto batch_size_exp = get_stream_batch_size(stream_name); - CHECK_EXPECTED_AS_STATUS(batch_size_exp); - const auto stream_index = edge_layer->stream_index; - auto vdma_channel_ptr = get_boundary_vdma_channel_by_stream_name(stream_name); - CHECK_EXPECTED_AS_STATUS(vdma_channel_ptr, "Failed to get vdma channel for output stream {}", stream_index); - - auto output_stream = PcieOutputStream::create(device, vdma_channel_ptr.release(), - edge_layer.value(), batch_size_exp.value(), m_network_group_activated_event); - CHECK_EXPECTED_AS_STATUS(output_stream); - m_output_streams.insert(make_pair(stream_name, output_stream.release())); - } - break; - case HAILO_STREAM_INTERFACE_CORE: - { - auto batch_size_exp = get_stream_batch_size(stream_name); - CHECK_EXPECTED_AS_STATUS(batch_size_exp); - const auto stream_index = edge_layer->stream_index; - auto vdma_channel_ptr = get_boundary_vdma_channel_by_stream_name(stream_name); - CHECK_EXPECTED_AS_STATUS(vdma_channel_ptr, "Failed to get vdma channel for output stream {}", stream_index); - - auto output_stream = CoreOutputStream::create(device, vdma_channel_ptr.release(), - edge_layer.value(), batch_size_exp.value(), m_network_group_activated_event); - CHECK_EXPECTED_AS_STATUS(output_stream); - m_output_streams.insert(make_pair(stream_name, output_stream.release())); - } - break; + // Fallthrough + case HAILO_STREAM_INTERFACE_INTEGRATED: + return create_vdma_output_stream(device, stream_name, layer_info.value(), stream_params); + case HAILO_STREAM_INTERFACE_ETH: { auto output_stream = EthernetOutputStream::create(device, - edge_layer.value(), stream_params.eth_output_params, - m_network_group_activated_event); + layer_info.value(), stream_params.eth_output_params, + m_core_op_activated_event); CHECK_EXPECTED_AS_STATUS(output_stream); m_output_streams.insert(make_pair(stream_name, output_stream.release())); + return HAILO_SUCCESS; } - break; + default: - { - LOGGER__ERROR("{} interface is not supported.", stream_params.stream_interface); - return HAILO_NOT_IMPLEMENTED; - } + LOGGER__ERROR("{} interface is not supported.", stream_params.stream_interface); + return HAILO_NOT_IMPLEMENTED; } +} + +hailo_status CoreOp::create_vdma_output_stream(Device &device, const std::string &stream_name, + const LayerInfo &layer_info, const hailo_stream_parameters_t &stream_params) +{ + // Make sure the downcast is safe + CHECK((Device::Type::INTEGRATED == device.get_type()) || (Device::Type::PCIE == device.get_type()), + HAILO_INTERNAL_FAILURE, "Invalid device type"); + VdmaDevice *vdma_device = reinterpret_cast(&device); + + auto batch_size_exp = get_stream_batch_size(stream_name); + CHECK_EXPECTED_AS_STATUS(batch_size_exp); + auto vdma_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(stream_name); + CHECK_EXPECTED_AS_STATUS(vdma_channel_ptr_exp, "Failed to get vdma channel for output stream {}", stream_name); + + auto output_stream = VdmaOutputStreamBase::create(stream_params.stream_interface, *vdma_device, vdma_channel_ptr_exp.value(), + layer_info, batch_size_exp.value(), stream_params, m_core_op_activated_event); + CHECK_EXPECTED_AS_STATUS(output_stream); + m_output_streams.insert(make_pair(stream_name, output_stream.release())); return HAILO_SUCCESS; } -hailo_status ConfiguredNetworkGroupBase::create_streams_from_config_params(Device &device) +hailo_status CoreOp::create_streams_from_config_params(Device &device) { for (const auto &stream_parameters_pair : m_config_params.stream_params_by_name) { switch (stream_parameters_pair.second.direction) { @@ -486,8 +476,8 @@ hailo_status ConfiguredNetworkGroupBase::create_streams_from_config_params(Devic case HAILO_D2H_STREAM: { auto status = create_output_stream_from_config_params(device, - stream_parameters_pair.second, - stream_parameters_pair.first); + stream_parameters_pair.second, + stream_parameters_pair.first); CHECK_SUCCESS(status); } break; @@ -500,9 +490,9 @@ hailo_status ConfiguredNetworkGroupBase::create_streams_from_config_params(Devic return HAILO_SUCCESS; } -Expected ConfiguredNetworkGroupBase::get_input_streams_by_network(const std::string &network_name) +Expected CoreOp::get_input_streams_by_network(const std::string &network_name) { - auto input_stream_infos = m_network_group_metadata.get_input_stream_infos(network_name); + auto input_stream_infos = m_metadata->get_input_stream_infos(network_name); CHECK_EXPECTED(input_stream_infos); InputStreamRefVector result; @@ -514,9 +504,9 @@ Expected ConfiguredNetworkGroupBase::get_input_streams_by_ return result; } -Expected ConfiguredNetworkGroupBase::get_output_streams_by_network(const std::string &network_name) +Expected CoreOp::get_output_streams_by_network(const std::string &network_name) { - auto output_stream_infos = m_network_group_metadata.get_output_stream_infos(network_name); + auto output_stream_infos = m_metadata->get_output_stream_infos(network_name); CHECK_EXPECTED(output_stream_infos); OutputStreamRefVector result; @@ -528,7 +518,7 @@ Expected ConfiguredNetworkGroupBase::get_output_streams_b return result; } -InputStreamRefVector ConfiguredNetworkGroupBase::get_input_streams() +InputStreamRefVector CoreOp::get_input_streams() { InputStreamRefVector result; for (auto& name_stream_pair : m_input_streams) { @@ -537,7 +527,7 @@ InputStreamRefVector ConfiguredNetworkGroupBase::get_input_streams() return result; } -OutputStreamRefVector ConfiguredNetworkGroupBase::get_output_streams() +OutputStreamRefVector CoreOp::get_output_streams() { OutputStreamRefVector result; for (auto& name_stream_pair : m_output_streams) { @@ -546,7 +536,7 @@ OutputStreamRefVector ConfiguredNetworkGroupBase::get_output_streams() return result; } -ExpectedRef ConfiguredNetworkGroupBase::get_input_stream_by_name(const std::string& name) +ExpectedRef CoreOp::get_input_stream_by_name(const std::string& name) { auto iterator = m_input_streams.find(name); if (m_input_streams.end() == iterator) { @@ -557,7 +547,7 @@ ExpectedRef ConfiguredNetworkGroupBase::get_input_stream_by_name(co return std::ref(*iterator->second); } -ExpectedRef ConfiguredNetworkGroupBase::get_output_stream_by_name(const std::string& name) +ExpectedRef CoreOp::get_output_stream_by_name(const std::string& name) { auto iterator = m_output_streams.find(name); if (m_output_streams.end() == iterator) { @@ -568,7 +558,7 @@ ExpectedRef ConfiguredNetworkGroupBase::get_output_stream_by_name( return std::ref(*iterator->second); } -std::vector> ConfiguredNetworkGroupBase::get_input_streams_by_interface( +std::vector> CoreOp::get_input_streams_by_interface( hailo_stream_interface_t stream_interface) { std::vector> results; @@ -580,7 +570,7 @@ std::vector> ConfiguredNetworkGroupBase::get return results; } -std::vector> ConfiguredNetworkGroupBase::get_output_streams_by_interface( +std::vector> CoreOp::get_output_streams_by_interface( hailo_stream_interface_t stream_interface) { std::vector> results; @@ -592,12 +582,12 @@ std::vector> ConfiguredNetworkGroupBase::ge return results; } -hailo_status ConfiguredNetworkGroupBase::wait_for_activation(const std::chrono::milliseconds &timeout) +hailo_status CoreOp::wait_for_activation(const std::chrono::milliseconds &timeout) { - return m_network_group_activated_event->wait(timeout); + return m_core_op_activated_event->wait(timeout); } -Expected>> ConfiguredNetworkGroupBase::get_output_vstream_groups() +Expected>> CoreOp::get_output_vstream_groups() { std::vector> results; @@ -610,7 +600,7 @@ Expected>> ConfiguredNetworkGroupBase::get_ return results; } -Expected>> ConfiguredNetworkGroupBase::make_output_vstream_params_groups( +Expected>> CoreOp::make_output_vstream_params_groups( bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) { auto params = make_output_vstream_params(quantized, format_type, timeout_ms, queue_size); @@ -634,11 +624,11 @@ Expected>> ConfiguredN return results; } -Expected> ConfiguredNetworkGroupBase::make_input_vstream_params( +Expected> CoreOp::make_input_vstream_params( bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size, const std::string &network_name) { - auto input_vstream_infos = m_network_group_metadata.get_input_vstream_infos(network_name); + auto input_vstream_infos = m_metadata->get_input_vstream_infos(network_name); CHECK_EXPECTED(input_vstream_infos); std::map res; @@ -648,11 +638,11 @@ Expected> ConfiguredNetworkGroupBa return res; } -Expected> ConfiguredNetworkGroupBase::make_output_vstream_params( +Expected> CoreOp::make_output_vstream_params( bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size, const std::string &network_name) { - auto output_vstream_infos = m_network_group_metadata.get_output_vstream_infos(network_name); + auto output_vstream_infos = m_metadata->get_output_vstream_infos(network_name); CHECK_EXPECTED(output_vstream_infos); std::map res; auto status = Hef::Impl::fill_missing_vstream_params_with_default(res, output_vstream_infos.value(), quantized, @@ -661,154 +651,57 @@ Expected> ConfiguredNetworkGroupBa return res; } -Expected> ConfiguredNetworkGroupBase::get_network_infos() const +Expected> CoreOp::get_network_infos() const { - return m_network_group_metadata.get_network_infos(); + return m_metadata->get_network_infos(); } -Expected> ConfiguredNetworkGroupBase::get_all_stream_infos( +Expected> CoreOp::get_all_stream_infos( const std::string &network_name) const { - return m_network_group_metadata.get_all_stream_infos(network_name); + return m_metadata->get_all_stream_infos(network_name); } -Expected> ConfiguredNetworkGroupBase::get_input_vstream_infos( +Expected> CoreOp::get_input_vstream_infos( const std::string &network_name) const { - return m_network_group_metadata.get_input_vstream_infos(network_name); + return m_metadata->get_input_vstream_infos(network_name); } -Expected> ConfiguredNetworkGroupBase::get_output_vstream_infos( +Expected> CoreOp::get_output_vstream_infos( const std::string &network_name) const { - return m_network_group_metadata.get_output_vstream_infos(network_name); + return m_metadata->get_output_vstream_infos(network_name); } -Expected> ConfiguredNetworkGroupBase::get_all_vstream_infos( +Expected> CoreOp::get_all_vstream_infos( const std::string &network_name) const { - return m_network_group_metadata.get_all_vstream_infos(network_name); + return m_metadata->get_all_vstream_infos(network_name); } -AccumulatorPtr ConfiguredNetworkGroupBase::get_activation_time_accumulator() const +AccumulatorPtr CoreOp::get_activation_time_accumulator() const { return m_activation_time_accumulator; } -AccumulatorPtr ConfiguredNetworkGroupBase::get_deactivation_time_accumulator() const +AccumulatorPtr CoreOp::get_deactivation_time_accumulator() const { return m_deactivation_time_accumulator; } -static hailo_vstream_params_t expand_vstream_params_autos(const hailo_stream_info_t &stream_info, - const hailo_vstream_params_t &vstream_params) -{ - auto local_vstream_params = vstream_params; - local_vstream_params.user_buffer_format = HailoRTDefaults::expand_auto_format(vstream_params.user_buffer_format, - stream_info.format); - return local_vstream_params; -} - -static std::map vstream_infos_vector_to_map(std::vector &&vstream_info_vector) -{ - std::map vstream_infos_map; - for (const auto &vstream_info : vstream_info_vector) { - vstream_infos_map.emplace(std::string(vstream_info.name), vstream_info); - } - - return vstream_infos_map; -} - -Expected> ConfiguredNetworkGroupBase::create_input_vstreams(const std::map &inputs_params) +Expected> CoreOp::get_shared_input_stream_by_name(const std::string &stream_name) { - auto input_vstream_infos = get_input_vstream_infos(); - CHECK_EXPECTED(input_vstream_infos); - auto input_vstream_infos_map = vstream_infos_vector_to_map(input_vstream_infos.release()); - - std::vector vstreams; - vstreams.reserve(inputs_params.size()); - for (const auto &name_params_pair : inputs_params) { - CHECK_AS_EXPECTED(contains(m_input_streams, name_params_pair.first), HAILO_NOT_FOUND); - auto input_stream = m_input_streams.at(name_params_pair.first); - - const auto vstream_info = input_vstream_infos_map.find(name_params_pair.first); - CHECK_AS_EXPECTED(vstream_info != input_vstream_infos_map.end(), HAILO_NOT_FOUND, - "Failed to find vstream info of {}", name_params_pair.first); - - const auto vstream_params = expand_vstream_params_autos(input_stream->get_info(), name_params_pair.second); - auto inputs = VStreamsBuilderUtils::create_inputs(input_stream, vstream_info->second, vstream_params); - CHECK_EXPECTED(inputs); - - vstreams.insert(vstreams.end(), std::make_move_iterator(inputs->begin()), std::make_move_iterator(inputs->end())); - } - return vstreams; + CHECK_AS_EXPECTED(contains(m_input_streams, stream_name), HAILO_NOT_FOUND, "Input stream {} not found."); + auto stream_ptr = m_input_streams.at(stream_name); + return stream_ptr; } -Expected> ConfiguredNetworkGroupBase::create_output_vstreams(const std::map &outputs_params) +Expected> CoreOp::get_shared_output_stream_by_name(const std::string &stream_name) { - std::vector vstreams; - vstreams.reserve(outputs_params.size()); - auto output_streams = get_output_streams_from_vstream_names(outputs_params); - CHECK_EXPECTED(output_streams); - - auto output_vstream_infos = get_output_vstream_infos(); - CHECK_EXPECTED(output_vstream_infos); - auto output_vstream_infos_map = vstream_infos_vector_to_map(output_vstream_infos.release()); - - // We iterate through all output streams, and if they are nms, we collect them together by their original stream name. - // We need this step because all nms output streams of the same original stream need to be fused together - - std::unordered_map> post_process_nms_ops; - std::set post_process_stream_inputs; - for (auto &op : m_net_flow_ops) { - CHECK_AS_EXPECTED(op->type == NetFlowElement::Type::YoloNmsOp, HAILO_INVALID_ARGUMENT, - "Unexpected operation: {}", op->name); - post_process_nms_ops.insert({op->name, op}); - post_process_stream_inputs.insert(op->input_streams.begin(), op->input_streams.end()); - } - std::map> nms_op_output_streams; - std::map> nms_output_streams; - for (auto &stream_params_pair : output_streams.value()) { - if ((HAILO_FORMAT_ORDER_HAILO_NMS == stream_params_pair.first->get_info().format.order && stream_params_pair.first->get_info().nms_info.is_defused) && - (outputs_params.end() != outputs_params.find(stream_params_pair.first->get_info().nms_info.defuse_info.original_name))) { - auto original_name = stream_params_pair.first->get_info().nms_info.defuse_info.original_name; - nms_output_streams.emplace(original_name, std::pair( - OutputStreamPtrVector(), outputs_params.at(original_name))); - nms_output_streams[original_name].first.push_back(stream_params_pair.first); - } else if (post_process_stream_inputs.count(stream_params_pair.first->get_info().name)) { - for (auto &op : m_net_flow_ops) { - if (op->input_streams.count(stream_params_pair.first->get_info().name)) { - CHECK_AS_EXPECTED(op->type == NetFlowElement::Type::YoloNmsOp, - HAILO_INVALID_ARGUMENT, "Expected post-process YOLO-NMS operation"); - assert(op->output_pads.size() == 1); - nms_op_output_streams.emplace(op->name, std::pair( - OutputStreamPtrVector(), outputs_params.at(op->output_pads[0].name))); - nms_op_output_streams[op->name].first.push_back(stream_params_pair.first); - } - } - } else { - auto outputs = VStreamsBuilderUtils::create_outputs(stream_params_pair.first, stream_params_pair.second, output_vstream_infos_map); - CHECK_EXPECTED(outputs); - vstreams.insert(vstreams.end(), std::make_move_iterator(outputs->begin()), std::make_move_iterator(outputs->end())); - } - } - for (auto &nms_output_stream_pair : nms_output_streams) { - auto outputs = VStreamsBuilderUtils::create_output_nms(nms_output_stream_pair.second.first, nms_output_stream_pair.second.second, - output_vstream_infos_map); - CHECK_EXPECTED(outputs); - vstreams.insert(vstreams.end(), std::make_move_iterator(outputs->begin()), std::make_move_iterator(outputs->end())); - } - for (auto &nms_output_stream_pair : nms_op_output_streams) { - auto op = post_process_nms_ops.at(nms_output_stream_pair.first); - assert(op->type == NetFlowElement::Type::YoloNmsOp); - auto nms_op = std::static_pointer_cast(op); - auto outputs = VStreamsBuilderUtils::create_output_post_process_nms(nms_output_stream_pair.second.first, - nms_output_stream_pair.second.second, output_vstream_infos_map, - *nms_op); - CHECK_EXPECTED(outputs); - vstreams.insert(vstreams.end(), std::make_move_iterator(outputs->begin()), std::make_move_iterator(outputs->end())); - } - return vstreams; + CHECK_AS_EXPECTED(contains(m_output_streams, stream_name), HAILO_NOT_FOUND, "Output stream {} not found."); + auto stream_ptr = m_output_streams.at(stream_name); + return stream_ptr; } } /* namespace hailort */ diff --git a/hailort/libhailort/src/core_op/core_op.hpp b/hailort/libhailort/src/core_op/core_op.hpp new file mode 100644 index 0000000..f5ba474 --- /dev/null +++ b/hailort/libhailort/src/core_op/core_op.hpp @@ -0,0 +1,211 @@ +/** + * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file core_op.hpp + * @brief Hence, the hierarchy is as follows: + * -------------------------------------------------------------------------------------------------------------- + * | CoreOp | (Base classes) + * | ________________________________|________________________________ | + * | / | \ | + * | VdmaConfigCoreOp VDeviceCoreOp HcpConfigCoreOp | (Actual implementations) + * | | | + * | | | + * | vector of VdmaConfigCoreOp | + * -------------------------------------------------------------------------------------------------------------| + * | ActivatedCoreOp | (Base classes) + * | __________________________________|_____________________________________ | + * | / | \ | + * | VdmaConfigActivatedCoreOp VDeviceActivatedCoreOp HcpConfigActivatedCoreOp | (Actual implementations) + * | | | + * | | | + * | vector of VdmaConfigActivatedCoreOp | + * -------------------------------------------------------------------------------------------------------------- + **/ + +#ifndef _HAILO_CORE_OP_HPP_ +#define _HAILO_CORE_OP_HPP_ + +#include "hailo/network_group.hpp" + +#include "common/latency_meter.hpp" + +#include "hef/hef_internal.hpp" +#include "hef/core_op_metadata.hpp" +#include "control_protocol.h" +#include "vdma/channel/boundary_channel.hpp" +#include "core_op/active_core_op_holder.hpp" + + +namespace hailort +{ +/** Represents a vector of InputStream ptrs */ +using InputStreamPtrVector = std::vector>; + +/** Represents a vector of OutputStream ptrs */ +using OutputStreamPtrVector = std::vector>; + +// ActivatedCoreOp is created with `hailo_activate_network_group_params_t` for legacy reasons. +// Currently hailo_activate_network_group_params_t is an empty struct holder, +// when adding parameters to it, consider `hailo_activate_network_group_params_t` should hold one core op in this case. +class ActivatedCoreOp : public ActivatedNetworkGroup +{ +public: + virtual ~ActivatedCoreOp() = default; + ActivatedCoreOp(const ActivatedCoreOp &other) = delete; + ActivatedCoreOp &operator=(const ActivatedCoreOp &other) = delete; + ActivatedCoreOp &operator=(ActivatedCoreOp &&other) = delete; + ActivatedCoreOp(ActivatedCoreOp &&other) = default; + + virtual uint32_t get_invalid_frames_count() override; + +protected: + hailo_activate_network_group_params_t m_network_group_params; + + ActivatedCoreOp(const hailo_activate_network_group_params_t &network_group_params, + std::map> &input_streams, + std::map> &output_streams, + EventPtr &&core_op_activated_event, hailo_status &status); + + EventPtr m_core_op_activated_event; + std::map> &m_input_streams; + std::map> &m_output_streams; + +private: + hailo_status validate_network_group_params(const hailo_activate_network_group_params_t &network_group_params); +}; + + +class CoreOp +{ +public: + virtual ~CoreOp() = default; + CoreOp(const CoreOp &other) = delete; + CoreOp &operator=(const CoreOp &other) = delete; + CoreOp &operator=(CoreOp &&other) = delete; + CoreOp(CoreOp &&other) = default; + + std::shared_ptr metadata() { + return m_metadata; + } + + Expected> activate_with_batch( + uint16_t dynamic_batch_size = CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE, + bool resume_pending_stream_transfers = false); + virtual Expected> activate(const hailo_activate_network_group_params_t &network_group_params); + virtual hailo_status wait_for_activation(const std::chrono::milliseconds &timeout); + + virtual const std::string &name() const; + + virtual bool is_scheduled() const = 0; + virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) = 0; + virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) = 0; + virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name) = 0; + virtual Expected get_default_streams_interface() = 0; + + virtual Expected get_input_streams_by_network(const std::string &network_name=""); + virtual Expected get_output_streams_by_network(const std::string &network_name=""); + virtual InputStreamRefVector get_input_streams(); + virtual OutputStreamRefVector get_output_streams(); + virtual std::vector> get_input_streams_by_interface(hailo_stream_interface_t stream_interface); + virtual std::vector> get_output_streams_by_interface(hailo_stream_interface_t stream_interface); + virtual ExpectedRef get_input_stream_by_name(const std::string& name); + virtual ExpectedRef get_output_stream_by_name(const std::string& name); + virtual Expected get_output_streams_from_vstream_names( + const std::map &outputs_params); + virtual Expected get_latency_measurement(const std::string &network_name=""); + + // TODO: HRT-9546 - Remove func, should be only in CNG + virtual Expected> make_input_vstream_params( + bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size, + const std::string &network_name=""); + // TODO: HRT-9546 - Remove func, should be only in CNG + virtual Expected> make_output_vstream_params( + bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size, + const std::string &network_name=""); + // TODO: HRT-9546 - Remove func, should be only in CNG + virtual Expected>> make_output_vstream_params_groups( + bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size); + + // TODO: HRT-9546 - Remove func, should be only in CNG + virtual Expected>> get_output_vstream_groups(); + + // TODO: HRT-9546 - Remove func, should be only in CNG + Expected> get_vstream_names_from_stream_name(const std::string &stream_name); + + virtual hailo_status activate_impl(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers = false) = 0; + virtual hailo_status deactivate_impl(bool keep_nn_config_during_reset = false) = 0; + + virtual Expected> get_network_infos() const; + virtual Expected> get_all_stream_infos(const std::string &network_name="") const; + virtual Expected> get_input_vstream_infos(const std::string &network_name="") const; + virtual Expected> get_output_vstream_infos(const std::string &network_name="") const; + virtual Expected> get_all_vstream_infos(const std::string &network_name="") const; + virtual AccumulatorPtr get_activation_time_accumulator() const; + virtual AccumulatorPtr get_deactivation_time_accumulator() const; + hailo_status create_streams_from_config_params(Device &device); + + virtual bool is_multi_context() const; + virtual const ConfigureNetworkParams get_config_params() const; + + + const SupportedFeatures &get_supported_features(); + Expected get_stream_batch_size(const std::string &stream_name); + + std::map> m_input_streams; + std::map> m_output_streams; + + // This function is called when a user is creating VStreams and is only relevant for VDeviceCoreOp. + // In case a user is using VdmaConfigCoreOp or HcpConfigCoreOp this function should do nothing. + virtual void set_vstreams_multiplexer_callbacks(std::vector &output_vstreams) + { + (void)output_vstreams; + } + +protected: + CoreOp(const ConfigureNetworkParams &config_params, std::shared_ptr metadata, hailo_status &status); + + virtual Expected> create_activated_network_group( + const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) = 0; + + hailo_status create_output_stream_from_config_params(Device &device, + const hailo_stream_parameters_t &stream_params, const std::string &stream_name); + hailo_status create_input_stream_from_config_params(Device &device, + const hailo_stream_parameters_t &stream_params, const std::string &stream_name); + hailo_status add_mux_streams_by_edges_names(OutputStreamWithParamsVector &result, + const std::unordered_map &outputs_edges_params); + Expected get_output_streams_by_vstream_name(const std::string &name); + + hailo_status activate_low_level_streams(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers); + hailo_status deactivate_low_level_streams(); + + Expected get_layer_info(const std::string &stream_name); + + virtual Expected> get_latency_meters() = 0; + virtual Expected get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) = 0; + + const ConfigureNetworkParams m_config_params; + const uint16_t m_min_configured_batch_size; // TODO: remove after HRT-6535 + EventPtr m_core_op_activated_event; + AccumulatorPtr m_activation_time_accumulator; + AccumulatorPtr m_deactivation_time_accumulator; + std::shared_ptr m_metadata; + +private: + static uint16_t get_smallest_configured_batch_size(const ConfigureNetworkParams &config_params); + hailo_status create_vdma_input_stream(Device &device, const std::string &stream_name, + const LayerInfo &layer_info, const hailo_stream_parameters_t &stream_params); + hailo_status create_vdma_output_stream(Device &device, const std::string &stream_name, + const LayerInfo &layer_info, const hailo_stream_parameters_t &stream_params); + Expected> get_shared_input_stream_by_name(const std::string &stream_name); + Expected> get_shared_output_stream_by_name(const std::string &stream_name); + + friend class VDeviceCoreOp; // VDeviceCoreOp is using protected members and functions from other CoreOps objects + friend class VDeviceActivatedCoreOp; // VDeviceActivatedCoreOp is calling CoreOp's protected function `create_activated_network_group` + friend class ConfiguredNetworkGroupBase; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_CORE_OP_HPP_ */ diff --git a/hailort/libhailort/src/channel_allocator.cpp b/hailort/libhailort/src/core_op/resource_manager/channel_allocator.cpp similarity index 88% rename from hailort/libhailort/src/channel_allocator.cpp rename to hailort/libhailort/src/core_op/resource_manager/channel_allocator.cpp index 28833a8..d3cb898 100644 --- a/hailort/libhailort/src/channel_allocator.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/channel_allocator.cpp @@ -7,7 +7,7 @@ * @brief Allocates vdma channel indexes, allows reusing non-boundary channels between contextes. **/ -#include "channel_allocator.hpp" +#include "core_op/resource_manager/channel_allocator.hpp" namespace hailort @@ -18,7 +18,7 @@ ChannelAllocator::ChannelAllocator(size_t max_engines_count) : {} Expected ChannelAllocator::get_available_channel_id(const LayerIdentifier &layer_identifier, - VdmaChannel::Direction direction, uint8_t engine_index) + HailoRTDriver::DmaDirection direction, uint8_t engine_index) { CHECK_AS_EXPECTED(engine_index < m_max_engines_count, HAILO_INVALID_ARGUMENT, "Invalid engine index {}, max is {}", engine_index, m_max_engines_count); @@ -37,9 +37,9 @@ Expected ChannelAllocator::get_available_channel_id(const Layer } uint8_t min_channel_index = - (direction == VdmaChannel::Direction::H2D) ? MIN_H2D_CHANNEL_INDEX : MIN_D2H_CHANNEL_INDEX; + (direction == HailoRTDriver::DmaDirection::H2D) ? MIN_H2D_CHANNEL_INDEX : MIN_D2H_CHANNEL_INDEX; uint8_t max_channel_index = - (direction == VdmaChannel::Direction::H2D) ? MAX_H2D_CHANNEL_INDEX : MAX_D2H_CHANNEL_INDEX; + (direction == HailoRTDriver::DmaDirection::H2D) ? MAX_H2D_CHANNEL_INDEX : MAX_D2H_CHANNEL_INDEX; for (uint8_t index = min_channel_index; index <= max_channel_index; ++index) { const vdma::ChannelId channel_id = {engine_index, index}; @@ -77,11 +77,6 @@ hailo_status ChannelAllocator::free_channel_index(const LayerIdentifier &layer_i return HAILO_SUCCESS; } -const std::set &ChannelAllocator::get_internal_channel_ids() const -{ - return m_internal_channel_ids; -} - void ChannelAllocator::insert_new_channel_id(const LayerIdentifier &layer_identifier, const vdma::ChannelId &channel_id) { if (LayerType::BOUNDARY == std::get<0>(layer_identifier)) { diff --git a/hailort/libhailort/src/channel_allocator.hpp b/hailort/libhailort/src/core_op/resource_manager/channel_allocator.hpp similarity index 84% rename from hailort/libhailort/src/channel_allocator.hpp rename to hailort/libhailort/src/core_op/resource_manager/channel_allocator.hpp index 165bd84..2f80a48 100644 --- a/hailort/libhailort/src/channel_allocator.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/channel_allocator.hpp @@ -11,12 +11,15 @@ #define _HAILO_CHANNEL_ALLOCATOR_HPP_ #include "hailo/hailort.h" -#include "vdma_descriptor_list.hpp" -#include "vdma/channel_id.hpp" -#include "vdma_channel.hpp" -#include "layer_info.hpp" + +#include "vdma/memory/descriptor_list.hpp" +#include "vdma/channel/channel_id.hpp" +#include "vdma/channel/boundary_channel.hpp" +#include "hef/layer_info.hpp" + #include + namespace hailort { @@ -27,11 +30,9 @@ public: ChannelAllocator(ChannelAllocator &&other) = default; Expected get_available_channel_id(const LayerIdentifier &layer_identifier, - VdmaChannel::Direction direction, uint8_t engine_index); + HailoRTDriver::DmaDirection direction, uint8_t engine_index); hailo_status free_channel_index(const LayerIdentifier &layer_identifier); - const std::set &get_internal_channel_ids() const; - private: void insert_new_channel_id(const LayerIdentifier &layer_identifier, const vdma::ChannelId &channel_id); diff --git a/hailort/libhailort/src/config_buffer.cpp b/hailort/libhailort/src/core_op/resource_manager/config_buffer.cpp similarity index 76% rename from hailort/libhailort/src/config_buffer.cpp rename to hailort/libhailort/src/core_op/resource_manager/config_buffer.cpp index f64b24a..d15f487 100644 --- a/hailort/libhailort/src/config_buffer.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/config_buffer.cpp @@ -8,12 +8,13 @@ * hw format (ccw). */ -#include "config_buffer.hpp" -#include "vdma/sg_buffer.hpp" -#include "vdma/continuous_buffer.hpp" +#include "core_op/resource_manager/config_buffer.hpp" +#include "vdma/memory/sg_buffer.hpp" +#include "vdma/memory/continuous_buffer.hpp" #include + namespace hailort { Expected ConfigBuffer::create(HailoRTDriver &driver, vdma::ChannelId channel_id, @@ -23,7 +24,7 @@ Expected ConfigBuffer::create(HailoRTDriver &driver, vdma::Channel auto buffer_ptr = should_use_ccb(driver) ? create_ccb_buffer(driver, buffer_size) : - create_sg_buffer(driver, channel_id.channel_index, cfg_sizes); + create_sg_buffer(driver, channel_id, cfg_sizes); CHECK_EXPECTED(buffer_ptr); return ConfigBuffer(buffer_ptr.release(), channel_id, buffer_size); @@ -39,9 +40,9 @@ ConfigBuffer::ConfigBuffer(std::unique_ptr &&buffer, Expected ConfigBuffer::program_descriptors() { + // TODO HRT-9657: remove DEVICE interrupts auto descriptors_count = - m_buffer->program_descriptors(m_acc_buffer_offset, VdmaInterruptsDomain::NONE, VdmaInterruptsDomain::DEVICE, - m_acc_desc_count, false); + m_buffer->program_descriptors(m_acc_buffer_offset, vdma::InterruptsDomain::DEVICE, m_acc_desc_count, false); CHECK_EXPECTED(descriptors_count); m_acc_desc_count += descriptors_count.value(); @@ -68,7 +69,8 @@ hailo_status ConfigBuffer::pad_with_nops() CCW of all zeros (64'h0) should be treated as NOP - ignore CCW and expect CCW in next 64b word. When CSM recognize it is a NOP it pops it from the channel FIFO without forward any address/data/command, does not contribute to CRC calculations but return credits to the peripheral as usual. */ - write_inner(MemoryView::create_const(reinterpret_cast(&CCW_NOP), sizeof(CCW_NOP))); + auto status = write_inner(MemoryView::create_const(reinterpret_cast(&CCW_NOP), sizeof(CCW_NOP))); + CHECK_SUCCESS(status); } return HAILO_SUCCESS; } @@ -77,7 +79,9 @@ hailo_status ConfigBuffer::pad_with_nops() hailo_status ConfigBuffer::write(const MemoryView &data) { CHECK(data.size() <= size_left(), HAILO_INTERNAL_FAILURE, "Write too many config words"); - write_inner(data); + auto status = write_inner(data); + CHECK_SUCCESS(status); + m_current_buffer_size += data.size(); return HAILO_SUCCESS; } @@ -119,16 +123,21 @@ hailo_status ConfigBuffer::write_inner(const MemoryView &data) } Expected> ConfigBuffer::create_sg_buffer(HailoRTDriver &driver, - uint8_t vdma_channel_index, const std::vector &cfg_sizes) + vdma::ChannelId channel_id, const std::vector &cfg_sizes) { - auto desc_sizes_pair = VdmaDescriptorList::get_desc_buffer_sizes_for_multiple_transfers(driver, 1, cfg_sizes); + auto desc_sizes_pair = vdma::DescriptorList::get_desc_buffer_sizes_for_multiple_transfers(driver, 1, cfg_sizes); CHECK_EXPECTED(desc_sizes_pair); + const auto page_size = desc_sizes_pair->first; + const auto descs_count = desc_sizes_pair->second; - auto page_size = desc_sizes_pair->first; - auto descs_count = desc_sizes_pair->second; + size_t buffer_size = 0; + for (const auto cfg_size : cfg_sizes) { + const auto descs_count_for_cfg = DIV_ROUND_UP(cfg_size, page_size); + buffer_size += descs_count_for_cfg * page_size; + } - auto buffer = vdma::SgBuffer::create(driver, descs_count, page_size, HailoRTDriver::DmaDirection::H2D, - vdma_channel_index); + auto buffer = vdma::SgBuffer::create(driver, buffer_size, descs_count, page_size, HailoRTDriver::DmaDirection::H2D, + channel_id); CHECK_EXPECTED(buffer); auto buffer_ptr = make_unique_nothrow(buffer.release()); @@ -156,7 +165,13 @@ bool ConfigBuffer::should_use_ccb(HailoRTDriver &driver) case HailoRTDriver::DmaType::PCIE: return false; case HailoRTDriver::DmaType::DRAM: - return true; + if (std::getenv("HAILO_FORCE_CONF_CHANNEL_OVER_DESC") != nullptr) { + LOGGER__WARNING("Using desc instead of CCB for config channel is not optimal for performance.\n"); + return false; + } + else { + return true; + } } // Shouldn't reach here diff --git a/hailort/libhailort/src/config_buffer.hpp b/hailort/libhailort/src/core_op/resource_manager/config_buffer.hpp similarity index 93% rename from hailort/libhailort/src/config_buffer.hpp rename to hailort/libhailort/src/core_op/resource_manager/config_buffer.hpp index d121c03..b27611e 100644 --- a/hailort/libhailort/src/config_buffer.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/config_buffer.hpp @@ -11,9 +11,11 @@ #ifndef _HAILO_CONFIG_BUFFER_HPP_ #define _HAILO_CONFIG_BUFFER_HPP_ -#include "vdma/vdma_buffer.hpp" #include "hailo/buffer.hpp" +#include "vdma/memory/vdma_buffer.hpp" + + namespace hailort { #define CCW_BYTES_IN_WORD (4) @@ -52,7 +54,7 @@ private: hailo_status write_inner(const MemoryView &data); static Expected> create_sg_buffer(HailoRTDriver &driver, - uint8_t vdma_channel_index, const std::vector &cfg_sizes); + vdma::ChannelId channel_id, const std::vector &cfg_sizes); static Expected> create_ccb_buffer(HailoRTDriver &driver, uint32_t buffer_size); diff --git a/hailort/libhailort/src/context_switch/context_switch_buffer_builder.cpp b/hailort/libhailort/src/core_op/resource_manager/context_switch_buffer_builder.cpp similarity index 97% rename from hailort/libhailort/src/context_switch/context_switch_buffer_builder.cpp rename to hailort/libhailort/src/core_op/resource_manager/context_switch_buffer_builder.cpp index 8f76282..e7c60b3 100644 --- a/hailort/libhailort/src/context_switch/context_switch_buffer_builder.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/context_switch_buffer_builder.cpp @@ -32,7 +32,6 @@ void ContextSwitchBufferBuilder::write_action(MemoryView action) auto &control = current_control(); memcpy(&control.context_network_data[control.context_network_data_length], action.data(), action_size); control.context_network_data_length += action_size; - control.actions_count++; } const std::vector &ContextSwitchBufferBuilder::get_controls() const @@ -62,7 +61,6 @@ void ContextSwitchBufferBuilder::start_new_control() m_controls.emplace_back(); auto &new_control = current_control(); new_control.context_network_data_length = 0; - new_control.actions_count = 0; new_control.context_type = static_cast(m_context_type); new_control.is_first_control_per_context = (1 == m_controls.size()); new_control.is_last_control_per_context = true; diff --git a/hailort/libhailort/src/context_switch/context_switch_buffer_builder.hpp b/hailort/libhailort/src/core_op/resource_manager/context_switch_buffer_builder.hpp similarity index 92% rename from hailort/libhailort/src/context_switch/context_switch_buffer_builder.hpp rename to hailort/libhailort/src/core_op/resource_manager/context_switch_buffer_builder.hpp index 645f30c..8e5e1f2 100644 --- a/hailort/libhailort/src/context_switch/context_switch_buffer_builder.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/context_switch_buffer_builder.hpp @@ -11,9 +11,11 @@ #define _HAILO_CONTEXT_SWITCH_BUFFER_BUILDER_HPP_ #include "hailo/hailort.h" -#include "control_protocol.hpp" -#include "layer_info.hpp" -#include "vdma/channel_id.hpp" + +#include "vdma/channel/channel_id.hpp" +#include "device_common/control_protocol.hpp" +#include "hef/layer_info.hpp" + namespace hailort { diff --git a/hailort/libhailort/src/ddr_channels_pair.cpp b/hailort/libhailort/src/core_op/resource_manager/ddr_channels_pair.cpp similarity index 75% rename from hailort/libhailort/src/ddr_channels_pair.cpp rename to hailort/libhailort/src/core_op/resource_manager/ddr_channels_pair.cpp index 68c99b2..b170300 100644 --- a/hailort/libhailort/src/ddr_channels_pair.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/ddr_channels_pair.cpp @@ -6,11 +6,13 @@ * @file ddr_channels_pair.cpp **/ -#include "ddr_channels_pair.hpp" -#include "vdma/continuous_buffer.hpp" -#include "vdma/sg_buffer.hpp" #include "common/utils.hpp" +#include "core_op/resource_manager/ddr_channels_pair.hpp" +#include "vdma/memory/continuous_buffer.hpp" +#include "vdma/memory/sg_buffer.hpp" + + namespace hailort { @@ -19,16 +21,16 @@ Expected DdrChannelsPair::create(HailoRTDriver &driver, const D { auto buffer_exp = should_use_ccb(driver) ? create_ccb_buffer(driver, ddr_channels_info.row_size, ddr_channels_info.min_buffered_rows) : - create_sg_buffer(driver, ddr_channels_info.row_size, ddr_channels_info.min_buffered_rows); + create_sg_buffer(driver, ddr_channels_info.row_size, ddr_channels_info.min_buffered_rows, ddr_channels_info.d2h_channel_id); CHECK_EXPECTED(buffer_exp); auto buffer_ptr = buffer_exp.release(); CHECK_AS_EXPECTED(0 == (ddr_channels_info.row_size % buffer_ptr->desc_page_size()), HAILO_INTERNAL_FAILURE, "DDR channel buffer row size must be a multiple of descriptor page size"); - const auto interrupts_domain = VdmaInterruptsDomain::NONE; + const auto interrupts_domain = vdma::InterruptsDomain::NONE; const auto total_size = buffer_ptr->descs_count() * buffer_ptr->desc_page_size(); - auto desc_count_local = buffer_ptr->program_descriptors(total_size, interrupts_domain, interrupts_domain, 0, true); + auto desc_count_local = buffer_ptr->program_descriptors(total_size, interrupts_domain, 0, true); CHECK_EXPECTED(desc_count_local); return DdrChannelsPair(std::move(buffer_ptr), ddr_channels_info); @@ -75,16 +77,18 @@ CONTROL_PROTOCOL__host_buffer_info_t DdrChannelsPair::get_host_buffer_info() con } Expected> DdrChannelsPair::create_sg_buffer(HailoRTDriver &driver, - uint32_t row_size, uint16_t buffered_rows) + uint32_t row_size, uint16_t buffered_rows, vdma::ChannelId d2h_channel_id) { - auto desc_sizes_pair = VdmaDescriptorList::get_desc_buffer_sizes_for_single_transfer(driver, + auto desc_sizes_pair = vdma::DescriptorList::get_desc_buffer_sizes_for_single_transfer(driver, buffered_rows, buffered_rows, row_size); CHECK_EXPECTED(desc_sizes_pair); - auto desc_page_size = desc_sizes_pair->first; - auto descs_count = desc_sizes_pair->second; + const auto desc_page_size = desc_sizes_pair->first; + const auto descs_count = desc_sizes_pair->second; + // DdrChannels are circular so we need to allocate the full descriptors list. + const auto buffer_size = desc_page_size * descs_count; - auto buffer = vdma::SgBuffer::create(driver, descs_count, desc_page_size, - HailoRTDriver::DmaDirection::BOTH); + auto buffer = vdma::SgBuffer::create(driver, buffer_size, descs_count, desc_page_size, + HailoRTDriver::DmaDirection::BOTH, d2h_channel_id); CHECK_EXPECTED(buffer); auto buffer_ptr = make_unique_nothrow(buffer.release()); @@ -119,10 +123,15 @@ bool DdrChannelsPair::should_use_ccb(HailoRTDriver &driver) case HailoRTDriver::DmaType::PCIE: return false; case HailoRTDriver::DmaType::DRAM: - return true; + if (std::getenv("HAILO_FORCE_DDR_CHANNEL_OVER_DESC") != nullptr) { + LOGGER__WARNING("Using desc instead of CCB for ddr channel is not optimal for performance.\n"); + return false; + } + else { + return true; + } } - // Shouldn't reach here assert(false); return false; diff --git a/hailort/libhailort/src/ddr_channels_pair.hpp b/hailort/libhailort/src/core_op/resource_manager/ddr_channels_pair.hpp similarity index 91% rename from hailort/libhailort/src/ddr_channels_pair.hpp rename to hailort/libhailort/src/core_op/resource_manager/ddr_channels_pair.hpp index 25f0214..4caadc3 100644 --- a/hailort/libhailort/src/ddr_channels_pair.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/ddr_channels_pair.hpp @@ -12,7 +12,9 @@ #include "hailo/hailort.h" #include "hailo/buffer.hpp" -#include "vdma/vdma_buffer.hpp" + +#include "vdma/memory/vdma_buffer.hpp" + namespace hailort { @@ -51,7 +53,7 @@ private: DdrChannelsPair(std::unique_ptr &&buffer, const DdrChannelsInfo &ddr_channels_info); static Expected> create_sg_buffer(HailoRTDriver &driver, - uint32_t row_size, uint16_t buffered_rows); + uint32_t row_size, uint16_t buffered_rows, vdma::ChannelId d2h_channel_id); static Expected> create_ccb_buffer(HailoRTDriver &driver, uint32_t row_size, uint16_t buffered_rows); diff --git a/hailort/libhailort/src/inter_context_buffer.cpp b/hailort/libhailort/src/core_op/resource_manager/inter_context_buffer.cpp similarity index 76% rename from hailort/libhailort/src/inter_context_buffer.cpp rename to hailort/libhailort/src/core_op/resource_manager/inter_context_buffer.cpp index 4b42705..5bac263 100644 --- a/hailort/libhailort/src/inter_context_buffer.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/inter_context_buffer.cpp @@ -7,32 +7,31 @@ * @brief Manages inter-context buffer. */ -#include "context_switch/multi_context/resource_manager.hpp" -#include "inter_context_buffer.hpp" -#include "vdma/sg_buffer.hpp" -#include "vdma/continuous_buffer.hpp" +#include "core_op/resource_manager/resource_manager.hpp" +#include "core_op/resource_manager/inter_context_buffer.hpp" +#include "vdma/memory/sg_buffer.hpp" +#include "vdma/memory/continuous_buffer.hpp" namespace hailort { Expected InterContextBuffer::create(HailoRTDriver &driver, uint32_t transfer_size, - uint16_t max_batch_size) + uint16_t max_batch_size, vdma::ChannelId d2h_channel_id) { auto buffer_exp = should_use_ccb(driver) ? create_ccb_buffer(driver, transfer_size, max_batch_size) : - create_sg_buffer(driver, transfer_size, max_batch_size); + create_sg_buffer(driver, transfer_size, max_batch_size, d2h_channel_id); CHECK_EXPECTED(buffer_exp); auto buffer_ptr = buffer_exp.release(); size_t acc_offset = 0; for (uint16_t i = 0; i < max_batch_size; i++) { - const auto first_desc_interrupts_domain = VdmaInterruptsDomain::NONE; const auto last_desc_interrupts_domain = ((max_batch_size - 1) == i) ? - VdmaInterruptsDomain::DEVICE : VdmaInterruptsDomain::NONE; + vdma::InterruptsDomain::DEVICE : vdma::InterruptsDomain::NONE; static const auto BUFFER_NOT_CIRCULAR = false; - auto desc_count_local = buffer_ptr->program_descriptors(transfer_size, first_desc_interrupts_domain, - last_desc_interrupts_domain, acc_offset, BUFFER_NOT_CIRCULAR); + auto desc_count_local = buffer_ptr->program_descriptors(transfer_size, last_desc_interrupts_domain, acc_offset, + BUFFER_NOT_CIRCULAR); CHECK_EXPECTED(desc_count_local, "Failed to program descs for inter context channels. Given max_batch_size is too big."); acc_offset += desc_count_local.value(); } @@ -52,11 +51,11 @@ hailo_status InterContextBuffer::reprogram(uint16_t batch_size) } status = m_buffer->reprogram_device_interrupts_for_end_of_batch(m_transfer_size, prev_batch_size, - VdmaInterruptsDomain::NONE); + vdma::InterruptsDomain::NONE); CHECK_SUCCESS(status, "Failed reprogramming device interrupts for the end of the previous batch (size {})", prev_batch_size); status = m_buffer->reprogram_device_interrupts_for_end_of_batch(m_transfer_size, m_dynamic_batch_size, - VdmaInterruptsDomain::DEVICE); + vdma::InterruptsDomain::DEVICE); CHECK_SUCCESS(status, "Failed reprogramming device interrupts for the end of the current batch (size {})", m_dynamic_batch_size); @@ -108,16 +107,19 @@ hailo_status InterContextBuffer::set_dynamic_batch_size(uint16_t batch_size) } Expected> InterContextBuffer::create_sg_buffer(HailoRTDriver &driver, - uint32_t transfer_size, uint16_t batch_size) + uint32_t transfer_size, uint16_t batch_size, vdma::ChannelId d2h_channel_id) { - auto desc_sizes_pair = VdmaDescriptorList::get_desc_buffer_sizes_for_single_transfer(driver, + auto desc_sizes_pair = vdma::DescriptorList::get_desc_buffer_sizes_for_single_transfer(driver, batch_size, batch_size, transfer_size); CHECK_EXPECTED(desc_sizes_pair); - auto desc_page_size = desc_sizes_pair->first; - auto descs_count = desc_sizes_pair->second; - - auto buffer = vdma::SgBuffer::create(driver, descs_count, desc_page_size, - HailoRTDriver::DmaDirection::BOTH); + const auto desc_page_size = desc_sizes_pair->first; + const auto descs_count = desc_sizes_pair->second; + + // TODO: HRT-9914 - Instead of using aligned descriptor for each transfer, we should do it for the all frame. + const size_t desc_per_transfer = DIV_ROUND_UP(transfer_size, desc_page_size); + const size_t buffer_size = desc_per_transfer * desc_page_size * batch_size; + auto buffer = vdma::SgBuffer::create(driver, buffer_size, descs_count, desc_page_size, + HailoRTDriver::DmaDirection::BOTH, d2h_channel_id); CHECK_EXPECTED(buffer); auto buffer_ptr = make_unique_nothrow(buffer.release()); @@ -147,7 +149,13 @@ bool InterContextBuffer::should_use_ccb(HailoRTDriver &driver) case HailoRTDriver::DmaType::PCIE: return false; case HailoRTDriver::DmaType::DRAM: - return true; + if (nullptr == std::getenv("HAILO_FORCE_INFER_CONTEXT_CHANNEL_OVER_DESC")) { + return false; + } + else { + LOGGER__INFO("Using (non default mode) CCB for inter context channels.\n"); + return true; + } } // Shouldn't reach here diff --git a/hailort/libhailort/src/inter_context_buffer.hpp b/hailort/libhailort/src/core_op/resource_manager/inter_context_buffer.hpp similarity index 85% rename from hailort/libhailort/src/inter_context_buffer.hpp rename to hailort/libhailort/src/core_op/resource_manager/inter_context_buffer.hpp index 37dcce0..912501e 100644 --- a/hailort/libhailort/src/inter_context_buffer.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/inter_context_buffer.hpp @@ -10,10 +10,12 @@ #ifndef _HAILO_INTER_CONTEXT_BUFFER_HPP_ #define _HAILO_INTER_CONTEXT_BUFFER_HPP_ -#include "os/hailort_driver.hpp" -#include "vdma/vdma_buffer.hpp" #include "hailo/expected.hpp" #include "hailo/buffer.hpp" + +#include "os/hailort_driver.hpp" +#include "vdma/memory/vdma_buffer.hpp" + #include "control_protocol.h" @@ -23,7 +25,7 @@ namespace hailort class InterContextBuffer final { public: static Expected create(HailoRTDriver &driver, uint32_t transfer_size, - uint16_t max_batch_size); + uint16_t max_batch_size, vdma::ChannelId d2h_channel_id); hailo_status reprogram(uint16_t batch_size); Expected read(); @@ -35,7 +37,7 @@ private: hailo_status set_dynamic_batch_size(uint16_t batch_size); static Expected> create_sg_buffer(HailoRTDriver &driver, - uint32_t transfer_size, uint16_t batch_size); + uint32_t transfer_size, uint16_t batch_size, vdma::ChannelId d2h_channel_id); static Expected> create_ccb_buffer(HailoRTDriver &driver, uint32_t transfer_size, uint16_t batch_size); diff --git a/hailort/libhailort/src/context_switch/resource_manager.cpp b/hailort/libhailort/src/core_op/resource_manager/resource_manager.cpp similarity index 51% rename from hailort/libhailort/src/context_switch/resource_manager.cpp rename to hailort/libhailort/src/core_op/resource_manager/resource_manager.cpp index 7345772..64b97e9 100644 --- a/hailort/libhailort/src/context_switch/resource_manager.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/resource_manager.cpp @@ -1,12 +1,15 @@ -#include "multi_context/resource_manager.hpp" -#include "control.hpp" -#include "hailort_defaults.hpp" +#include "hailo/hailort_defaults.hpp" + +#include "core_op/resource_manager/resource_manager.hpp" +#include "vdma/channel/boundary_channel.hpp" +#include "device_common/control.hpp" + #include + namespace hailort { - Expected ContextResources::create(HailoRTDriver &driver, CONTROL_PROTOCOL__context_switch_context_type_t context_type, const std::vector &config_channels_ids, const ConfigBufferInfoMap &config_buffer_infos) @@ -39,21 +42,57 @@ ContextSwitchBufferBuilder &ContextResources::builder() return m_builder; } -const std::vector &ContextResources::get_boundary_layers() const +void ContextResources::add_edge_layer(const LayerInfo &layer_info, vdma::ChannelId channel_id, + const CONTROL_PROTOCOL__host_buffer_info_t &buffer_info) +{ + m_edge_layers.emplace_back(EdgeLayer{ + layer_info, + channel_id, + buffer_info + }); +} + +std::vector ContextResources::get_edge_layers() const +{ + return m_edge_layers; +} + +std::vector ContextResources::get_edge_layers(LayerType layer_type) const { - return m_boundary_layers; + return get_edge_layers(layer_type, HAILO_STREAM_DIRECTION_MAX_ENUM); } -const std::vector &ContextResources::get_inter_context_layers() const +std::vector ContextResources::get_edge_layers(hailo_stream_direction_t direction) const { - return m_inter_context_layers; + return get_edge_layers(LayerType::NOT_SET, direction); } -const std::vector &ContextResources::get_ddr_channel_layers() const +std::vector ContextResources::get_edge_layers(LayerType layer_type, hailo_stream_direction_t direction) const { - return m_ddr_channel_layers; + std::vector edge_layers; + for (const auto &edge_layer : m_edge_layers) { + const bool layer_type_ok = (layer_type == LayerType::NOT_SET) || (edge_layer.layer_info.type == layer_type); + const bool direction_ok = (direction == HAILO_STREAM_DIRECTION_MAX_ENUM) || (edge_layer.layer_info.direction == direction); + if (layer_type_ok && direction_ok) { + edge_layers.emplace_back(edge_layer); + } + } + return edge_layers; +} + +Expected ContextResources::get_edge_layer_by_stream_index(uint8_t stream_index) const +{ + for (const auto &edge_layer : m_edge_layers) { + if (edge_layer.layer_info.stream_index == stream_index) { + return EdgeLayer(edge_layer); + } + } + + LOGGER__ERROR("Edge layer does not exists for stream {}", stream_index); + return make_unexpected(HAILO_INTERNAL_FAILURE); } + ExpectedRef ContextResources::create_ddr_channels_pair(const DdrChannelsInfo &ddr_info) { auto buffer = DdrChannelsPair::create(m_driver, ddr_info); @@ -80,55 +119,10 @@ const std::vector &ContextResources::get_ddr_channels_pairs() c return m_ddr_channels_pairs; } -std::vector ContextResources::get_boundary_layers(hailo_stream_direction_t direction) const -{ - std::vector edge_layers; - for (const auto &edge_layer : m_boundary_layers) { - if (edge_layer.layer_info.direction == direction) { - edge_layers.push_back(edge_layer); - } - } - return edge_layers; -} - -std::vector ContextResources::get_inter_context_layers(hailo_stream_direction_t direction) const -{ - std::vector edge_layers; - for (const auto &edge_layer : m_inter_context_layers) { - if (edge_layer.layer_info.direction == direction) { - edge_layers.push_back(edge_layer); - } - } - return edge_layers; -} - -std::vector ContextResources::get_ddr_channel_layers(hailo_stream_direction_t direction) const -{ - std::vector edge_layers; - for (const auto &edge_layer : m_ddr_channel_layers) { - if (edge_layer.layer_info.direction == direction) { - edge_layers.push_back(edge_layer); - } - } - return edge_layers; -} - hailo_status ContextResources::validate_edge_layers() { std::set used_channel_ids; - for (const auto &edge_layer : get_boundary_layers()) { - CHECK(used_channel_ids.find(edge_layer.channel_id) == used_channel_ids.end(), HAILO_INTERNAL_FAILURE, - "Same stream use the same channel id {}", edge_layer.channel_id); - used_channel_ids.insert(edge_layer.channel_id); - } - - for (const auto &edge_layer : get_inter_context_layers()) { - CHECK(used_channel_ids.find(edge_layer.channel_id) == used_channel_ids.end(), HAILO_INTERNAL_FAILURE, - "Same stream use the same channel id {}", edge_layer.channel_id); - used_channel_ids.insert(edge_layer.channel_id); - } - - for (const auto &edge_layer : get_ddr_channel_layers()) { + for (const auto &edge_layer : m_edge_layers) { CHECK(used_channel_ids.find(edge_layer.channel_id) == used_channel_ids.end(), HAILO_INTERNAL_FAILURE, "Same stream use the same channel id {}", edge_layer.channel_id); used_channel_ids.insert(edge_layer.channel_id); @@ -170,15 +164,15 @@ static Expected create_hw_latency_meter(const std::vector create_latency_meters_from_config_params( - const ConfigureNetworkParams &config_params, std::shared_ptr network_group_metadata) + const ConfigureNetworkParams &config_params, std::shared_ptr core_op_metadata) { LatencyMetersMap latency_meters_map; if ((config_params.latency & HAILO_LATENCY_MEASURE) == HAILO_LATENCY_MEASURE) { // Best affort for starting latency meter. - auto networks_names = network_group_metadata->get_network_names(); + auto networks_names = core_op_metadata->get_network_names(); for (auto &network_name : networks_names) { - auto layer_infos = network_group_metadata->get_all_layer_infos(network_name); + auto layer_infos = core_op_metadata->get_all_layer_infos(network_name); CHECK_EXPECTED(layer_infos); auto latency_meter = create_hw_latency_meter(layer_infos.value()); if (latency_meter) { @@ -192,29 +186,29 @@ static Expected create_latency_meters_from_config_params( } Expected ResourcesManager::create(VdmaDevice &vdma_device, HailoRTDriver &driver, - const ConfigureNetworkParams &config_params, std::shared_ptr network_group_metadata, - uint8_t net_group_index) + const ConfigureNetworkParams &config_params, std::shared_ptr core_op_metadata, + uint8_t core_op_index) { // Allocate config channels. In order to use the same channel ids for config channels in all contexts, // we allocate all of them here, and use in preliminary/dynamic context. ChannelAllocator allocator(driver.dma_engines_count()); std::vector config_channels_ids; - const auto &config_channels_info = network_group_metadata->config_channels_info(); + const auto &config_channels_info = core_op_metadata->config_channels_info(); config_channels_ids.reserve(config_channels_info.size()); for (uint8_t cfg_index = 0; cfg_index < config_channels_info.size(); cfg_index++) { const auto layer_identifier = std::make_tuple(LayerType::CFG, "", cfg_index); const auto engine_index = config_channels_info[cfg_index].engine_index; - auto channel_id = allocator.get_available_channel_id(layer_identifier, VdmaChannel::Direction::H2D, engine_index); + auto channel_id = allocator.get_available_channel_id(layer_identifier, HailoRTDriver::DmaDirection::H2D, engine_index); CHECK_EXPECTED(channel_id); config_channels_ids.push_back(channel_id.release()); } - auto network_index_map = network_group_metadata->get_network_names(); + auto network_index_map = core_op_metadata->get_network_names(); - auto latency_meters = create_latency_meters_from_config_params(config_params, network_group_metadata); + auto latency_meters = create_latency_meters_from_config_params(config_params, core_op_metadata); CHECK_EXPECTED(latency_meters); ResourcesManager resources_manager(vdma_device, driver, std::move(allocator), config_params, - std::move(network_group_metadata), net_group_index, + std::move(core_op_metadata), core_op_index, std::move(network_index_map), latency_meters.release(), std::move(config_channels_ids)); return resources_manager; @@ -222,8 +216,8 @@ Expected ResourcesManager::create(VdmaDevice &vdma_device, Hai ResourcesManager::ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &driver, ChannelAllocator &&channel_allocator, const ConfigureNetworkParams config_params, - std::shared_ptr &&network_group_metadata, - uint8_t net_group_index, const std::vector &&network_index_map, + std::shared_ptr &&core_op_metadata, + uint8_t core_op_index, const std::vector &&network_index_map, LatencyMetersMap &&latency_meters, std::vector &&config_channels_ids) : m_contexts_resources(), @@ -232,16 +226,16 @@ ResourcesManager::ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &drive m_driver(driver), m_config_params(config_params), m_inter_context_buffers(), - m_internal_channels(), - m_network_group_metadata(std::move(network_group_metadata)), - m_net_group_index(net_group_index), + m_core_op_metadata(std::move(core_op_metadata)), + m_core_op_index(core_op_index), m_dynamic_context_count(0), m_total_context_count(0), m_network_index_map(std::move(network_index_map)), m_latency_meters(std::move(latency_meters)), m_boundary_channels(), m_is_configured(false), - m_config_channels_ids(std::move(config_channels_ids)) + m_config_channels_ids(std::move(config_channels_ids)), + m_hw_only_boundary_buffers() {} ResourcesManager::ResourcesManager(ResourcesManager &&other) noexcept : @@ -251,21 +245,21 @@ ResourcesManager::ResourcesManager(ResourcesManager &&other) noexcept : m_driver(other.m_driver), m_config_params(other.m_config_params), m_inter_context_buffers(std::move(other.m_inter_context_buffers)), - m_internal_channels(std::move(other.m_internal_channels)), - m_network_group_metadata(std::move(other.m_network_group_metadata)), - m_net_group_index(other.m_net_group_index), + m_core_op_metadata(std::move(other.m_core_op_metadata)), + m_core_op_index(other.m_core_op_index), m_dynamic_context_count(std::exchange(other.m_dynamic_context_count, static_cast(0))), m_total_context_count(std::exchange(other.m_total_context_count, static_cast(0))), m_network_index_map(std::move(other.m_network_index_map)), m_latency_meters(std::move(other.m_latency_meters)), m_boundary_channels(std::move(other.m_boundary_channels)), m_is_configured(std::exchange(other.m_is_configured, false)), - m_config_channels_ids(std::move(other.m_config_channels_ids)) + m_config_channels_ids(std::move(other.m_config_channels_ids)), + m_hw_only_boundary_buffers(std::move(other.m_hw_only_boundary_buffers)) {} hailo_status ResourcesManager::fill_infer_features(CONTROL_PROTOCOL__application_header_t &app_header) { - app_header.infer_features.preliminary_run_asap = m_network_group_metadata->supported_features().preliminary_run_asap; + app_header.infer_features.preliminary_run_asap = m_core_op_metadata->supported_features().preliminary_run_asap; return HAILO_SUCCESS; } @@ -307,26 +301,50 @@ hailo_status ResourcesManager::fill_network_batch_size(CONTROL_PROTOCOL__applica return HAILO_SUCCESS; } -hailo_status ResourcesManager::create_internal_vdma_channels() +hailo_status ResourcesManager::fill_csm_buffer_size(CONTROL_PROTOCOL__application_header_t &app_header) { - auto internal_channel_ids = m_channel_allocator.get_internal_channel_ids(); + // All config buffers on the same platform will have the same desc_page_size - because it is derived from the host + app_header.csm_buffer_size = std::min(m_driver.desc_max_page_size(), vdma::DEFAULT_DESC_PAGE_SIZE); + return HAILO_SUCCESS; +} - m_internal_channels.reserve(internal_channel_ids.size()); - for (const auto &ch : internal_channel_ids) { - auto direction = (ch.channel_index < MIN_D2H_CHANNEL_INDEX) ? VdmaChannel::Direction::H2D : VdmaChannel::Direction::D2H; - auto vdma_channel = VdmaChannel::create(ch, direction, m_driver, m_vdma_device.get_default_desc_page_size()); - CHECK_EXPECTED_AS_STATUS(vdma_channel); - m_internal_channels.emplace_back(vdma_channel.release()); - } +void ResourcesManager::process_interrupts(IrqData &&irq_data) +{ + assert(irq_data.channels_count <= ARRAY_ENTRIES(irq_data.channels_irq_data)); + for (uint8_t irq_index = 0; irq_index < irq_data.channels_count; irq_index++) { + const auto &channel_irq_data = irq_data.channels_irq_data[irq_index]; + auto boundary_channel = m_boundary_channels.find(channel_irq_data.channel_id); + if (std::end(m_boundary_channels) == boundary_channel) { + LOGGER__ERROR("Got interrupt for channel {}, but there is no such boundary channel", channel_irq_data.channel_id); + continue; + } - return HAILO_SUCCESS; + if ((channel_irq_data.host_error != 0) || (channel_irq_data.device_error != 0)) { + LOGGER__CRITICAL("Got error on channel {} host_error=0x{:x} device_error=0x{:x}", + channel_irq_data.channel_id, channel_irq_data.host_error, channel_irq_data.device_error); + continue; + } + + if (!channel_irq_data.is_active) { + LOGGER__CRITICAL("Channel {} was aborted by external source", channel_irq_data.channel_id); + continue; + } + + auto status = boundary_channel->second->trigger_channel_completion(channel_irq_data.desc_num_processed); + if ((status != HAILO_SUCCESS) && + (status != HAILO_STREAM_ABORTED_BY_USER) && + (status != HAILO_STREAM_NOT_ACTIVATED)) { + // Log error and continue gracefully to process other interrupts + LOGGER__ERROR("Trigger channel completion failed on channel {} with status {}", channel_irq_data.channel_id, status); + } + } } hailo_status ResourcesManager::create_boundary_vdma_channel(const LayerInfo &layer_info) { // TODO: put in layer info - const auto channel_direction = layer_info.direction == HAILO_H2D_STREAM ? VdmaChannel::Direction::H2D : - VdmaChannel::Direction::D2H; + const auto channel_direction = layer_info.direction == HAILO_H2D_STREAM ? HailoRTDriver::DmaDirection::H2D : + HailoRTDriver::DmaDirection::D2H; const auto channel_id = get_available_channel_id(to_layer_identifier(layer_info), channel_direction, layer_info.dma_engine_index); CHECK_EXPECTED_AS_STATUS(channel_id); @@ -347,43 +365,44 @@ hailo_status ResourcesManager::create_boundary_vdma_channel(const LayerInfo &lay /* TODO - HRT-6829- page_size should be calculated inside the vDMA channel class create function */ const auto transfer_size = (layer_info.nn_stream_config.periph_bytes_per_buffer * layer_info.nn_stream_config.core_buffers_per_frame); - auto desc_sizes_pair = VdmaDescriptorList::get_desc_buffer_sizes_for_single_transfer(m_driver, + auto desc_sizes_pair = vdma::DescriptorList::get_desc_buffer_sizes_for_single_transfer(m_driver, static_cast(min_active_trans), static_cast(max_active_trans), transfer_size); CHECK_EXPECTED_AS_STATUS(desc_sizes_pair); const auto page_size = desc_sizes_pair->first; - const auto descs_count = desc_sizes_pair->second; - auto channel = VdmaChannel::create(channel_id.value(), channel_direction, m_driver, page_size, - layer_info.name, latency_meter, network_batch_size.value()); - CHECK_EXPECTED_AS_STATUS(channel); - const auto status = channel->allocate_resources(descs_count); - CHECK_SUCCESS(status); + const auto descs_count = (nullptr != std::getenv("HAILO_CONFIGURE_FOR_HW_INFER")) ? + MAX_DESCS_COUNT : desc_sizes_pair->second; - auto channel_ptr = make_shared_nothrow(channel.release()); - CHECK_NOT_NULL(channel_ptr, HAILO_OUT_OF_HOST_MEMORY); + const auto channel_type = (0 == (m_config_params.stream_params_by_name.at(layer_info.name).flags & HAILO_STREAM_FLAGS_ASYNC)) ? + vdma::BoundaryChannel::Type::BUFFERED : vdma::BoundaryChannel::Type::ASYNC; + auto channel = vdma::BoundaryChannel::create(channel_id.value(), channel_direction, m_driver, descs_count, page_size, + layer_info.name, latency_meter, network_batch_size.value(), channel_type); + CHECK_EXPECTED_AS_STATUS(channel); - m_boundary_channels.emplace(layer_info.name, channel_ptr); + m_boundary_channels.emplace(channel_id.value(), channel.release()); return HAILO_SUCCESS; } -Expected> ResourcesManager::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) +Expected ResourcesManager::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) { - auto boundary_channel_it = m_boundary_channels.find(stream_name); - if (std::end(m_boundary_channels) == boundary_channel_it) { - return make_unexpected(HAILO_NOT_FOUND); + for (const auto &boundary_channel : m_boundary_channels) { + if (boundary_channel.second->stream_name() == stream_name) { + return vdma::BoundaryChannelPtr(boundary_channel.second); + } } - return std::shared_ptr(boundary_channel_it->second); + return make_unexpected(HAILO_NOT_FOUND); } -Expected> ResourcesManager::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) const +Expected> ResourcesManager::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) const { - auto boundary_channel_it = m_boundary_channels.find(stream_name); - if (std::end(m_boundary_channels) == boundary_channel_it) { - return make_unexpected(HAILO_NOT_FOUND); + for (const auto &boundary_channel : m_boundary_channels) { + if (boundary_channel.second->stream_name() == stream_name) { + return std::shared_ptr(boundary_channel.second); + } } - return std::shared_ptr(boundary_channel_it->second); + return make_unexpected(HAILO_NOT_FOUND); } hailo_power_mode_t ResourcesManager::get_power_mode() const @@ -392,13 +411,13 @@ hailo_power_mode_t ResourcesManager::get_power_mode() const } ExpectedRef ResourcesManager::create_inter_context_buffer(uint32_t transfer_size, - uint8_t src_stream_index, uint8_t src_context_index, const std::string &network_name) + uint8_t src_stream_index, uint8_t src_context_index, const std::string &network_name, vdma::ChannelId d2h_channel_id) { auto network_batch_size_exp = get_network_batch_size(network_name); CHECK_EXPECTED(network_batch_size_exp); auto network_batch_size = network_batch_size_exp.value(); - auto buffer = InterContextBuffer::create(m_driver, transfer_size, network_batch_size); + auto buffer = InterContextBuffer::create(m_driver, transfer_size, network_batch_size, d2h_channel_id); CHECK_EXPECTED(buffer); const auto key = std::make_pair(src_context_index, src_stream_index); @@ -416,7 +435,7 @@ ExpectedRef ResourcesManager::get_inter_context_buffer(const return std::ref(buffer_it->second); } -Expected ResourcesManager::get_control_network_group_header() +Expected ResourcesManager::get_control_core_op_header() { CONTROL_PROTOCOL__application_header_t app_header{}; app_header.dynamic_contexts_count = m_dynamic_context_count; @@ -427,6 +446,8 @@ Expected ResourcesManager::get_control_n CHECK_SUCCESS_AS_EXPECTED(status, "Invalid validation features"); status = fill_network_batch_size(app_header); CHECK_SUCCESS_AS_EXPECTED(status, "Invalid network batch sizes"); + status = fill_csm_buffer_size(app_header); + CHECK_SUCCESS_AS_EXPECTED(status, "Invalid csm buffer size"); return app_header; } @@ -449,7 +470,7 @@ Expected> ResourcesManager::add_new_con } Expected ResourcesManager::get_available_channel_id(const LayerIdentifier &layer_identifier, - VdmaChannel::Direction direction, uint8_t engine_index) + HailoRTDriver::DmaDirection direction, uint8_t engine_index) { if (m_driver.dma_type() == HailoRTDriver::DmaType::PCIE) { // On PCIe we have only 1 engine. To support the same HEF with both PCIe and DRAM, we use default engine here @@ -469,38 +490,6 @@ Expected ResourcesManager::get_default_streams_interfa return m_vdma_device.get_default_streams_interface(); } -hailo_status ResourcesManager::register_fw_managed_vdma_channels() -{ - hailo_status status = HAILO_UNINITIALIZED; - - for (auto &ch : m_internal_channels) { - status = ch.register_fw_controlled_channel(); - CHECK_SUCCESS(status); - } - - for (auto &ch : m_boundary_channels) { - status = ch.second->register_fw_controlled_channel(); - CHECK_SUCCESS(status); - } - - return HAILO_SUCCESS; -} - -hailo_status ResourcesManager::unregister_fw_managed_vdma_channels() -{ - hailo_status status = HAILO_UNINITIALIZED; - - // Note: We don't "unregister" the m_boundary_channels here, beacuse the Vdma*Stream objects will unregister their - // own channels. - // TODO: Add one icotl to stop all channels at once (HRT-6097) - for (auto &ch : m_internal_channels) { - status = ch.unregister_fw_controlled_channel(); - CHECK_SUCCESS(status); - } - - return HAILO_SUCCESS; -} - hailo_status ResourcesManager::set_inter_context_channels_dynamic_batch_size(uint16_t dynamic_batch_size) { for (auto &key_buff_pair : m_inter_context_buffers) { @@ -553,13 +542,13 @@ Expected ResourcesManager::read_intermediate_buffer(const IntermediateBu hailo_status ResourcesManager::configure() { - CHECK(!m_is_configured, HAILO_INTERNAL_FAILURE, "Can't configure the same network group twice"); + CHECK(!m_is_configured, HAILO_INTERNAL_FAILURE, "Can't configure the same core-op twice"); m_is_configured = true; - auto net_group_header = get_control_network_group_header(); - CHECK_EXPECTED_AS_STATUS(net_group_header); + auto core_op_header = get_control_core_op_header(); + CHECK_EXPECTED_AS_STATUS(core_op_header); - auto status = Control::context_switch_set_network_group_header(m_vdma_device, net_group_header.release()); + auto status = Control::context_switch_set_network_group_header(m_vdma_device, core_op_header.release()); CHECK_SUCCESS(status); for (const auto &context : m_contexts_resources) { @@ -572,7 +561,7 @@ hailo_status ResourcesManager::configure() hailo_status ResourcesManager::enable_state_machine(uint16_t dynamic_batch_size) { - return Control::enable_network_group(m_vdma_device, m_net_group_index, dynamic_batch_size); + return Control::enable_core_op(m_vdma_device, m_core_op_index, dynamic_batch_size); } hailo_status ResourcesManager::reset_state_machine(bool keep_nn_config_during_reset) @@ -580,7 +569,7 @@ hailo_status ResourcesManager::reset_state_machine(bool keep_nn_config_during_re auto status = Control::reset_context_switch_state_machine(m_vdma_device, keep_nn_config_during_reset); CHECK_SUCCESS(status); - if (!keep_nn_config_during_reset && (Device::Type::CORE == m_vdma_device.get_type())) { + if (!keep_nn_config_during_reset && (Device::Type::INTEGRATED == m_vdma_device.get_type())) { // On core device, the nn_manager is not responsible to reset the nn-core so // we use the SCU control for that. status = m_vdma_device.reset(HAILO_RESET_DEVICE_MODE_NN_CORE); @@ -590,4 +579,189 @@ hailo_status ResourcesManager::reset_state_machine(bool keep_nn_config_during_re return HAILO_SUCCESS; } +hailo_status ResourcesManager::cancel_pending_async_transfers() +{ + for (const auto &boundary_channel : m_boundary_channels) { + if (boundary_channel.second->type() != vdma::BoundaryChannel::Type::ASYNC) { + continue; + } + + // Best effort + const auto status = boundary_channel.second->cancel_pending_transfers(); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed cancellation of pending transfers on async channel {}", boundary_channel.second->stream_name()); + } + } + return HAILO_SUCCESS; +} + +hailo_status ResourcesManager::start_vdma_interrupts_dispatcher() +{ + auto interrupts_dispatcher = m_vdma_device.get_vdma_interrupts_dispatcher(); + CHECK_EXPECTED_AS_STATUS(interrupts_dispatcher); + + ChannelsBitmap channels_bitmap{}; + for (const auto &boundary_channel : m_boundary_channels) { + const auto channel_id = boundary_channel.first; + channels_bitmap[channel_id.engine_index] |= (1 << channel_id.channel_index); + } + + const bool enable_timestamp_measure = !m_latency_meters.empty(); + return interrupts_dispatcher->get().start(channels_bitmap, enable_timestamp_measure, [this](IrqData &&irq_data){ + process_interrupts(std::move(irq_data)); + }); +} + +hailo_status ResourcesManager::stop_vdma_interrupts_dispatcher() +{ + auto interrupts_dispatcher = m_vdma_device.get_vdma_interrupts_dispatcher(); + CHECK_EXPECTED_AS_STATUS(interrupts_dispatcher); + return interrupts_dispatcher->get().stop(); +} + +Expected ResourcesManager::program_desc_for_hw_only_flow(std::shared_ptr desc_list, + const uint32_t single_transfer_size, const uint16_t dynamic_batch_size, const uint16_t batch_count) +{ + size_t acc_desc_offset = 0; + for (uint16_t batch_index = 0; batch_index < batch_count; batch_index++) { + for (uint16_t transfer_index = 0; transfer_index < dynamic_batch_size; transfer_index++) { + const auto last_desc_interrupts_domain = ((dynamic_batch_size - 1) == transfer_index) ? + vdma::InterruptsDomain::DEVICE : vdma::InterruptsDomain::NONE; + static const auto BUFFER_NOT_CIRCULAR = false; + auto desc_count_local = desc_list->program_last_descriptor(single_transfer_size, + last_desc_interrupts_domain, acc_desc_offset, BUFFER_NOT_CIRCULAR); + CHECK_EXPECTED(desc_count_local, "Failed to program descs for inter context channels. Given max_batch_size is too big."); + acc_desc_offset += desc_count_local.value(); + } + } + CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(acc_desc_offset), HAILO_INTERNAL_FAILURE, + "calculated acc_desc_offset for vdma descriptor list is out of UINT16 range"); + return static_cast(acc_desc_offset); +} + +Expected> ResourcesManager::create_mapped_buffer_for_hw_only_infer( + vdma::BoundaryChannelPtr boundary_channel_ptr, const hailo_vdma_buffer_direction_flags_t direction, + const uint32_t single_transfer_size, const uint16_t dynamic_batch_size, const uint16_t batch_count) +{ + auto total_frames_per_run = dynamic_batch_size * batch_count; + auto total_run_transfer_size = total_frames_per_run * single_transfer_size; + + auto desc_list = boundary_channel_ptr->get_desc_list(); + auto total_desc_count = desc_list->descriptors_in_buffer(total_run_transfer_size); + + CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(total_desc_count), HAILO_INVALID_ARGUMENT, + "calculated total_desc_count for vdma descriptor list is out of UINT16 range"); + + auto mapped_buffer_exp = DmaMappedBuffer::create(total_desc_count * desc_list->desc_page_size(), direction, m_vdma_device); + CHECK_EXPECTED(mapped_buffer_exp); + + auto mapped_buffer = make_shared_nothrow(mapped_buffer_exp.release()); + CHECK_NOT_NULL_AS_EXPECTED(mapped_buffer, HAILO_OUT_OF_HOST_MEMORY); + m_hw_only_boundary_buffers.push_back(mapped_buffer); + + uint32_t STARTING_DESC = 0; + auto status = desc_list->configure_to_use_buffer(*mapped_buffer, boundary_channel_ptr->get_channel_id(), STARTING_DESC); + CHECK_SUCCESS_AS_EXPECTED(status); + + auto desc_programed = program_desc_for_hw_only_flow(desc_list, single_transfer_size, dynamic_batch_size, batch_count); + CHECK_EXPECTED(desc_programed); + + auto channel_info_pair = std::make_pair(boundary_channel_ptr->get_channel_id(), desc_programed.release()); + + return channel_info_pair; +} + +void ResourcesManager::add_channel_to_hw_infer_channel_info(std::pair channel_info, + CONTROL_PROTOCOL__hw_infer_channels_info_t &channels_info) +{ + auto next_chnanel_info = &channels_info.channel_info[channels_info.channel_count]; + assert(channels_info.channel_count < CONTROL_PROTOCOL__MAX_TOTAL_CHANNEL_COUNT); + + next_chnanel_info->engine_index = channel_info.first.engine_index; + next_chnanel_info->channel_index = channel_info.first.channel_index; + next_chnanel_info->desc_programed = channel_info.second; + + channels_info.channel_count++; +} + +Expected ResourcesManager::calc_hw_infer_batch_count(uint16_t dynamic_batch_size) +{ + uint16_t batch_count = UINT16_MAX; + for (const auto &layer_info : m_core_op_metadata->get_all_layer_infos()) { + const auto stream_info = LayerInfoUtils::get_stream_info_from_layer_info(layer_info); + const auto single_transfer_size = (HAILO_FORMAT_ORDER_HAILO_NMS == stream_info.format.order) ? + stream_info.nms_info.bbox_size : stream_info.hw_frame_size; + auto boundary_channel_ptr_exp = get_boundary_vdma_channel_by_stream_name(layer_info.name); + CHECK_EXPECTED(boundary_channel_ptr_exp); + auto boundary_channel_ptr = boundary_channel_ptr_exp.release(); + const auto max_batch_transfers = boundary_channel_ptr->get_desc_list()->max_transfers(single_transfer_size * dynamic_batch_size); + // infer batch count is the lowest number of "Max transfers" per descriptor list that for all given boundary channels. + batch_count = MIN(batch_count, max_batch_transfers); + } + return batch_count; +} + +void ResourcesManager::hw_infer_calc_stats(uint16_t batch_count, uint16_t dynamic_batch_size, + size_t single_frame_transfer_size, uint32_t infer_cycles) +{ + const auto total_transfer_size = single_frame_transfer_size * dynamic_batch_size * batch_count; + const auto total_frames = dynamic_batch_size * batch_count; + + // TODO - get clock rate from Chip (still not supported in VPU mode) + const float32_t CPU_CLOCK_RATE = static_cast(5.0 / (1000 * 1000 * 1000)); + const float32_t time_sec = static_cast(infer_cycles) * CPU_CLOCK_RATE; + const float32_t fps = static_cast(total_frames) / time_sec; + const float32_t BYTE_TO_BIT = 8.0; + const float32_t BITS_TO_GBIT = static_cast(1.0 * 1000 * 1000 * 1000); + const float32_t BW_Gbps = static_cast(total_transfer_size) * BYTE_TO_BIT / time_sec / BITS_TO_GBIT; + LOGGER__ERROR("\nBatch count - {}\nTotal transfer size: {}\ntotal_frames - {}\ntime_sec - {}\nfps - {}\nBW_Gbps - {}", + batch_count, total_transfer_size, total_frames, time_sec, fps, BW_Gbps); +} + +Expected ResourcesManager::run_hw_only_infer(uint16_t dynamic_batch_size) +{ + CONTROL_PROTOCOL__hw_only_infer_results_t infer_results = {}; + CONTROL_PROTOCOL__hw_infer_channels_info_t channels_info = {}; + channels_info.channel_count = 0; + + CHECK_AS_EXPECTED(dynamic_batch_size <= m_config_params.batch_size, HAILO_INVALID_ARGUMENT, + "Dynamic batch size must be up to configured batch size"); + + auto batch_count = calc_hw_infer_batch_count(dynamic_batch_size); + CHECK_EXPECTED(batch_count); + + for (const auto &layer_info : m_core_op_metadata->get_all_layer_infos()) { + auto boundary_channel_ptr = get_boundary_vdma_channel_by_stream_name(layer_info.name); + CHECK_EXPECTED(boundary_channel_ptr); + auto stream_info = LayerInfoUtils::get_stream_info_from_layer_info(layer_info); + auto single_transfer_size = (HAILO_FORMAT_ORDER_HAILO_NMS == stream_info.format.order) ? + stream_info.nms_info.bbox_size : stream_info.hw_frame_size; + const auto direction = (layer_info.direction == HAILO_H2D_STREAM) ? + HAILO_VDMA_BUFFER_DIRECTION_FLAGS_H2D : HAILO_VDMA_BUFFER_DIRECTION_FLAGS_D2H; + + auto channel_info_pair = create_mapped_buffer_for_hw_only_infer(boundary_channel_ptr.release(), direction, + single_transfer_size, dynamic_batch_size, batch_count.value()); + CHECK_EXPECTED(channel_info_pair); + + add_channel_to_hw_infer_channel_info(channel_info_pair.release(), channels_info); + } + + auto status = Control::start_hw_only_infer(m_vdma_device, m_core_op_index, dynamic_batch_size, &channels_info); + CHECK_SUCCESS_AS_EXPECTED(status); + + // Delay until infer ends + // TODO HRT-9829 - chagne to notification from FW + std::this_thread::sleep_for(std::chrono::milliseconds(20000)); + + status = Control::stop_hw_only_infer(m_vdma_device, &infer_results); + CHECK_SUCCESS_AS_EXPECTED(status); + + auto single_frame_transfer_size = m_core_op_metadata->get_total_transfer_size(); + CHECK_EXPECTED(single_frame_transfer_size); + + hw_infer_calc_stats(batch_count.value(), dynamic_batch_size, single_frame_transfer_size.release(), infer_results.infer_cycles); + + return infer_results; +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/context_switch/multi_context/resource_manager.hpp b/hailort/libhailort/src/core_op/resource_manager/resource_manager.hpp similarity index 58% rename from hailort/libhailort/src/context_switch/multi_context/resource_manager.hpp rename to hailort/libhailort/src/core_op/resource_manager/resource_manager.hpp index 140592b..9417167 100644 --- a/hailort/libhailort/src/context_switch/multi_context/resource_manager.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/resource_manager.hpp @@ -4,21 +4,21 @@ **/ /** * @file resource_manager.hpp - * @brief Manager for vdma-config network group resources, for a specific physical device + * @brief Manager for vdma-config core-op resources, for a specific physical device * * ResourceManager is used on 2 possible flows with the following dependencies: * * !-Working with physical device-! - * VdmaDevice (either PcieDevice or CoreDevice) - * |--vector of VdmaConfigNetworkGroup + * VdmaDevice (either PcieDevice or IntegratedDevice) + * |--vector of VdmaConfigCoreOp * |--ResourceManager * |--reference to physical device * * !-Working with virtual device-! * VDevice - * |--vector of VdmaDevice (either PcieDevice or CoreDevice) - * |--vector of VDeviceNetworkGroup - * |-- vector of VdmaConfigNetworkGroup + * |--vector of VdmaDevice (either PcieDevice or IntegratedDevice) + * |--vector of VDeviceCoreOp + * |-- vector of VdmaConfigCoreOp * |--ResourceManager * |--reference to physical device **/ @@ -27,14 +27,15 @@ #define _HAILO_CONTEXT_SWITCH_RESOURCE_MANAGER_HPP_ #include "hailo/hailort.h" -#include "inter_context_buffer.hpp" -#include "ddr_channels_pair.hpp" -#include "config_buffer.hpp" -#include "vdma_channel.hpp" -#include "control_protocol.hpp" -#include "pcie_device.hpp" -#include "channel_allocator.hpp" -#include "context_switch/context_switch_buffer_builder.hpp" + +#include "core_op/resource_manager/inter_context_buffer.hpp" +#include "core_op/resource_manager/ddr_channels_pair.hpp" +#include "core_op/resource_manager/config_buffer.hpp" +#include "core_op/resource_manager/channel_allocator.hpp" +#include "core_op/resource_manager/context_switch_buffer_builder.hpp" +#include "device_common/control_protocol.hpp" +#include "vdma/channel/boundary_channel.hpp" +#include "vdma/pcie/pcie_device.hpp" namespace hailort @@ -43,19 +44,7 @@ namespace hailort #define DEFAULT_ACTUAL_BATCH_SIZE (1) -struct BoundaryEdgeLayer { - LayerInfo layer_info; - vdma::ChannelId channel_id; - CONTROL_PROTOCOL__host_buffer_info_t buffer_info; -}; - -struct InterContextEdgeLayer { - LayerInfo layer_info; - vdma::ChannelId channel_id; - CONTROL_PROTOCOL__host_buffer_info_t buffer_info; -}; - -struct DdrChannelEdgeLayer { +struct EdgeLayer { LayerInfo layer_info; vdma::ChannelId channel_id; CONTROL_PROTOCOL__host_buffer_info_t buffer_info; @@ -69,40 +58,26 @@ public: const std::vector &get_controls() const; ContextSwitchBufferBuilder &builder(); - void add_edge_layer(const BoundaryEdgeLayer &edge_layer) - { - m_boundary_layers.emplace_back(std::move(edge_layer)); - } + void add_edge_layer(const LayerInfo &layer_info, vdma::ChannelId channel_id, + const CONTROL_PROTOCOL__host_buffer_info_t &buffer_info); - void add_edge_layer(const InterContextEdgeLayer &edge_layer) - { - m_inter_context_layers.emplace_back(std::move(edge_layer)); - } + std::vector get_edge_layers() const; + std::vector get_edge_layers(LayerType layer_type) const; + std::vector get_edge_layers(hailo_stream_direction_t direction) const; + std::vector get_edge_layers(LayerType layer_type, hailo_stream_direction_t direction) const; - void add_edge_layer(const DdrChannelEdgeLayer &edge_layer) - { - m_ddr_channel_layers.emplace_back(std::move(edge_layer)); - } - - const std::vector &get_boundary_layers() const; - const std::vector &get_inter_context_layers() const; - const std::vector &get_ddr_channel_layers() const; + Expected get_edge_layer_by_stream_index(uint8_t stream_index) const; ExpectedRef create_ddr_channels_pair(const DdrChannelsInfo &ddr_info); ExpectedRef get_ddr_channels_pair(uint8_t d2h_stream_index) const; const std::vector &get_ddr_channels_pairs() const; - // Gets edge layer for a specific direction - std::vector get_boundary_layers(hailo_stream_direction_t direction) const; - std::vector get_inter_context_layers(hailo_stream_direction_t direction) const; - std::vector get_ddr_channel_layers(hailo_stream_direction_t direction) const; - hailo_status validate_edge_layers(); std::vector &get_config_buffers(); private: - explicit ContextResources(HailoRTDriver &driver, CONTROL_PROTOCOL__context_switch_context_type_t context_type, + ContextResources(HailoRTDriver &driver, CONTROL_PROTOCOL__context_switch_context_type_t context_type, std::vector &&config_buffers) : m_driver(std::ref(driver)), m_builder(context_type), @@ -114,18 +89,17 @@ private: std::vector m_config_buffers; std::vector m_ddr_channels_pairs; - std::vector m_boundary_layers; - std::vector m_inter_context_layers; - std::vector m_ddr_channel_layers; + std::vector m_edge_layers; }; class ResourcesManager final { public: static Expected create(VdmaDevice &vdma_device, HailoRTDriver &driver, - const ConfigureNetworkParams &config_params, std::shared_ptr network_group_metadata, - uint8_t net_group_index); + const ConfigureNetworkParams &config_params, std::shared_ptr core_op_metadata, + uint8_t core_op_index); + // TODO: HRT-9432 needs to call stop_vdma_interrupts_dispatcher and any other resource on dtor. ~ResourcesManager() = default; ResourcesManager(const ResourcesManager &other) = delete; ResourcesManager &operator=(const ResourcesManager &other) = delete; @@ -133,18 +107,18 @@ public: ResourcesManager(ResourcesManager &&other) noexcept; ExpectedRef create_inter_context_buffer(uint32_t transfer_size, uint8_t src_stream_index, - uint8_t src_context_index, const std::string &network_name); + uint8_t src_context_index, const std::string &network_name, vdma::ChannelId d2h_channel_id); ExpectedRef get_inter_context_buffer(const IntermediateBufferKey &key); hailo_status create_boundary_vdma_channel(const LayerInfo &layer_info); - Expected get_control_network_group_header(); + Expected get_control_core_op_header(); Expected> add_new_context(CONTROL_PROTOCOL__context_switch_context_type_t type, const ConfigBufferInfoMap &config_info={}); const SupportedFeatures &get_supported_features() const { - return m_network_group_metadata->supported_features(); + return m_core_op_metadata->supported_features(); } VdmaDevice &get_device() @@ -153,7 +127,7 @@ public: } Expected get_available_channel_id(const LayerIdentifier &layer_identifier, - VdmaChannel::Direction direction, uint8_t engine_index); + HailoRTDriver::DmaDirection direction, uint8_t engine_index); hailo_status free_channel_index(const LayerIdentifier &layer_identifier); const char* get_dev_id() const @@ -166,29 +140,44 @@ public: return m_latency_meters; } + std::map get_boundary_vdma_channels() const + { + return m_boundary_channels; + } + Expected get_default_streams_interface(); Expected read_intermediate_buffer(const IntermediateBufferKey &key); - hailo_status create_internal_vdma_channels(); - hailo_status register_fw_managed_vdma_channels(); - hailo_status unregister_fw_managed_vdma_channels(); hailo_status set_inter_context_channels_dynamic_batch_size(uint16_t dynamic_batch_size); - hailo_status open_ddr_channels(); - void abort_ddr_channels(); - void close_ddr_channels(); hailo_status configure(); hailo_status enable_state_machine(uint16_t dynamic_batch_size); hailo_status reset_state_machine(bool keep_nn_config_during_reset = false); + hailo_status cancel_pending_async_transfers(); + hailo_status start_vdma_interrupts_dispatcher(); + hailo_status stop_vdma_interrupts_dispatcher(); Expected get_network_batch_size(const std::string &network_name) const; - Expected> get_boundary_vdma_channel_by_stream_name(const std::string &stream_name); - Expected> get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) const; + Expected get_boundary_vdma_channel_by_stream_name(const std::string &stream_name); + Expected> get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) const; hailo_power_mode_t get_power_mode() const; + Expected program_desc_for_hw_only_flow(std::shared_ptr desc_list, + const uint32_t single_transfer_size, const uint16_t dynamic_batch_size, const uint16_t batch_count); + Expected> create_mapped_buffer_for_hw_only_infer( + vdma::BoundaryChannelPtr boundary_channel_ptr, const hailo_vdma_buffer_direction_flags_t direction, + const uint32_t single_transfer_size, const uint16_t dynamic_batch_size, const uint16_t batch_count); + void add_channel_to_hw_infer_channel_info(std::pair channel_info, + CONTROL_PROTOCOL__hw_infer_channels_info_t &channels_info); + Expected calc_hw_infer_batch_count(uint16_t dynamic_batch_size); + void hw_infer_calc_stats(uint16_t batch_count, uint16_t dynamic_batch_size, + size_t single_frame_transfer_size, uint32_t infer_cycles); + Expected run_hw_only_infer(uint16_t dynamic_batch_size); private: hailo_status fill_infer_features(CONTROL_PROTOCOL__application_header_t &app_header); hailo_status fill_validation_features(CONTROL_PROTOCOL__application_header_t &app_header); hailo_status fill_network_batch_size(CONTROL_PROTOCOL__application_header_t &app_header); + hailo_status fill_csm_buffer_size(CONTROL_PROTOCOL__application_header_t &app_header); + void process_interrupts(IrqData &&irq_data); std::vector m_contexts_resources; ChannelAllocator m_channel_allocator; @@ -196,22 +185,24 @@ private: HailoRTDriver &m_driver; const ConfigureNetworkParams m_config_params; std::map m_inter_context_buffers; - std::vector m_internal_channels; - std::shared_ptr m_network_group_metadata; - uint8_t m_net_group_index; + std::shared_ptr m_core_op_metadata; + uint8_t m_core_op_index; uint8_t m_dynamic_context_count; uint8_t m_total_context_count; const std::vector m_network_index_map; LatencyMetersMap m_latency_meters; // Latency meter per network - std::map> m_boundary_channels; //map of string name and connected vDMA channel + // TODO: HRT-9429 - fast access to channel by id, using array, using engine_index and channel_index. + std::map m_boundary_channels; bool m_is_configured; // Config channels ids are shared between all context. The following vector contains the channel id for each // config_stream_index. std::vector m_config_channels_ids; + // Mapped buffers would be used only in hw only flow + std::vector> m_hw_only_boundary_buffers; ResourcesManager(VdmaDevice &vdma_device, HailoRTDriver &driver, ChannelAllocator &&channel_allocator, const ConfigureNetworkParams config_params, - std::shared_ptr &&network_group_metadata, uint8_t net_group_index, + std::shared_ptr &&core_op_metadata, uint8_t core_op_index, const std::vector &&network_index_map, LatencyMetersMap &&latency_meters, std::vector &&config_channels_ids); }; diff --git a/hailort/libhailort/src/context_switch/resource_manager_builder.cpp b/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.cpp similarity index 72% rename from hailort/libhailort/src/context_switch/resource_manager_builder.cpp rename to hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.cpp index cea5408..0218b10 100644 --- a/hailort/libhailort/src/context_switch/resource_manager_builder.cpp +++ b/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.cpp @@ -8,7 +8,8 @@ **/ #include "resource_manager_builder.hpp" -#include "control.hpp" +#include "device_common/control.hpp" + namespace hailort { @@ -102,19 +103,14 @@ static hailo_status fill_boundary_input_layer(ContextResources &context_resource auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name); CHECK_EXPECTED_AS_STATUS(vdma_channel); - auto buffer_info = vdma_channel.value()->get_boundary_buffer_info(transfer_size); - CHECK_EXPECTED_AS_STATUS(buffer_info); - - auto local_layer_info = update_layer_info(layer_info, *buffer_info, hw_consts, should_optimize_credits); + const auto buffer_info = vdma_channel.value()->get_boundary_buffer_info(transfer_size); + auto local_layer_info = update_layer_info(layer_info, buffer_info, hw_consts, should_optimize_credits); CHECK_EXPECTED_AS_STATUS(local_layer_info); - BoundaryEdgeLayer edge_layer{}; - edge_layer.layer_info = local_layer_info.release(); - edge_layer.channel_id = vdma_channel.value()->get_channel_id(); - edge_layer.buffer_info = buffer_info.value(); - context_resources.add_edge_layer(edge_layer); + const auto channel_id = vdma_channel.value()->get_channel_id(); + context_resources.add_edge_layer(local_layer_info.value(), channel_id, buffer_info); - LOGGER__DEBUG("Boundary input stream: {} h2d_channel: {}.", layer_info.stream_index, edge_layer.channel_id); + LOGGER__DEBUG("Boundary input stream: {} h2d_channel: {}.", layer_info.stream_index, channel_id); return HAILO_SUCCESS; } @@ -123,7 +119,7 @@ static hailo_status fill_inter_context_input_layer(ContextResources &context_res bool should_optimize_credits) { const auto channel_id = resources_manager.get_available_channel_id(to_layer_identifier(layer_info), - VdmaChannel::Direction::H2D, layer_info.dma_engine_index); + HailoRTDriver::DmaDirection::H2D, layer_info.dma_engine_index); CHECK_EXPECTED_AS_STATUS(channel_id); /* Get inter context buffer previously created */ @@ -138,11 +134,8 @@ static hailo_status fill_inter_context_input_layer(ContextResources &context_res should_optimize_credits); CHECK_EXPECTED_AS_STATUS(local_layer_info); - InterContextEdgeLayer edge_layer{}; - edge_layer.layer_info = local_layer_info.release(); - edge_layer.channel_id = channel_id.value(); - edge_layer.buffer_info = inter_context_buffer.get_host_buffer_info(); - context_resources.add_edge_layer(edge_layer); + context_resources.add_edge_layer(local_layer_info.value(), channel_id.value(), + inter_context_buffer.get_host_buffer_info()); LOGGER__DEBUG("Intermediate input stream {}, src_context:{}, dst_context: {}, h2d_channel {}.", layer_info.stream_index, layer_info.context_index, layer_info.connected_context_info.context_index, @@ -161,19 +154,14 @@ static hailo_status fill_boundary_output_layer(ContextResources &context_resourc auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name); CHECK_EXPECTED_AS_STATUS(vdma_channel); - auto buffer_info = vdma_channel.value()->get_boundary_buffer_info(transfer_size); - CHECK_EXPECTED_AS_STATUS(buffer_info); - - auto local_layer_info = update_layer_info(layer_info, *buffer_info, hw_consts, should_optimize_credits); + const auto buffer_info = vdma_channel.value()->get_boundary_buffer_info(transfer_size); + auto local_layer_info = update_layer_info(layer_info, buffer_info, hw_consts, should_optimize_credits); CHECK_EXPECTED_AS_STATUS(local_layer_info); - BoundaryEdgeLayer edge_layer{}; - edge_layer.layer_info = local_layer_info.release(); - edge_layer.channel_id = vdma_channel.value()->get_channel_id(); - edge_layer.buffer_info = buffer_info.value(); - context_resources.add_edge_layer(edge_layer); + const auto channel_id = vdma_channel.value()->get_channel_id(); + context_resources.add_edge_layer(local_layer_info.value(), channel_id, buffer_info); - LOGGER__DEBUG("Boundary output stream: {} d2h_channel: {}.", layer_info.stream_index, edge_layer.channel_id); + LOGGER__DEBUG("Boundary output stream: {} d2h_channel: {}.", layer_info.stream_index, channel_id); return HAILO_SUCCESS; } @@ -182,14 +170,14 @@ static hailo_status fill_inter_context_output_layer(ContextResources &context_re const CONTROL_PROTOCOL__hw_consts_t &hw_consts, bool should_optimize_credits) { const auto channel_id = resources_manager.get_available_channel_id(to_layer_identifier(layer_info), - VdmaChannel::Direction::D2H, layer_info.dma_engine_index); + HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index); CHECK_EXPECTED_AS_STATUS(channel_id); const auto frame_credits_in_bytes = (layer_info.nn_stream_config.periph_bytes_per_buffer * layer_info.nn_stream_config.core_buffers_per_frame); auto inter_context_buffer_exp = resources_manager.create_inter_context_buffer(frame_credits_in_bytes, - layer_info.stream_index, layer_info.context_index, layer_info.network_name); + layer_info.stream_index, layer_info.context_index, layer_info.network_name, channel_id.value()); CHECK_EXPECTED_AS_STATUS(inter_context_buffer_exp); auto &inter_context_buffer = inter_context_buffer_exp->get(); @@ -197,11 +185,8 @@ static hailo_status fill_inter_context_output_layer(ContextResources &context_re should_optimize_credits); CHECK_EXPECTED_AS_STATUS(local_layer_info); - InterContextEdgeLayer edge_layer{}; - edge_layer.layer_info = local_layer_info.release(); - edge_layer.channel_id = channel_id.value(); - edge_layer.buffer_info = inter_context_buffer.get_host_buffer_info(); - context_resources.add_edge_layer(edge_layer); + context_resources.add_edge_layer(local_layer_info.value(), channel_id.value(), + inter_context_buffer.get_host_buffer_info()); LOGGER__DEBUG("Inter-context output stream {}, src_context:{}, d2h_channel {}.", layer_info.stream_index, layer_info.context_index, channel_id.value()); @@ -226,13 +211,13 @@ static hailo_status fill_ddr_output_layer(ContextResources &context_resources, // Allocate vdma channel index for both edges const auto h2d_layer_identifier = std::make_tuple(LayerType::DDR, layer_info.name, ddr_pair_info.h2d_stream_index); const auto h2d_channel_id = resources_manager.get_available_channel_id(h2d_layer_identifier, - VdmaChannel::Direction::H2D, layer_info.connected_context_info.dma_engine_index); + HailoRTDriver::DmaDirection::H2D, layer_info.connected_context_info.dma_engine_index); CHECK_EXPECTED_AS_STATUS(h2d_channel_id); ddr_pair_info.h2d_channel_id = h2d_channel_id.value(); const auto d2h_layer_identifier = std::make_tuple(LayerType::DDR, layer_info.name, ddr_pair_info.d2h_stream_index); const auto d2h_channel_id = resources_manager.get_available_channel_id(d2h_layer_identifier, - VdmaChannel::Direction::D2H, layer_info.dma_engine_index); + HailoRTDriver::DmaDirection::D2H, layer_info.dma_engine_index); CHECK_EXPECTED_AS_STATUS(d2h_channel_id); ddr_pair_info.d2h_channel_id = d2h_channel_id.value(); @@ -251,11 +236,8 @@ static hailo_status fill_ddr_output_layer(ContextResources &context_resources, should_optimize_credits); CHECK_EXPECTED_AS_STATUS(local_layer_info); - DdrChannelEdgeLayer edge_layer{}; - edge_layer.layer_info = local_layer_info.release(); - edge_layer.channel_id = ddr_pair_info.d2h_channel_id; - edge_layer.buffer_info = ddr_channels_pair->get().get_host_buffer_info(); - context_resources.add_edge_layer(edge_layer); + context_resources.add_edge_layer(local_layer_info.value(), ddr_pair_info.d2h_channel_id, + ddr_channels_pair->get().get_host_buffer_info()); return HAILO_SUCCESS; } @@ -283,11 +265,8 @@ static hailo_status fill_ddr_input_layer(ContextResources &context_resources, should_optimize_credits); CHECK_EXPECTED_AS_STATUS(local_layer_info); - DdrChannelEdgeLayer edge_layer{}; - edge_layer.layer_info = local_layer_info.release(); - edge_layer.channel_id = ddr_info.h2d_channel_id; - edge_layer.buffer_info = ddr_channels_pair->get().get_host_buffer_info(); - context_resources.add_edge_layer(edge_layer); + context_resources.add_edge_layer(local_layer_info.value(), ddr_channels_pair->get().info().h2d_channel_id, + ddr_channels_pair->get().get_host_buffer_info()); return HAILO_SUCCESS; } @@ -496,43 +475,22 @@ static std::set> get_indexes_of_action_type( return result; } -static std::set get_end_indexes_of_action_type( - const std::vector &actions, - const std::vector> &repeated_indexes, - const ContextSwitchConfigAction::Type &required_action_type) -{ - std::set result; - for (const auto &index_pair : get_indexes_of_action_type(actions, repeated_indexes, required_action_type)) { - result.insert(index_pair.second); - } - - return result; -} - static hailo_status push_fetch_config_actions( - std::vector &config_resources, const std::set &pending_config_stream_indexes, - std::vector &total_ccw_bursts, const bool support_pre_fetch, + ConfigBuffer &config_resources, uint8_t config_stream_index, + uint16_t total_ccw_bursts, bool support_pre_fetch, std::vector &processed_configuration_actions) { - CHECK(total_ccw_bursts.size() == config_resources.size(), HAILO_INTERNAL_FAILURE, "Invalid cfg channels count"); - for (const auto config_stream_index : pending_config_stream_indexes) { - CHECK(config_stream_index < config_resources.size(), HAILO_INTERNAL_FAILURE, "Invalid cfg channel index"); - - if (support_pre_fetch) { - auto action = AddCcwBurstAction::create(config_stream_index, total_ccw_bursts[config_stream_index]); - CHECK_EXPECTED_AS_STATUS(action); - processed_configuration_actions.emplace_back(action.release()); - } else { - const auto desc_count = config_resources[config_stream_index].program_descriptors(); - CHECK_EXPECTED_AS_STATUS(desc_count); - CHECK(IS_FIT_IN_UINT16(desc_count.value()), HAILO_INVALID_OPERATION, - "On cfg with continuous mode, max descriptors size must fit in uint16_t"); - - auto action = FetchCfgChannelDescriptorsAction::create(config_resources[config_stream_index].channel_id(), - static_cast(desc_count.value())); - CHECK_EXPECTED_AS_STATUS(action); - processed_configuration_actions.emplace_back(action.release()); - } + if (support_pre_fetch) { + auto action = AddCcwBurstAction::create(config_stream_index, total_ccw_bursts); + CHECK_EXPECTED_AS_STATUS(action); + processed_configuration_actions.emplace_back(action.release()); + } else { + const auto desc_count = config_resources.program_descriptors(); + CHECK_EXPECTED_AS_STATUS(desc_count); + + auto action = FetchCfgChannelDescriptorsAction::create(config_resources.channel_id(), desc_count.value()); + CHECK_EXPECTED_AS_STATUS(action); + processed_configuration_actions.emplace_back(action.release()); } return HAILO_SUCCESS; @@ -559,82 +517,39 @@ static hailo_status write_ccw_to_buffer(ConfigBuffer& config_buffer, const Write } static hailo_status proccess_write_ccw_action(const ContextSwitchConfigActionPtr &configuration_action, - std::vector &config_resources, std::set &pending_config_stream_indexes, - std::vector &total_ccw_bursts, const std::set &end_indexes_of_write_ccw_actions, - const uint32_t &action_index, const bool support_pre_fetch, + std::vector &config_resources, + const bool support_pre_fetch, std::vector &processed_configuration_actions) { assert(ContextSwitchConfigAction::Type::WriteDataCcw == configuration_action->get_type()); const auto &write_ccw_action = *static_cast(configuration_action.get()); - // Add the config stream index of the current WriteDataCcwAction const auto config_stream_index = write_ccw_action.config_stream_index(); - pending_config_stream_indexes.insert(config_stream_index); - - // TODO: get CCW headers from proto (need to add it into the proto) - const uint16_t ccw_bursts = 1; - auto accum_ccw_bursts = total_ccw_bursts[config_stream_index] + ccw_bursts; - CHECK(IS_FIT_IN_UINT16(accum_ccw_bursts), HAILO_INTERNAL_FAILURE, - "Failed to parse HEF. action fetch ccw burst supports only to 2^16 bursts."); - assert(config_stream_index < total_ccw_bursts.size()); - total_ccw_bursts[config_stream_index] = static_cast(accum_ccw_bursts); - assert(config_stream_index < config_resources.size()); auto status = write_ccw_to_buffer(config_resources[config_stream_index], write_ccw_action, support_pre_fetch); CHECK_SUCCESS(status); - // At the end of a consecutive group of WriteDataCcwActions, we program the - // descriptors for all the config channels used. - if (contains(end_indexes_of_write_ccw_actions, action_index)) { - // Add the last CCW write into the buffer - processed_configuration_actions.emplace_back(configuration_action); - - status = push_fetch_config_actions(config_resources, pending_config_stream_indexes, total_ccw_bursts, - support_pre_fetch, processed_configuration_actions); - CHECK_SUCCESS(status); - - // Cleanups - pending_config_stream_indexes.clear(); - for (uint8_t cleanup_ch_index = 0; cleanup_ch_index < total_ccw_bursts.size(); cleanup_ch_index++) { - total_ccw_bursts[cleanup_ch_index] = 0; - } - } else { - // Add the current action - processed_configuration_actions.emplace_back(configuration_action); - } + status = push_fetch_config_actions(config_resources[config_stream_index], config_stream_index, + write_ccw_action.total_ccw_burst(), support_pre_fetch, processed_configuration_actions); + CHECK_SUCCESS(status); return HAILO_SUCCESS; } static Expected find_dummy_stream(const LayerInfo &layer_info, const ContextResources &context_resources) { - // TODO: HRT-8611 use one loop for all edge layers - for (const auto &edge_layer : context_resources.get_boundary_layers()) { - if (edge_layer.layer_info.direction != layer_info.direction) { - return Expected(edge_layer.layer_info.stream_index); - } - } - for (const auto &edge_layer : context_resources.get_inter_context_layers()) { - if (edge_layer.layer_info.direction != layer_info.direction) { - return Expected(edge_layer.layer_info.stream_index); - } - } - for (const auto &edge_layer : context_resources.get_ddr_channel_layers()) { - if (edge_layer.layer_info.direction != layer_info.direction) { - return Expected(edge_layer.layer_info.stream_index); - } - } - - LOGGER__ERROR("Couldn't find dummy stream from context"); - return make_unexpected(HAILO_INTERNAL_FAILURE); + const auto other_direction = (HAILO_H2D_STREAM == layer_info.direction) ? HAILO_D2H_STREAM : HAILO_H2D_STREAM; + const auto other_direction_edge_layers = context_resources.get_edge_layers(other_direction); + CHECK_AS_EXPECTED(!other_direction_edge_layers.empty(), HAILO_INTERNAL_FAILURE, "Couldn't find dummy stream"); + return Expected(other_direction_edge_layers.front().layer_info.stream_index); } static hailo_status add_change_vdma_to_stream_mapping( - const NetworkGroupMetadata &network_group_metadata, const ResourcesManager &resources_manager, + const CoreOpMetadata &core_op_metadata, const ResourcesManager &resources_manager, ContextResources &context_resources, uint8_t context_index, std::vector &processed_configuration_actions) { - for (const auto &layer_info : network_group_metadata.get_all_layer_infos()) { + for (const auto &layer_info : core_op_metadata.get_all_layer_infos()) { auto vdma_channel = resources_manager.get_boundary_vdma_channel_by_stream_name(layer_info.name); CHECK_EXPECTED_AS_STATUS(vdma_channel); @@ -663,7 +578,7 @@ static hailo_status push_edge_layer_activation_actions( // In order to insure that input data can enter the chip only after all other elements are configured. // We parse ddr inputs before boundary/inter-context because otherwise on C2C mode we may lose some credit. - for (const auto &edge_layer : context_resources.get_ddr_channel_layers(HAILO_D2H_STREAM)) { + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::DDR, HAILO_D2H_STREAM)) { auto activate_action = ActivateDdrOutputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info, edge_layer.layer_info.ddr_info.min_buffered_rows); @@ -671,14 +586,14 @@ static hailo_status push_edge_layer_activation_actions( actions.emplace_back(activate_action.release()); } - for (const auto &edge_layer : context_resources.get_boundary_layers(HAILO_D2H_STREAM)) { + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_D2H_STREAM)) { auto activate_action = ActivateBoundaryOutputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info); CHECK_EXPECTED_AS_STATUS(activate_action); actions.emplace_back(activate_action.release()); } - for (const auto &edge_layer : context_resources.get_inter_context_layers(HAILO_D2H_STREAM)) { + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::INTER_CONTEXT, HAILO_D2H_STREAM)) { auto activate_action = ActivateInterContextOutputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.network_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info); @@ -686,7 +601,7 @@ static hailo_status push_edge_layer_activation_actions( actions.emplace_back(activate_action.release()); } - for (const auto &edge_layer : context_resources.get_ddr_channel_layers(HAILO_H2D_STREAM)) { + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::DDR, HAILO_H2D_STREAM)) { const auto d2h_stream_index = edge_layer.layer_info.connected_context_info.stream_index; auto pair = context_resources.get_ddr_channels_pair(d2h_stream_index); CHECK_EXPECTED_AS_STATUS(pair); @@ -699,7 +614,7 @@ static hailo_status push_edge_layer_activation_actions( actions.emplace_back(activate_action.release()); } - for (const auto &edge_layer : context_resources.get_boundary_layers(HAILO_H2D_STREAM)) { + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY, HAILO_H2D_STREAM)) { auto activate_action = ActivateBoundaryInputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info, edge_layer.layer_info.max_shmifo_size); @@ -707,7 +622,7 @@ static hailo_status push_edge_layer_activation_actions( actions.emplace_back(activate_action.release()); } - for (const auto &edge_layer : context_resources.get_inter_context_layers(HAILO_H2D_STREAM)) { + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::INTER_CONTEXT, HAILO_H2D_STREAM)) { auto activate_action = ActivateInterContextInputChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.stream_index, edge_layer.layer_info.nn_stream_config, edge_layer.buffer_info, edge_layer.layer_info.max_shmifo_size); @@ -722,7 +637,7 @@ static hailo_status proccess_trigger_new_data_input_action(const ContextSwitchCo uint32_t trigger_new_data_from_input_group_start, uint32_t trigger_new_data_from_input_group_end, const uint32_t &action_index, - const NetworkGroupMetadata &network_group_metadata, + const CoreOpMetadata &core_op_metadata, const ResourcesManager &resources_manager, ContextResources &context_resources, uint8_t context_index, @@ -733,7 +648,7 @@ static hailo_status proccess_trigger_new_data_input_action(const ContextSwitchCo CHECK_SUCCESS(status); if (!is_single_context) { - status = add_change_vdma_to_stream_mapping(network_group_metadata, resources_manager, + status = add_change_vdma_to_stream_mapping(core_op_metadata, resources_manager, context_resources, context_index, processed_configuration_actions); CHECK_SUCCESS(status); } @@ -761,18 +676,13 @@ static hailo_status proccess_trigger_new_data_input_action(const ContextSwitchCo static hailo_status add_fetch_config_actions(std::vector &configuration_actions, std::vector &config_resources, bool support_pre_fetch) { - const auto repeated_indexes = get_repreated_actions_boundary_indices(configuration_actions); - const auto end_indexes_of_write_ccws = get_end_indexes_of_action_type(configuration_actions, - repeated_indexes, ContextSwitchConfigAction::Type::WriteDataCcw); - std::set pending_config_stream_indexes; - std::vector total_ccw_bursts(config_resources.size(), 0); std::vector processed_configuration_actions; for (uint32_t action_index = 0; action_index < configuration_actions.size(); action_index++) { const auto &configuration_action = configuration_actions[action_index]; if (ContextSwitchConfigAction::Type::WriteDataCcw == configuration_action->get_type()) { - auto status = proccess_write_ccw_action(configuration_action, config_resources, pending_config_stream_indexes, - total_ccw_bursts, end_indexes_of_write_ccws, action_index, support_pre_fetch, processed_configuration_actions); + auto status = proccess_write_ccw_action(configuration_action, config_resources, + support_pre_fetch, processed_configuration_actions); CHECK_SUCCESS(status); } else { // Add the current action @@ -825,19 +735,10 @@ static hailo_status add_config_channel_activation_actions(std::vector &configuration_actions, - const NetworkGroupMetadata &network_group_metadata, + const CoreOpMetadata &core_op_metadata, const ResourcesManager &resources_manager, ContextResources &context_resources, uint8_t context_index, - bool is_preliminary_context, bool is_first_operation, bool is_single_context) + bool is_single_context) { - if (is_preliminary_context && !resources_manager.get_supported_features().preliminary_run_asap) { - // Nothing to do - no edge layers in the preliminary context if not running in preliminary_run_asap mode. - return HAILO_SUCCESS; - } - if (!is_preliminary_context && !is_first_operation) { - // Nothing to do - edge layers in dynamic contexts only appear in the first operation. - return HAILO_SUCCESS; - } - const auto repeated_indexes = get_repreated_actions_boundary_indices(configuration_actions); const auto trigger_new_data_from_input_group_indexes = get_indexes_of_action_type( configuration_actions, repeated_indexes, ContextSwitchConfigAction::Type::TriggerNewDataFromDataInput); @@ -852,7 +753,7 @@ static hailo_status handle_edge_layer_activation_actions(std::vectorget_type()) { auto status = proccess_trigger_new_data_input_action(configuration_action, trigger_new_data_from_input_group_start, trigger_new_data_from_input_group_end, action_index, - network_group_metadata, resources_manager, context_resources, context_index, processed_configuration_actions, is_single_context); + core_op_metadata, resources_manager, context_resources, context_index, processed_configuration_actions, is_single_context); CHECK_SUCCESS(status); } else { // Add the current action @@ -908,36 +809,11 @@ static hailo_status handle_repeated_actions(std::vector> process_operation(const ContextSwitchOperation &operation, - const NetworkGroupMetadata &network_group_metadata, - const ProtoHEFHwArch &hw_arch, uint8_t context_index, bool is_preliminary_context, - bool is_first_operation, bool is_single_context, std::vector &config_resources, - ResourcesManager &resources_manager, - ContextResources &context_resources) -{ - auto configuration_actions = operation.actions(); - - // Next, we process the actions from the HEF. The resulting vector contains the configuration actions to be - // executed in chronological order. - const auto support_pre_fetch = is_mercury_device_type(hw_arch); - auto status = add_fetch_config_actions(configuration_actions, config_resources, support_pre_fetch); - CHECK_SUCCESS_AS_EXPECTED(status); - - status = handle_edge_layer_activation_actions(configuration_actions, network_group_metadata, resources_manager, - context_resources, context_index, is_preliminary_context, is_first_operation, is_single_context); - CHECK_SUCCESS_AS_EXPECTED(status); - - status = handle_repeated_actions(configuration_actions); - CHECK_SUCCESS_AS_EXPECTED(status); - - return configuration_actions; + // Compare with HW_ARCH__LAVENDER and HW_ARCH__GINGER to support hefs compiled for them + return (PROTO__HW_ARCH__GINGER == hw_arch) || (PROTO__HW_ARCH__LAVENDER == hw_arch) || + (PROTO__HW_ARCH__HAILO15H == hw_arch); } static hailo_status write_action_list(const ContextResources & context_resources, ContextSwitchBufferBuilder &builder, @@ -958,33 +834,13 @@ static hailo_status write_action_list(const ContextResources & context_resources static hailo_status add_edge_layer_end_of_context_actions(const ContextResources &context_resources, std::vector &actions) { - - for (const auto &edge_layer : context_resources.get_boundary_layers()) { - const bool is_inter_context = false; - // Validate boundary channels of current context. - auto validate_action = ValidateChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.direction, - is_inter_context, static_cast(edge_layer.buffer_info.buffer_type), - edge_layer.layer_info.max_shmifo_size); - CHECK_EXPECTED_AS_STATUS(validate_action); - actions.push_back(validate_action.release()); - } - - for (const auto &edge_layer : context_resources.get_inter_context_layers()) { - const bool is_inter_context = true; - auto deactivate_action = DeactivateChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.direction, - is_inter_context, static_cast(edge_layer.buffer_info.buffer_type), - edge_layer.layer_info.max_shmifo_size); - CHECK_EXPECTED_AS_STATUS(deactivate_action); - actions.push_back(deactivate_action.release()); - } - - for (const auto &edge_layer : context_resources.get_ddr_channel_layers()) { - const bool is_inter_context = false; - auto deactivate_action = DeactivateChannelAction::create(edge_layer.channel_id, edge_layer.layer_info.direction, - is_inter_context, static_cast(edge_layer.buffer_info.buffer_type), - edge_layer.layer_info.max_shmifo_size); - CHECK_EXPECTED_AS_STATUS(deactivate_action); - actions.push_back(deactivate_action.release()); + for (const auto &edge_layer : context_resources.get_edge_layers()) { + const bool should_validate = (edge_layer.layer_info.type == LayerType::BOUNDARY); + auto action = should_validate ? + ValidateChannelAction::create(edge_layer) : + DeactivateChannelAction::create(edge_layer); + CHECK_EXPECTED_AS_STATUS(action); + actions.emplace_back(action.release()); } return HAILO_SUCCESS; @@ -992,7 +848,7 @@ static hailo_status add_edge_layer_end_of_context_actions(const ContextResources static hailo_status fill_context_recipes_for_multi_context(const ProtoHEFHwArch &hw_arch, ContextResources &context_resources, ResourcesManager &resources_manager, - uint8_t context_index, const NetworkGroupMetadata &network_group_metadata, const ContextMetadata &context_metadata, + uint8_t context_index, const CoreOpMetadata &core_op_metadata, const ContextMetadata &context_metadata, bool is_single_context) { hailo_status status = HAILO_UNINITIALIZED; @@ -1002,18 +858,15 @@ static hailo_status fill_context_recipes_for_multi_context(const ProtoHEFHwArch CHECK_SUCCESS(status); // Parse context - bool first_operation = true; - std::vector actions; - for (const auto &operation : context_metadata.get_operations()) { - static const auto NOT_PRELIMINARY_CONTEXT = false; - auto new_actions = process_operation(operation, network_group_metadata, hw_arch, context_index, NOT_PRELIMINARY_CONTEXT, - first_operation, is_single_context, context_resources.get_config_buffers(), resources_manager, - context_resources); - CHECK_EXPECTED_AS_STATUS(new_actions); - - actions.insert(actions.end(), new_actions->begin(), new_actions->end()); - first_operation = false; - } + std::vector actions = context_metadata.get_actions(); + + const auto support_pre_fetch = is_hailo15_device_type(hw_arch); + status = add_fetch_config_actions(actions, context_resources.get_config_buffers(), support_pre_fetch); + CHECK_SUCCESS(status); + + status = handle_edge_layer_activation_actions(actions, core_op_metadata, resources_manager, + context_resources, context_index, is_single_context); + CHECK_SUCCESS(status); status = add_config_channel_activation_actions(actions, context_resources.get_config_buffers()); CHECK_SUCCESS(status); @@ -1028,13 +881,16 @@ static hailo_status fill_context_recipes_for_multi_context(const ProtoHEFHwArch status = add_edge_layer_end_of_context_actions(context_resources, actions); } + status = handle_repeated_actions(actions); + CHECK_SUCCESS(status); + return write_action_list(context_resources, context_resources.builder(), actions); } static hailo_status create_boundary_channels(ResourcesManager &resources_manager, - NetworkGroupMetadata &network_group_metadata) + CoreOpMetadata &core_op_metadata) { - for (const auto &layer_info : network_group_metadata.get_all_layer_infos()) { + for (const auto &layer_info : core_op_metadata.get_all_layer_infos()) { auto status = resources_manager.create_boundary_vdma_channel(layer_info); CHECK_SUCCESS(status); } @@ -1043,20 +899,20 @@ static hailo_status create_boundary_channels(ResourcesManager &resources_manager static hailo_status fill_activation_config_recepies_for_multi_context( ContextResources &context_resources, ResourcesManager &resources_manager, - std::shared_ptr network_group_metadata) + std::shared_ptr core_op_metadata) { auto hw_consts = Control::get_hw_consts(resources_manager.get_device()); CHECK_EXPECTED_AS_STATUS(hw_consts); const bool should_optimize_credits = hw_consts->should_optimize_credits && (HAILO_POWER_MODE_PERFORMANCE == resources_manager.get_power_mode()); - for (const auto &layer_info : network_group_metadata->get_output_layer_infos()){ + for (const auto &layer_info : core_op_metadata->get_output_layer_infos()){ auto status = fill_boundary_output_layer(context_resources, resources_manager, layer_info, *hw_consts, should_optimize_credits); CHECK_SUCCESS(status); } - for (const auto &layer_info : network_group_metadata->get_input_layer_infos()) { + for (const auto &layer_info : core_op_metadata->get_input_layer_infos()) { auto status = fill_boundary_input_layer(context_resources, resources_manager, layer_info, *hw_consts, should_optimize_credits); CHECK_SUCCESS(status); @@ -1066,7 +922,7 @@ static hailo_status fill_activation_config_recepies_for_multi_context( CHECK_SUCCESS(status); std::vector actions; - for (const auto &edge_layer : context_resources.get_boundary_layers()) { + for (const auto &edge_layer : context_resources.get_edge_layers(LayerType::BOUNDARY)) { auto action = edge_layer.layer_info.direction == HAILO_H2D_STREAM ? OpenBoundaryInputChannelAction::create(edge_layer.channel_id, edge_layer.buffer_info) : OpenBoundaryOutputChannelAction::create(edge_layer.channel_id, edge_layer.buffer_info); @@ -1078,7 +934,7 @@ static hailo_status fill_activation_config_recepies_for_multi_context( } static hailo_status fill_batch_switching_context_config_recepies_for_multi_context( - ContextResources &context_resources, const NetworkGroupMetadata &network_group_metadata) + ContextResources &context_resources, const CoreOpMetadata &core_op_metadata) { std::vector actions; @@ -1093,50 +949,47 @@ static hailo_status fill_batch_switching_context_config_recepies_for_multi_conte ContextSwitchConfigAction::Type::EnableLcuDefault, ContextSwitchConfigAction::Type::EnableLcuNonDefault }; - for (const auto &operation : network_group_metadata.preliminary_context().get_operations()) { - auto operation_actions = operation.get_actions_of_type(BATCH_SWITCHING_ACTIONS); + const auto batch_switch_actions = core_op_metadata.preliminary_context().get_actions_of_type(BATCH_SWITCHING_ACTIONS); + actions.insert(actions.end(), batch_switch_actions.begin(), batch_switch_actions.end()); - // Allowing repeated actions - auto status = handle_repeated_actions(operation_actions); - CHECK_SUCCESS(status); - - actions.insert(actions.end(), operation_actions.begin(), operation_actions.end()); - } + auto status = handle_repeated_actions(actions); + CHECK_SUCCESS(status); return write_action_list(context_resources, context_resources.builder(), actions); } static hailo_status fill_preliminary_config_recepies_for_multi_context(const ProtoHEFHwArch &hw_arch, ContextResources &context_resources, ResourcesManager &resources_manager, - std::shared_ptr network_group_metadata, const PreliminaryContextMetadata &preliminary_context, + std::shared_ptr core_op_metadata, const ContextMetadata &preliminary_context, bool is_single_context) { + static const auto PRELIMINARY_CONTEXT_INDEX = 0; // First context in the hef if (resources_manager.get_supported_features().preliminary_run_asap) { // Add edge layers mapping (only preliminary_run_asap networks have edge layers in the preliminary context) - static const auto PRELIMINARY_CONTEXT_INDEX = 0; - assert(PRELIMINARY_CONTEXT_INDEX < network_group_metadata->dynamic_contexts().size()); + assert(PRELIMINARY_CONTEXT_INDEX < core_op_metadata->dynamic_contexts().size()); auto status = parse_and_fill_edge_layers_mapping(context_resources, - network_group_metadata->dynamic_contexts()[PRELIMINARY_CONTEXT_INDEX], resources_manager); + core_op_metadata->dynamic_contexts()[PRELIMINARY_CONTEXT_INDEX], resources_manager); CHECK_SUCCESS(status); } // Parse preliminary config - std::vector actions; - bool first_operation = true; - for (const auto &operation : preliminary_context.get_operations()) { - static const auto PRELIMINARY_CONTEXT_INDEX = 0; // First context in the hef - static const auto PRELIMINARY_CONTEXT = true; - auto new_actions = process_operation(operation, *network_group_metadata, hw_arch, PRELIMINARY_CONTEXT_INDEX, - PRELIMINARY_CONTEXT, first_operation, is_single_context, context_resources.get_config_buffers(), resources_manager, - context_resources); - CHECK_EXPECTED_AS_STATUS(new_actions); - - actions.insert(actions.end(), new_actions->begin(), new_actions->end()); - first_operation = false; + std::vector actions = preliminary_context.get_actions(); + + const auto support_pre_fetch = is_hailo15_device_type(hw_arch); + auto status = add_fetch_config_actions(actions, context_resources.get_config_buffers(), support_pre_fetch); + CHECK_SUCCESS(status); + + if (resources_manager.get_supported_features().preliminary_run_asap) { + status = handle_edge_layer_activation_actions(actions, *core_op_metadata, resources_manager, + context_resources, PRELIMINARY_CONTEXT_INDEX, is_single_context); + CHECK_SUCCESS(status); } - auto status = add_config_channel_activation_actions(actions, context_resources.get_config_buffers()); + status = add_config_channel_activation_actions(actions, context_resources.get_config_buffers()); + CHECK_SUCCESS(status); + + status = handle_repeated_actions(actions); CHECK_SUCCESS(status); return write_action_list(context_resources, context_resources.builder(), actions); @@ -1144,67 +997,67 @@ static hailo_status fill_preliminary_config_recepies_for_multi_context(const Pro -Expected> ResourcesManagerBuilder::build(uint8_t net_group_index, VdmaDevice &device, +Expected> ResourcesManagerBuilder::build(uint8_t current_core_op_index, VdmaDevice &device, HailoRTDriver &driver, const ConfigureNetworkParams &config_params, - std::shared_ptr network_group_metadata, const ProtoHEFHwArch &hw_arch) + std::shared_ptr core_op_metadata, const ProtoHEFHwArch &hw_arch) { - const auto num_contexts = network_group_metadata->dynamic_contexts().size() + + const auto num_contexts = core_op_metadata->dynamic_contexts().size() + CONTROL_PROTOCOL__CONTEXT_SWITCH_NUMBER_OF_NON_DYNAMIC_CONTEXTS; CHECK_AS_EXPECTED(CONTROL_PROTOCOL__MAX_CONTEXTS_PER_NETWORK_GROUP >= num_contexts, HAILO_INVALID_HEF, "App '{}' contains more contexts than allowed ({} > {})", - network_group_metadata->network_group_name(), num_contexts, CONTROL_PROTOCOL__MAX_CONTEXTS_PER_NETWORK_GROUP); + core_op_metadata->core_op_name(), num_contexts, CONTROL_PROTOCOL__MAX_CONTEXTS_PER_NETWORK_GROUP); for (auto &network_params : config_params.network_params_by_name) { CHECK(HAILO_MAX_BATCH_SIZE >= network_params.second.batch_size, make_unexpected(HAILO_INVALID_ARGUMENT), "Given batch size ({}) for network group {}, network {} is bigger than max allowed ({})", network_params.second.batch_size, - network_group_metadata->network_group_name(), network_params.first, HAILO_MAX_BATCH_SIZE); + core_op_metadata->core_op_name(), network_params.first, HAILO_MAX_BATCH_SIZE); } - auto resources_manager = ResourcesManager::create(device, driver, config_params, network_group_metadata, - net_group_index); + auto resources_manager = ResourcesManager::create(device, driver, config_params, core_op_metadata, + current_core_op_index); CHECK_EXPECTED(resources_manager); - auto status = create_boundary_channels(resources_manager.value(), *network_group_metadata); + // TODO: Use a new flag in config_params.stream_params_by_name to mark channels as async channels. + // will also used to mark streams as async in ConfiguredNetworkGroupBase::create_in/output_stream_from_config_params + // (HRT-9104) + auto status = create_boundary_channels(resources_manager.value(), *core_op_metadata); CHECK_SUCCESS_AS_EXPECTED(status); auto activation_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_ACTIVATION); CHECK_EXPECTED(activation_context); status = fill_activation_config_recepies_for_multi_context(activation_context.value().get(), - resources_manager.value(), network_group_metadata); + resources_manager.value(), core_op_metadata); CHECK_SUCCESS_AS_EXPECTED(status); auto batch_switching_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_BATCH_SWITCHING); CHECK_EXPECTED(batch_switching_context); status = fill_batch_switching_context_config_recepies_for_multi_context(batch_switching_context.value().get(), - *network_group_metadata); + *core_op_metadata); CHECK_SUCCESS_AS_EXPECTED(status); - const bool is_single_context = network_group_metadata->dynamic_contexts().size() == 1; + const bool is_single_context = core_op_metadata->dynamic_contexts().size() == 1; auto preliminary_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_PRELIMINARY, - network_group_metadata->preliminary_context().config_buffers_info()); + core_op_metadata->preliminary_context().config_buffers_info()); CHECK_EXPECTED(preliminary_context); status = fill_preliminary_config_recepies_for_multi_context(hw_arch, preliminary_context.value().get(), - resources_manager.value(), network_group_metadata, network_group_metadata->preliminary_context(), is_single_context); + resources_manager.value(), core_op_metadata, core_op_metadata->preliminary_context(), is_single_context); CHECK_SUCCESS_AS_EXPECTED(status); uint8_t context_index = 0; - for (const auto &context_metadata : network_group_metadata->dynamic_contexts()) { + for (const auto &context_metadata : core_op_metadata->dynamic_contexts()) { auto new_context = resources_manager->add_new_context(CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC, context_metadata.config_buffers_info()); CHECK_EXPECTED(new_context); status = fill_context_recipes_for_multi_context(hw_arch, new_context.value().get(), resources_manager.value(), - context_index, *network_group_metadata, + context_index, *core_op_metadata, context_metadata, is_single_context); CHECK_SUCCESS_AS_EXPECTED(status); context_index++; } - status = resources_manager->create_internal_vdma_channels(); - CHECK_SUCCESS_AS_EXPECTED(status); - status = resources_manager->configure(); CHECK_SUCCESS_AS_EXPECTED(status); diff --git a/hailort/libhailort/src/context_switch/resource_manager_builder.hpp b/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.hpp similarity index 81% rename from hailort/libhailort/src/context_switch/resource_manager_builder.hpp rename to hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.hpp index 4ce2f26..e596cb0 100644 --- a/hailort/libhailort/src/context_switch/resource_manager_builder.hpp +++ b/hailort/libhailort/src/core_op/resource_manager/resource_manager_builder.hpp @@ -10,8 +10,8 @@ #ifndef _HAILO_RESOURCE_MANAGER_BUILDER_HPP_ #define _HAILO_RESOURCE_MANAGER_BUILDER_HPP_ -#include "hef_internal.hpp" -#include "context_switch/multi_context/resource_manager.hpp" +#include "hef/hef_internal.hpp" +#include "core_op/resource_manager/resource_manager.hpp" namespace hailort @@ -24,7 +24,7 @@ public: /* TODO HRT-5067 - work with hailo_device_architecture_t instead of ProtoHEFHwArch */ static Expected> build(uint8_t net_group_index, VdmaDevice &device, HailoRTDriver &driver, const ConfigureNetworkParams &config_params, - std::shared_ptr network_group_metadata, const ProtoHEFHwArch &hw_arch); + std::shared_ptr core_op, const ProtoHEFHwArch &hw_arch); }; diff --git a/hailort/libhailort/src/core_stream.cpp b/hailort/libhailort/src/core_stream.cpp deleted file mode 100644 index 6ba66fe..0000000 --- a/hailort/libhailort/src/core_stream.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file core_stream.cpp - **/ - -#include "core_stream.hpp" -#include "control.hpp" - -namespace hailort -{ - -Expected> CoreInputStream::create(Device &device, - std::shared_ptr channel, const LayerInfo &edge_layer, - uint16_t batch_size, EventPtr network_group_activated_event) -{ - hailo_status status = HAILO_UNINITIALIZED; - - CHECK_AS_EXPECTED(device.get_type() == Device::Type::CORE, HAILO_INTERNAL_FAILURE, - "Invalid device type"); - - CoreDevice *core_device = reinterpret_cast(&device); - std::unique_ptr local_stream(new (std::nothrow) CoreInputStream(*core_device, - std::move(channel), edge_layer, std::move(network_group_activated_event), batch_size, - DEFAULT_TRANSFER_TIMEOUT, status)); - CHECK((nullptr != local_stream), make_unexpected(HAILO_OUT_OF_HOST_MEMORY)); - CHECK_SUCCESS_AS_EXPECTED(status); - - return local_stream; -} - -CoreInputStream::CoreInputStream( - CoreDevice &device, - std::shared_ptr channel, - const LayerInfo &edge_layer, - EventPtr network_group_activated_event, - uint16_t batch_size, - const std::chrono::milliseconds &transfer_timeout, - hailo_status &status) : - VdmaInputStream(device, std::move(channel), edge_layer, network_group_activated_event, - batch_size, transfer_timeout, HAILO_STREAM_INTERFACE_CORE, status) -{} - -Expected> CoreOutputStream::create(Device &device, - std::shared_ptr channel, const LayerInfo &edge_layer, - uint16_t batch_size, EventPtr network_group_activated_event) -{ - hailo_status status = HAILO_UNINITIALIZED; - CHECK_AS_EXPECTED(device.get_type() == Device::Type::CORE, HAILO_INTERNAL_FAILURE, - "Invalid device type"); - - CoreDevice *core_device = reinterpret_cast(&device); - std::unique_ptr local_stream(new (std::nothrow) CoreOutputStream(*core_device, - std::move(channel), edge_layer, std::move(network_group_activated_event), - batch_size, DEFAULT_TRANSFER_TIMEOUT, status)); - CHECK((nullptr != local_stream), make_unexpected(HAILO_OUT_OF_HOST_MEMORY)); - CHECK_SUCCESS_AS_EXPECTED(status); - - return local_stream; -} - -CoreOutputStream::CoreOutputStream( - CoreDevice &device, - std::shared_ptr channel, - const LayerInfo &edge_layer, - EventPtr network_group_activated_event, - uint16_t batch_size, - const std::chrono::milliseconds &transfer_timeout, - hailo_status &status) : - VdmaOutputStream(device, std::move(channel), edge_layer, - network_group_activated_event, batch_size, transfer_timeout, status) -{} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/core_stream.hpp b/hailort/libhailort/src/core_stream.hpp deleted file mode 100644 index dceef1c..0000000 --- a/hailort/libhailort/src/core_stream.hpp +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file core_stream.hpp - * @brief Stream object for Core device - **/ - -#ifndef _HAILO_CORE_STREAM_HPP_ -#define _HAILO_CORE_STREAM_HPP_ - -#include "vdma_stream.hpp" -#include "core_device.hpp" - - -namespace hailort -{ - -class CoreInputStream : public VdmaInputStream { -public: - CoreInputStream(CoreInputStream &&other) = default; - virtual ~CoreInputStream() = default; - - static Expected> create(Device &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event); - - virtual hailo_stream_interface_t get_interface() const override { return HAILO_STREAM_INTERFACE_CORE; } - -private: - CoreInputStream( - CoreDevice &device, - std::shared_ptr channel, - const LayerInfo &edge_layer, - EventPtr network_group_activated_event, - uint16_t batch_size, - const std::chrono::milliseconds &transfer_timeout, - hailo_status &status); -}; - -class CoreOutputStream : public VdmaOutputStream { -public: - CoreOutputStream(CoreOutputStream &&other) = default; - virtual ~CoreOutputStream() = default; - - static Expected> create(Device &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event); - - virtual hailo_stream_interface_t get_interface() const override { return HAILO_STREAM_INTERFACE_CORE; } - -private: - explicit CoreOutputStream( - CoreDevice &device, - std::shared_ptr channel, - const LayerInfo &edge_layer, - EventPtr network_group_activated_event, - uint16_t batch_size, - const std::chrono::milliseconds &transfer_timeout, - hailo_status &status); -}; - -} /* namespace hailort */ - -#endif /* _HAILO_CORE_STREAM_HPP_ */ diff --git a/hailort/libhailort/src/device_common/CMakeLists.txt b/hailort/libhailort/src/device_common/CMakeLists.txt new file mode 100644 index 0000000..eeb3318 --- /dev/null +++ b/hailort/libhailort/src/device_common/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/device_internal.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/d2h_events_parser.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/control.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/control_protocol.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/control.cpp b/hailort/libhailort/src/device_common/control.cpp similarity index 92% rename from hailort/libhailort/src/control.cpp rename to hailort/libhailort/src/device_common/control.cpp index 23a3da5..19946b9 100644 --- a/hailort/libhailort/src/control.cpp +++ b/hailort/libhailort/src/device_common/control.cpp @@ -8,16 +8,19 @@ **/ #include "common/utils.hpp" -#include "control.hpp" #include "common/logger_macros.hpp" + +#include "hef/hef_internal.hpp" +#include "device_common/control.hpp" +#include "hw_consts.hpp" + #include "control_protocol.h" #include "byte_order.h" #include "firmware_status.h" #include "firmware_header_utils.h" #include "d2h_events.h" -#include "hw_consts.hpp" #include -#include "hef_internal.hpp" + namespace hailort { @@ -194,6 +197,18 @@ hailo_status control__parse_core_identify_results(CONTROL_PROTOCOL__core_identif return HAILO_SUCCESS; } +hailo_status Control::validate_arch_supported(Device &device, const std::vector &supported_archs) +{ + auto dev_arch = device.get_architecture(); + CHECK_EXPECTED_AS_STATUS(dev_arch); + for (const auto &arch : supported_archs) { + if (*dev_arch == arch) { + return HAILO_SUCCESS; + } + } + LOGGER__ERROR("Control is not supported for this device architecture - {}", HailoRTCommon::get_device_arch_str(*dev_arch)); + return HAILO_NOT_SUPPORTED; +} hailo_status Control::parse_and_validate_response(uint8_t *message, uint32_t message_size, CONTROL_PROTOCOL__response_header_t **header, CONTROL_PROTOCOL__payload_t **payload, @@ -344,10 +359,14 @@ hailo_status Control::set_fw_logger(Device &device, hailo_fw_logger_level_t leve CONTROL_PROTOCOL__request_t request = {}; size_t request_size = 0; + /* Validate arch */ + auto status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + auto common_status = CONTROL_PROTOCOL__pack_set_fw_logger_request(&request, &request_size, device.get_control_sequence(), level, static_cast(interface_mask)); - auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS(status); uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; @@ -370,9 +389,13 @@ hailo_status Control::set_clock_freq(Device &device, uint32_t clock_freq) CONTROL_PROTOCOL__request_t request = {}; size_t request_size = 0; + /* Validate arch */ + auto status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + auto common_status = CONTROL_PROTOCOL__pack_set_clock_freq_request(&request, &request_size, device.get_control_sequence(), clock_freq); - auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS(status); uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; @@ -395,9 +418,13 @@ hailo_status Control::set_throttling_state(Device &device, bool should_activate) CONTROL_PROTOCOL__request_t request = {}; size_t request_size = 0; + /* Validate arch */ + auto status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + auto common_status = CONTROL_PROTOCOL__pack_set_throttling_state_request(&request, &request_size, device.get_control_sequence(), should_activate); - auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS(status); uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; @@ -427,6 +454,10 @@ Expected Control::get_throttling_state(Device &device) CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__get_throttling_state_response_t *get_throttling_state_response = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS_AS_EXPECTED(status); + common_status = CONTROL_PROTOCOL__pack_get_throttling_state_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -448,9 +479,13 @@ hailo_status Control::set_overcurrent_state(Device &device, bool should_activate CONTROL_PROTOCOL__request_t request = {}; size_t request_size = 0; + /* Validate arch */ + auto status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + auto common_status = CONTROL_PROTOCOL__pack_set_overcurrent_state_request(&request, &request_size, device.get_control_sequence(), should_activate); - auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS(status); uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; @@ -479,6 +514,10 @@ Expected Control::get_overcurrent_state(Device &device) CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__get_overcurrent_state_response_t *get_overcurrent_state_response = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS_AS_EXPECTED(status); + common_status = CONTROL_PROTOCOL__pack_get_overcurrent_state_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -528,7 +567,6 @@ hailo_status Control::write_memory_chunk(Device &device, uint32_t address, const CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; - /* Validate arguments */ ASSERT(NULL != data); @@ -605,11 +643,9 @@ hailo_status Control::read_memory_chunk(Device &device, uint32_t address, uint8_ CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__read_memory_response_t *read_memory_response = NULL; - /* Validate arguments */ ASSERT(NULL != data); - /* Validate chunk size is valid */ ASSERT(CONTROL__MAX_WRITE_MEMORY_CHUNK_SIZE >= chunk_size); ASSERT(0 != chunk_size); @@ -690,6 +726,12 @@ hailo_status Control::open_stream(Device &device, uint8_t dataflow_manager_id, b CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_open_stream_request(&request, &request_size, device.get_control_sequence(), dataflow_manager_id, is_input); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -725,6 +767,12 @@ hailo_status Control::close_stream(Device &device, uint8_t dataflow_manager_id, CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_close_stream_request(&request, &request_size, device.get_control_sequence(), dataflow_manager_id, is_input); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -786,6 +834,12 @@ hailo_status Control::config_stream_udp_input(Device &device, CONTROL_PROTOCOL__ /* Validate arguments */ CHECK_ARG_NOT_NULL(params); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_config_stream_udp_input_request(&request, &request_size, device.get_control_sequence(), params); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -837,6 +891,12 @@ hailo_status Control::config_stream_udp_output(Device &device, CONTROL_PROTOCOL_ /* Validate arguments */ CHECK_ARG_NOT_NULL(params); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_config_stream_udp_output_request(&request, &request_size, device.get_control_sequence(), params); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -888,6 +948,12 @@ hailo_status Control::config_stream_mipi_input(Device &device, CONTROL_PROTOCOL_ /* Validate arguments */ CHECK_ARG_NOT_NULL(params); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_config_stream_mipi_input_request(&request, &request_size, device.get_control_sequence(), params); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -939,6 +1005,12 @@ hailo_status Control::config_stream_mipi_output(Device &device, CONTROL_PROTOCOL /* Validate arguments */ CHECK_ARG_NOT_NULL(params); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_config_stream_mipi_output_request(&request, &request_size, device.get_control_sequence(), params); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -990,6 +1062,12 @@ hailo_status Control::config_stream_pcie_input(Device &device, CONTROL_PROTOCOL_ /* Validate arguments */ CHECK_ARG_NOT_NULL(params); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_config_stream_pcie_input_request(&request, &request_size, device.get_control_sequence(), params); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -1041,6 +1119,12 @@ hailo_status Control::config_stream_pcie_output(Device &device, CONTROL_PROTOCOL /* Validate arguments */ CHECK_ARG_NOT_NULL(params); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_config_stream_pcie_output_request(&request, &request_size, device.get_control_sequence(), params); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -1091,6 +1175,12 @@ hailo_status Control::power_measurement(Device &device, CONTROL_PROTOCOL__dvm_op CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__power_measurement_response_t *response = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + /* Validate arguments */ CHECK_ARG_NOT_NULL(measurement); @@ -1140,6 +1230,12 @@ hailo_status Control::set_power_measurement(Device &device, hailo_measurement_bu CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__set_power_measurement_response_t *response = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + CHECK(CONTROL_PROTOCOL__MAX_NUMBER_OF_POWER_MEASUREMETS > buffer_index, HAILO_INVALID_ARGUMENT, "Invalid power measurement index {}", buffer_index); @@ -1187,6 +1283,12 @@ hailo_status Control::get_power_measurement(Device &device, hailo_measurement_bu CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__get_power_measurement_response_t *get_power_response = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + /* Validate arguments */ CHECK(CONTROL_PROTOCOL__MAX_NUMBER_OF_POWER_MEASUREMETS > buffer_index, HAILO_INVALID_ARGUMENT, "Invalid power measurement index {}", buffer_index); @@ -1235,8 +1337,15 @@ hailo_status Control::start_power_measurement(Device &device, size_t response_size = RESPONSE_MAX_BUFFER_SIZE; CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + uint32_t delay_milliseconds = 0; + + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } - uint32_t delay_milliseconds = POWER_MEASUREMENT_DELAY_MS(sampling_period, averaging_factor); + delay_milliseconds = POWER_MEASUREMENT_DELAY_MS(sampling_period, averaging_factor); // There is no logical way that measurement delay can be 0 - because sampling_period and averaging_factor cant be 0 // Hence if it is 0 - it means it was 0.xx and we want to round up to 1 in that case if (0 == delay_milliseconds) { @@ -1278,6 +1387,12 @@ hailo_status Control::stop_power_measurement(Device &device) CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_stop_power_measurement_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -1317,6 +1432,12 @@ hailo_status Control::i2c_write(Device &device, const hailo_i2c_slave_config_t * CHECK_ARG_NOT_NULL(slave_config); CHECK_ARG_NOT_NULL(data); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + /* Pack request */ common_status = CONTROL_PROTOCOL__pack_i2c_write_request(&request, &request_size, device.get_control_sequence(), register_address, static_cast(slave_config->endianness), @@ -1362,6 +1483,12 @@ hailo_status Control::i2c_read(Device &device, const hailo_i2c_slave_config_t *s CHECK_ARG_NOT_NULL(slave_config); CHECK_ARG_NOT_NULL(data); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + /* Pack request */ common_status = CONTROL_PROTOCOL__pack_i2c_read_request(&request, &request_size, device.get_control_sequence(), register_address, static_cast(slave_config->endianness), @@ -1419,6 +1546,12 @@ hailo_status Control::config_core_top(Device &device, CONTROL_PROTOCOL__config_c /* Validate arguments */ CHECK_ARG_NOT_NULL(params); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_config_core_top_request(&request, &request_size, device.get_control_sequence(), config_type, params); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -1453,6 +1586,12 @@ hailo_status Control::phy_operation(Device &device, CONTROL_PROTOCOL__phy_operat CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_phy_operation_request(&request, &request_size, device.get_control_sequence(), operation_type); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -1491,6 +1630,12 @@ hailo_status Control::examine_user_config(Device &device, hailo_fw_user_config_i /* Validate arguments */ CHECK_ARG_NOT_NULL(info); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_examine_user_config(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -1533,6 +1678,10 @@ hailo_status Control::read_user_config_chunk(Device &device, uint32_t read_offse CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__read_user_config_response_t *response = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + common_status = CONTROL_PROTOCOL__pack_read_user_config(&request, &request_size, device.get_control_sequence(), read_offset, read_length); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -1564,6 +1713,10 @@ hailo_status Control::read_user_config(Device &device, uint8_t *buffer, uint32_t /* Validate arguments */ CHECK_ARG_NOT_NULL(buffer); + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + status = examine_user_config(device, &user_config_info); CHECK_SUCCESS(status); @@ -1594,6 +1747,10 @@ hailo_status Control::write_user_config_chunk(Device &device, uint32_t offset, c CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + common_status = CONTROL_PROTOCOL__pack_write_user_config_request(&request, &request_size, device.get_control_sequence(), offset, data + offset, chunk_size); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -1620,6 +1777,10 @@ hailo_status Control::write_user_config(Device &device, const uint8_t *data, uin /* Validate arguments */ CHECK_ARG_NOT_NULL(data); + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + while (offset < data_length) { chunk_size = MIN(WRITE_CHUNK_SIZE, (data_length - offset)); status = write_user_config_chunk(device, offset, data, chunk_size); @@ -1641,6 +1802,12 @@ hailo_status Control::erase_user_config(Device &device) CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_erase_user_config_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -1682,6 +1849,10 @@ hailo_status Control::read_board_config(Device &device, uint8_t *buffer, uint32_ /* Validate arguments */ CHECK_ARG_NOT_NULL(buffer); + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + CHECK(buffer_length >= BOARD_CONFIG_SIZE, HAILO_INSUFFICIENT_BUFFER, "read buffer is too small. provided buffer size: {} bytes, board config size: {} bytes", buffer_length, BOARD_CONFIG_SIZE); @@ -1725,6 +1896,10 @@ hailo_status Control::write_board_config(Device &device, const uint8_t *data, ui /* Validate arguments */ CHECK_ARG_NOT_NULL(data); + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + CHECK(BOARD_CONFIG_SIZE >= data_length, HAILO_INVALID_OPERATION, "Invalid size of board config. data_length={}, max_size={}" , data_length, BOARD_CONFIG_SIZE); @@ -1759,6 +1934,12 @@ hailo_status Control::write_second_stage_to_internal_memory(Device &device, uint /* Validate arguments */ CHECK_ARG_NOT_NULL(data); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__write_second_stage_to_internal_memory_request(&request, &request_size, device.get_control_sequence(), offset, data, data_length); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -1798,6 +1979,12 @@ hailo_status Control::copy_second_stage_to_flash(Device &device, MD5_SUM_t *expe /* Validate arguments */ CHECK_ARG_NOT_NULL(expected_md5); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__copy_second_stage_to_flash_request(&request, &request_size, device.get_control_sequence(), expected_md5, second_stage_size); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -1832,6 +2019,12 @@ hailo_status Control::start_firmware_update(Device &device) CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_start_firmware_update_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -1866,6 +2059,12 @@ hailo_status Control::finish_firmware_update(Device &device) CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_finish_firmware_update_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -1903,6 +2102,12 @@ hailo_status Control::write_firmware_update(Device &device, uint32_t offset, con /* Validate arguments */ CHECK_ARG_NOT_NULL(data); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__write_firmware_update_request(&request, &request_size, device.get_control_sequence(), offset, data, data_length); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -1941,6 +2146,12 @@ hailo_status Control::validate_firmware_update(Device &device, MD5_SUM_t *expect /* Validate arguments */ CHECK_ARG_NOT_NULL(expected_md5); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_validate_firmware_update_request(&request, &request_size, device.get_control_sequence(), expected_md5, firmware_size); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -1980,6 +2191,12 @@ hailo_status Control::latency_measurement_read(Device &device, uint32_t *inbound /* Validate arguments */ CHECK_ARG_NOT_NULL(inbound_to_outbound_latency_nsec); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_latency_measurement_read_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -2019,6 +2236,12 @@ hailo_status Control::latency_measurement_config(Device &device, uint8_t latency CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_latency_measurement_config_request(&request, &request_size, device.get_control_sequence(), latency_measurement_en, inbound_start_buffer_number, outbound_stop_buffer_number, inbound_stream_index, outbound_stream_index); @@ -2059,11 +2282,15 @@ hailo_status Control::sensor_store_config(Device &device, uint32_t is_first, uin CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; - /* Validate arguments */ CHECK_ARG_NOT_NULL(data); CHECK_ARG_NOT_NULL(config_name); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } common_status = CONTROL_PROTOCOL__pack_sensor_store_config_request(&request, &request_size, device.get_control_sequence(), is_first, section_index, start_offset, reset_data_size, sensor_type, total_data_size, data, data_length, config_height, @@ -2101,6 +2328,10 @@ hailo_status Control::sensor_set_i2c_bus_index(Device &device, uint32_t sensor_t CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + status = CONTROL_PROTOCOL__pack_sensor_set_i2c_bus_index_request(&request, &request_size, device.get_control_sequence(), sensor_type, bus_index); CHECK_SUCCESS(status); @@ -2125,6 +2356,12 @@ hailo_status Control::sensor_load_and_start_config(Device &device, uint32_t sect CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_sensor_load_and_start_config_request(&request, &request_size, device.get_control_sequence(), section_index); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -2159,6 +2396,12 @@ hailo_status Control::sensor_reset(Device &device, uint32_t section_index) CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_sensor_reset_request(&request, &request_size, device.get_control_sequence(), section_index); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -2194,6 +2437,12 @@ hailo_status Control::sensor_set_generic_i2c_slave(Device &device, uint16_t slav CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_sensor_set_generic_i2c_slave_request(&request, &request_size, device.get_control_sequence(), slave_address, register_address_size, bus_index, should_hold_bus, endianness); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -2235,6 +2484,12 @@ hailo_status Control::sensor_get_config(Device &device, uint32_t section_index, /* Validate arguments */ CHECK_ARG_NOT_NULL(data); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_sensor_get_config_request(&request, &request_size, device.get_control_sequence(), section_index, offset, data_length); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -2284,6 +2539,12 @@ hailo_status Control::sensor_get_sections_info(Device &device, uint8_t *data) /* Validate arguments */ CHECK_ARG_NOT_NULL(data); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_sensor_get_sections_info_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -2418,6 +2679,12 @@ hailo_status Control::idle_time_get_measurement(Device &device, uint64_t *measur /* Validate arguments */ CHECK_ARG_NOT_NULL(measurement); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_idle_time_get_measuremment_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -2465,7 +2732,11 @@ hailo_status Control::idle_time_set_measurement(Device &device, uint8_t measurem CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; - LOGGER__DEBUG("Sending idle_time_set_measurement with parameter {}", measurement_enable); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } common_status = CONTROL_PROTOCOL__pack_idle_time_set_measuremment_request(&request, &request_size, device.get_control_sequence(), measurement_enable); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -2493,14 +2764,16 @@ exit: hailo_status Control::set_pause_frames(Device &device, uint8_t rx_pause_frames_enable) { - - LOGGER__DEBUG("Sending set_pause_frames with parameter {}", rx_pause_frames_enable); - CONTROL_PROTOCOL__request_t request = {}; size_t request_size = 0; + + /* Validate arch */ + auto status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + HAILO_COMMON_STATUS_t common_status = CONTROL_PROTOCOL__pack_set_pause_frames_request(&request, &request_size, device.get_control_sequence(), rx_pause_frames_enable); - hailo_status status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS(status); uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; @@ -2668,7 +2941,7 @@ exit: return status; } -hailo_status Control::enable_network_group(Device &device, uint8_t network_group_index, uint16_t dynamic_batch_size) +hailo_status Control::enable_core_op(Device &device, uint8_t network_group_index, uint16_t dynamic_batch_size) { static const auto REMOVE_NN_CONFIG_DURING_RESET = false; return Control::change_context_switch_status(device, CONTROL_PROTOCOL__CONTEXT_SWITCH_STATUS_ENABLED, @@ -2694,7 +2967,11 @@ hailo_status Control::wd_enable(Device &device, uint8_t cpu_id, bool should_enab CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; - LOGGER__DEBUG("Sending wd_enable with parameters cpu_id: {}, should_enable: {}", cpu_id, should_enable); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } common_status = CONTROL_PROTOCOL__pack_wd_enable(&request, &request_size, device.get_control_sequence(), cpu_id, should_enable); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -2731,7 +3008,11 @@ hailo_status Control::wd_config(Device &device, uint8_t cpu_id, uint32_t wd_cycl CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; - LOGGER__DEBUG("Sending wd_config with parameters cpu_id: {}, wd_cycles: {} wd_mode {}", cpu_id, wd_cycles, wd_mode); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } common_status = CONTROL_PROTOCOL__pack_wd_config(&request, &request_size, device.get_control_sequence(), cpu_id, wd_cycles, wd_mode); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -2770,10 +3051,14 @@ hailo_status Control::previous_system_state(Device &device, uint8_t cpu_id, CONT CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__previous_system_state_response_t *previous_system_state_response = NULL; - LOGGER__DEBUG("Sending previous system state with parameter cpu_id: {}", cpu_id); - CHECK_ARG_NOT_NULL(system); + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_previous_system_state(&request, &request_size, device.get_control_sequence(), cpu_id); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -2850,11 +3135,9 @@ hailo_status Control::d2h_notification_manager_set_host_info(Device &device, uin size_t response_size = sizeof(response_buffer); CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; - auto connection_type = ((Device::Type::PCIE == device.get_type() || Device::Type::CORE == device.get_type()) ? + auto connection_type = ((Device::Type::PCIE == device.get_type() || Device::Type::INTEGRATED == device.get_type()) ? D2H_EVENT_COMMUNICATION_TYPE_VDMA : D2H_EVENT_COMMUNICATION_TYPE_UDP); - LOGGER__DEBUG("Set d2h notification manager new host info : connection_type {}, remote_port {}, remote_ip_address {}", connection_type, host_port, host_ip_address); - common_status = CONTROL_PROTOCOL__pack_d2h_event_manager_set_host_info_request(&request, &request_size, device.get_control_sequence(), static_cast(connection_type), host_port, host_ip_address); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -2890,8 +3173,6 @@ hailo_status Control::d2h_notification_manager_send_host_info_notification(Devic CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; - LOGGER__DEBUG("Send d2h host notification with priority {}", notification_priority); - common_status = CONTROL_PROTOCOL__pack_d2h_event_manager_send_host_info_event_request(&request, &request_size, device.get_control_sequence(), notification_priority); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -2927,8 +3208,6 @@ hailo_status Control::clear_configured_apps(Device &device) CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; - LOGGER__DEBUG("Sending clear_configured_apps"); - common_status = CONTROL_PROTOCOL__pack_context_switch_clear_configured_apps_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; @@ -2967,6 +3246,12 @@ hailo_status Control::get_chip_temperature(Device &device, hailo_chip_temperatur CONTROL_PROTOCOL__payload_t *payload = NULL; CONTROL_PROTOCOL__get_chip_temperature_response_t* temps = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_get_chip_temperature_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -3006,6 +3291,12 @@ hailo_status Control::enable_debugging(Device &device, bool is_rma) CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + status = Control::validate_arch_supported(device); + if (HAILO_SUCCESS != status) { + goto exit; + } + common_status = CONTROL_PROTOCOL__pack_enable_debugging_request(&request, &request_size, device.get_control_sequence(), is_rma); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; if (HAILO_SUCCESS != status) { @@ -3042,6 +3333,10 @@ Expected Control:: /* Validate arguments */ + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS_AS_EXPECTED(status); + common_status = CONTROL_PROTOCOL__pack_get_extended_device_information_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS_AS_EXPECTED(status); @@ -3090,6 +3385,10 @@ Expected Control::get_health_information(Device &device) /* Validate arguments */ + /* Validate arch */ + status = Control::validate_arch_supported(device); + CHECK_SUCCESS_AS_EXPECTED(status); + common_status = CONTROL_PROTOCOL__pack_get_health_information_request(&request, &request_size, device.get_control_sequence()); status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS_AS_EXPECTED(status); @@ -3299,10 +3598,14 @@ hailo_status Control::run_bist_test(Device &device, bool is_top_test, uint32_t t CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + auto status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + auto common_status = CONTROL_PROTOCOL__pack_run_bist_test_request( &request, &request_size, device.get_control_sequence(), is_top_test, top_bypass_bitmap, cluster_index, cluster_bypass_bitmap_0, cluster_bypass_bitmap_1); - auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS(status); status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size); @@ -3325,8 +3628,43 @@ hailo_status Control::set_sleep_state(Device &device, hailo_sleep_state_t sleep_ CONTROL_PROTOCOL__response_header_t *header = NULL; CONTROL_PROTOCOL__payload_t *payload = NULL; + /* Validate arch */ + auto status = Control::validate_arch_supported(device); + CHECK_SUCCESS(status); + auto common_status = CONTROL_PROTOCOL__pack_set_sleep_state_request( &request, &request_size, device.get_control_sequence(), static_cast(sleep_state)); + status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; + CHECK_SUCCESS(status); + + status = device.fw_interact((uint8_t*)(&request), request_size, (uint8_t*)&response_buffer, &response_size); + CHECK_SUCCESS(status); + + /* Parse response */ + status = parse_and_validate_response(response_buffer, (uint32_t)(response_size), &header, &payload, + &request); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +hailo_status Control::change_hw_infer_status(Device &device, CONTROL_PROTOCOL__hw_infer_state_t state, + uint8_t network_group_index, uint16_t dynamic_batch_size, + CONTROL_PROTOCOL__hw_infer_channels_info_t *channels_info, CONTROL_PROTOCOL__hw_only_infer_results_t *results) +{ + CONTROL_PROTOCOL__request_t request = {}; + size_t request_size = 0; + uint8_t response_buffer[RESPONSE_MAX_BUFFER_SIZE] = {}; + size_t response_size = sizeof(response_buffer); + CONTROL_PROTOCOL__response_header_t *header = NULL; + CONTROL_PROTOCOL__payload_t *payload = NULL; + CONTROL_PROTOCOL__change_hw_infer_status_response_t *change_hw_infer_status_response = NULL; + + RETURN_IF_ARG_NULL(results); + + auto common_status = CONTROL_PROTOCOL__pack_change_hw_infer_status_request( + &request, &request_size, device.get_control_sequence(), static_cast(state), + network_group_index, dynamic_batch_size, channels_info); auto status = (HAILO_COMMON_STATUS__SUCCESS == common_status) ? HAILO_SUCCESS : HAILO_INTERNAL_FAILURE; CHECK_SUCCESS(status); @@ -3338,7 +3676,28 @@ hailo_status Control::set_sleep_state(Device &device, hailo_sleep_state_t sleep_ &request); CHECK_SUCCESS(status); + change_hw_infer_status_response = (CONTROL_PROTOCOL__change_hw_infer_status_response_t *)(payload->parameters); + + memcpy(results, &(change_hw_infer_status_response->results), sizeof((*results))); + return HAILO_SUCCESS; } +hailo_status Control::start_hw_only_infer(Device &device, uint8_t network_group_index, uint16_t dynamic_batch_size, + CONTROL_PROTOCOL__hw_infer_channels_info_t *channels_info) +{ + CONTROL_PROTOCOL__hw_only_infer_results_t results = {}; + return Control::change_hw_infer_status(device, CONTROL_PROTOCOL__HW_INFER_STATE_START, + network_group_index, dynamic_batch_size, channels_info ,&results); +} + +hailo_status Control::stop_hw_only_infer(Device &device, CONTROL_PROTOCOL__hw_only_infer_results_t *results) +{ + const uint8_t DEFAULT_NETWORK_GROUP = 0; + const uint16_t DEFAULT_DYNAMIC_BATCH_SIZE = 1; + CONTROL_PROTOCOL__hw_infer_channels_info_t channels_info_default = {}; + return Control::change_hw_infer_status(device, CONTROL_PROTOCOL__HW_INFER_STATE_STOP, + DEFAULT_NETWORK_GROUP, DEFAULT_DYNAMIC_BATCH_SIZE, &channels_info_default, results); +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/control.hpp b/hailort/libhailort/src/device_common/control.hpp similarity index 95% rename from hailort/libhailort/src/control.hpp rename to hailort/libhailort/src/device_common/control.hpp index dd768af..d79ad85 100644 --- a/hailort/libhailort/src/control.hpp +++ b/hailort/libhailort/src/device_common/control.hpp @@ -7,15 +7,17 @@ * @brief Contains Defines and declarations related to controlling hailo8 **/ -#ifndef __CONTROL_H__ -#define __CONTROL_H__ +#ifndef __CONTROL_HPP__ +#define __CONTROL_HPP__ -#include - -#include +#include "hailo/hailort.h" #include "hailo/device.hpp" + +#include "device_common/control_protocol.hpp" + #include "control_protocol.h" -#include "control_protocol.hpp" +#include + namespace hailort { @@ -283,14 +285,14 @@ public: uint32_t *batch_counter); /** - * Enable network group + * Enable core-op * * @param[in] device - The Hailo device. - * @param[in] network_group_index - network_group index + * @param[in] core_op_index - core_op index * * @return Upon success, returns @a HAILO_SUCCESS. Otherwise, returns an @a static hailo_status error. */ - static hailo_status enable_network_group(Device &device, uint8_t network_group_index, uint16_t dynamic_batch_size); + static hailo_status enable_core_op(Device &device, uint8_t core_op_index, uint16_t dynamic_batch_size); /** * reset context switch state machine * @@ -370,7 +372,12 @@ public: static Expected get_overcurrent_state(Device &device); static Expected get_hw_consts(Device &device); static hailo_status set_sleep_state(Device &device, hailo_sleep_state_t sleep_state); - + static hailo_status change_hw_infer_status(Device &device, CONTROL_PROTOCOL__hw_infer_state_t state, + uint8_t network_group_index, uint16_t dynamic_batch_size, + CONTROL_PROTOCOL__hw_infer_channels_info_t *channels_info, CONTROL_PROTOCOL__hw_only_infer_results_t *results); + static hailo_status start_hw_only_infer(Device &device, uint8_t network_group_index, uint16_t dynamic_batch_size, + CONTROL_PROTOCOL__hw_infer_channels_info_t *channels_info); + static hailo_status stop_hw_only_infer(Device &device, CONTROL_PROTOCOL__hw_only_infer_results_t *results); // TODO: needed? static hailo_status power_measurement(Device &device, CONTROL_PROTOCOL__dvm_options_t dvm, CONTROL_PROTOCOL__power_measurement_types_t measurement_type, float32_t *measurement); @@ -400,8 +407,9 @@ private: CONTROL_PROTOCOL__CONTEXT_SWITCH_STATUS_t state_machine_status, uint8_t network_group_index, uint16_t dynamic_batch_size, bool keep_nn_config_during_reset); static Expected get_extended_device_info_response(Device &device); + static hailo_status validate_arch_supported(Device &device, const std::vector &supported_archs = { HAILO_ARCH_HAILO8, HAILO_ARCH_HAILO8L }); }; } /* namespace hailort */ -#endif /* __CONTROL_H__ */ +#endif /* __CONTROL_HPP__ */ diff --git a/hailort/libhailort/src/control_protocol.cpp b/hailort/libhailort/src/device_common/control_protocol.cpp similarity index 97% rename from hailort/libhailort/src/control_protocol.cpp rename to hailort/libhailort/src/device_common/control_protocol.cpp index 6d35440..92412bb 100644 --- a/hailort/libhailort/src/control_protocol.cpp +++ b/hailort/libhailort/src/device_common/control_protocol.cpp @@ -17,13 +17,16 @@ * *=============================================================================*/ -#include -#include +#include "common/utils.hpp" + +#include "device_common/control_protocol.hpp" + #include "control_protocol.h" -#include "control_protocol.hpp" #include "byte_order.h" #include "status.h" -#include "common/utils.hpp" +#include +#include + using namespace hailort; @@ -54,6 +57,7 @@ const char *CONTROL_PROTOCOL__get_textual_opcode(CONTROL_PROTOCOL__OPCODE_t opco return CONTROL_PROTOCOL__textual_format[opcode]; } +#define CHANGE_HW_INFER_REQUEST_PARAMETER_COUNT (4) /* Functions declarations */ HAILO_COMMON_STATUS_t control_protocol__parse_message(uint8_t *message, @@ -1672,7 +1676,7 @@ HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_set_context_info_req /* Header */ local_request_size = CONTROL_PROTOCOL__REQUEST_BASE_SIZE + sizeof(CONTROL_PROTOCOL__context_switch_set_context_info_request_t) + context_info->context_network_data_length; - control_protocol__pack_request_header(request, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_SET_CONTEXT_INFO, 5); + control_protocol__pack_request_header(request, sequence, HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_SET_CONTEXT_INFO, 4); /* is_first_control_per_context */ request->parameters.context_switch_set_context_info_request.is_first_control_per_context_length = @@ -1692,12 +1696,6 @@ HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_context_switch_set_context_info_req request->parameters.context_switch_set_context_info_request.context_type = context_info->context_type; - /* actions_count */ - request->parameters.context_switch_set_context_info_request.actions_count_length = - BYTE_ORDER__htonl(sizeof(request->parameters.context_switch_set_context_info_request.actions_count)); - request->parameters.context_switch_set_context_info_request.actions_count = - BYTE_ORDER__htonl(context_info->actions_count); - /* Network data (edge layers + Trigger groups) */ if (CONTROL_PROTOCOL__CONTEXT_NETWORK_DATA_SINGLE_CONTROL_MAX_SIZE < context_info->context_network_data_length) { status = HAILO_STATUS__CONTROL_PROTOCOL__INVALID_BUFFER_SIZE; @@ -2391,4 +2389,51 @@ exit: return status; } +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_change_hw_infer_status_request( + CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, + uint8_t hw_infer_state, uint8_t network_group_index, uint16_t dynamic_batch_size, + CONTROL_PROTOCOL__hw_infer_channels_info_t *channels_info) +{ + HAILO_COMMON_STATUS_t status = HAILO_COMMON_STATUS__UNINITIALIZED; + size_t local_request_size = 0; + + if ((NULL == request) || (NULL == request_size)) { + status = HAILO_STATUS__CONTROL_PROTOCOL__NULL_ARGUMENT_PASSED; + goto exit; + } + + /* Header */ + local_request_size = CONTROL_PROTOCOL__REQUEST_BASE_SIZE + + sizeof(CONTROL_PROTOCOL__change_hw_infer_status_request_t); + control_protocol__pack_request_header(request, sequence, HAILO_CONTROL_OPCODE_CHANGE_HW_INFER_STATUS, + CHANGE_HW_INFER_REQUEST_PARAMETER_COUNT); + + /* hw_infer_state */ + request->parameters.change_hw_infer_status_request.hw_infer_state_length = + BYTE_ORDER__htonl(sizeof(request->parameters.change_hw_infer_status_request.hw_infer_state)); + request->parameters.change_hw_infer_status_request.hw_infer_state = hw_infer_state; + + /* network_group_index */ + request->parameters.change_hw_infer_status_request.application_index_length = + BYTE_ORDER__htonl(sizeof(request->parameters.change_hw_infer_status_request.application_index)); + request->parameters.change_hw_infer_status_request.application_index = network_group_index; + + /* dynamic_batch_size */ + request->parameters.change_hw_infer_status_request.dynamic_batch_size_length = + BYTE_ORDER__htonl(sizeof(request->parameters.change_hw_infer_status_request.dynamic_batch_size)); + request->parameters.change_hw_infer_status_request.dynamic_batch_size = dynamic_batch_size; + + /* channels_info */ + request->parameters.change_hw_infer_status_request.channels_info_length = + BYTE_ORDER__htonl(sizeof(request->parameters.change_hw_infer_status_request.channels_info)); + memcpy(&(request->parameters.change_hw_infer_status_request.channels_info), + channels_info, + sizeof(request->parameters.change_hw_infer_status_request.channels_info)); + + *request_size = local_request_size; + status = HAILO_COMMON_STATUS__SUCCESS; +exit: + return status; +} + #endif /* FIRMWARE_ARCH */ diff --git a/hailort/libhailort/src/control_protocol.hpp b/hailort/libhailort/src/device_common/control_protocol.hpp similarity index 98% rename from hailort/libhailort/src/control_protocol.hpp rename to hailort/libhailort/src/device_common/control_protocol.hpp index 9092cd8..544f4e2 100644 --- a/hailort/libhailort/src/control_protocol.hpp +++ b/hailort/libhailort/src/device_common/control_protocol.hpp @@ -170,5 +170,8 @@ HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_set_overcurrent_state_request(CONTR HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_get_overcurrent_state_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence); HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_get_hw_consts_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence); HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_set_sleep_state_request(CONTROL_PROTOCOL__request_t *request, size_t *request_size, uint32_t sequence, uint8_t sleep_state); +HAILO_COMMON_STATUS_t CONTROL_PROTOCOL__pack_change_hw_infer_status_request(CONTROL_PROTOCOL__request_t *request, + size_t *request_size, uint32_t sequence, uint8_t hw_infer_state, uint8_t network_group_index, + uint16_t dynamic_batch_size, CONTROL_PROTOCOL__hw_infer_channels_info_t *channels_info); #endif /* _CONTROL_PROTOCOL_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/d2h_event_queue.hpp b/hailort/libhailort/src/device_common/d2h_event_queue.hpp similarity index 93% rename from hailort/libhailort/src/d2h_event_queue.hpp rename to hailort/libhailort/src/device_common/d2h_event_queue.hpp index 5e8d13e..80844be 100644 --- a/hailort/libhailort/src/d2h_event_queue.hpp +++ b/hailort/libhailort/src/device_common/d2h_event_queue.hpp @@ -12,8 +12,10 @@ #ifndef HAILO_D2H_EVENT_QUEUE_HPP_ #define HAILO_D2H_EVENT_QUEUE_HPP_ +#include "utils/thread_safe_queue.hpp" + #include "d2h_events.h" -#include "thread_safe_queue.hpp" + namespace hailort { diff --git a/hailort/libhailort/src/d2h_events_parser.cpp b/hailort/libhailort/src/device_common/d2h_events_parser.cpp similarity index 100% rename from hailort/libhailort/src/d2h_events_parser.cpp rename to hailort/libhailort/src/device_common/d2h_events_parser.cpp diff --git a/hailort/libhailort/src/device.cpp b/hailort/libhailort/src/device_common/device.cpp similarity index 93% rename from hailort/libhailort/src/device.cpp rename to hailort/libhailort/src/device_common/device.cpp index a14bec8..2043e3e 100644 --- a/hailort/libhailort/src/device.cpp +++ b/hailort/libhailort/src/device_common/device.cpp @@ -9,22 +9,25 @@ * TODO: doc **/ -#include +#include "hailo/hailort.h" #include "hailo/device.hpp" + #include "common/utils.hpp" -#include "control.hpp" -#include + +#include "device_common/control.hpp" +#include "vdma/pcie/pcie_device.hpp" +#include "vdma/integrated/integrated_device.hpp" +#include "eth/eth_device.hpp" + #include "byte_order.h" #include "firmware_header_utils.h" #include "control_protocol.h" -#include "pcie_device.hpp" -#include "eth_device.hpp" -#include "core_device.hpp" - +#include #ifndef _MSC_VER #include #endif + namespace hailort { @@ -51,8 +54,8 @@ Device::Device(Type type) : Expected> Device::scan() { // TODO: HRT-7530 support both CORE and PCIE - if (CoreDevice::is_loaded()) { - return std::vector{CoreDevice::DEVICE_ID}; + if (IntegratedDevice::is_loaded()) { + return std::vector{IntegratedDevice::DEVICE_ID}; } else { auto pcie_device_infos = PcieDevice::scan(); @@ -101,7 +104,7 @@ Expected> Device::create() Expected> Device::create(const std::string &device_id) { const bool DONT_LOG_ON_FAILURE = false; - if (CoreDevice::DEVICE_ID == device_id) { + if (IntegratedDevice::DEVICE_ID == device_id) { return create_core(); } else if (auto pcie_info = PcieDevice::parse_pcie_device_info(device_id, DONT_LOG_ON_FAILURE)) { @@ -166,8 +169,8 @@ Expected Device::pcie_device_info_to_string(const hailo_pcie_device Expected Device::get_device_type(const std::string &device_id) { const bool DONT_LOG_ON_FAILURE = false; - if (CoreDevice::DEVICE_ID == device_id) { - return Type::CORE; + if (IntegratedDevice::DEVICE_ID == device_id) { + return Type::INTEGRATED; } else if (auto pcie_info = PcieDevice::parse_pcie_device_info(device_id, DONT_LOG_ON_FAILURE)) { return Type::PCIE; @@ -201,8 +204,8 @@ Expected Device::get_default_streams_interface() const switch(m_type) { case Type::PCIE: return HAILO_STREAM_INTERFACE_PCIE; - case Type::CORE: - return HAILO_STREAM_INTERFACE_CORE; + case Type::INTEGRATED: + return HAILO_STREAM_INTERFACE_INTEGRATED; case Type::ETH: return HAILO_STREAM_INTERFACE_ETH; default: @@ -557,11 +560,27 @@ Expected Device::get_context_switch_breakpoint_status(uint8_t breakpoin Expected> Device::create_core() { - auto core_device = CoreDevice::create(); - CHECK_EXPECTED(core_device); - // Upcasting to Device unique_ptr (from CoreDevice unique_ptr) - auto device = std::unique_ptr(core_device.release()); + auto integrated_device = IntegratedDevice::create(); + CHECK_EXPECTED(integrated_device); + // Upcasting to Device unique_ptr (from IntegratedDevice unique_ptr) + auto device = std::unique_ptr(integrated_device.release()); return device; } +Expected Device::create_configure_params(Hef &hef) const +{ + auto stream_interface = get_default_streams_interface(); + CHECK_EXPECTED(stream_interface, "Failed to get default streams interface"); + + return hef.create_configure_params(stream_interface.release()); +} + +Expected Device::create_configure_params(Hef &hef, const std::string &network_group_name) const +{ + auto stream_interface = get_default_streams_interface(); + CHECK_EXPECTED(stream_interface, "Failed to get default streams interface"); + + return hef.create_configure_params(stream_interface.release(), network_group_name); +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/device_internal.cpp b/hailort/libhailort/src/device_common/device_internal.cpp similarity index 95% rename from hailort/libhailort/src/device_internal.cpp rename to hailort/libhailort/src/device_common/device_internal.cpp index c4ea862..5fd1ea8 100644 --- a/hailort/libhailort/src/device_internal.cpp +++ b/hailort/libhailort/src/device_common/device_internal.cpp @@ -7,10 +7,14 @@ * @brief Implementation of DeviceBase class **/ -#include "device_internal.hpp" #include "hailo/hailort.h" -#include "control.hpp" -#include "sensor_config_utils.hpp" + +#include "common/os_utils.hpp" + +#include "device_common/control.hpp" +#include "device_common/device_internal.hpp" +#include "utils/sensor_config_utils.hpp" + namespace hailort { @@ -139,6 +143,7 @@ void DeviceBase::start_notification_fetch_thread(D2hEventQueue *write_queue) void DeviceBase::notification_fetch_thread(std::shared_ptr params) { + OsUtils::set_current_thread_name("NOTIFY_READ"); while (params->is_running) { auto expected_notification = this->read_notification(); if (HAILO_SUCCESS != expected_notification.status()) { @@ -160,8 +165,8 @@ Expected DeviceBase::get_fw_type() if ((architecture.value() == HAILO_ARCH_HAILO8) || (architecture.value() == HAILO_ARCH_HAILO8L)) { firmware_type = FIRMWARE_TYPE_HAILO8; } - else if (architecture.value() == HAILO_ARCH_MERCURY_CA || architecture.value() == HAILO_ARCH_MERCURY_VPU) { - firmware_type = FIRMWARE_TYPE_MERCURY; + else if (architecture.value() == HAILO_ARCH_HAILO15) { + firmware_type = FIRMWARE_TYPE_HAILO15; } else { LOGGER__ERROR("Invalid device arcitecture. {}", architecture.value()); @@ -525,7 +530,10 @@ hailo_status DeviceBase::erase_user_config() void DeviceBase::start_d2h_notification_thread(const std::string &device_id) { - m_d2h_notification_thread = std::thread([this, device_id]() { this->d2h_notification_thread_main(device_id); }); + m_d2h_notification_thread = std::thread([this, device_id]() { + OsUtils::set_current_thread_name("NOTIFY_PROC"); + d2h_notification_thread_main(device_id); + }); } void DeviceBase::stop_d2h_notification_thread() @@ -591,13 +599,15 @@ hailo_status DeviceBase::check_hef_is_compatible(Hef &hef) CHECK_EXPECTED_AS_STATUS(device_arch, "Can't get device architecture (is the FW loaded?)"); if (!is_hef_compatible(device_arch.value(), hef.pimpl->get_device_arch())) { - // TODO: print here the actual device_arch as a string + auto device_arch_str = HailoRTCommon::get_device_arch_str(device_arch.value()); + auto hef_arch_str = HailoRTCommon::get_device_arch_str(hef_arch_to_device_arch(hef.pimpl->get_device_arch())); + LOGGER__ERROR("HEF format is not compatible with device. Device arch: {}, HEF arch: {}", - device_arch.value(), hef.pimpl->get_device_arch()); + device_arch_str.c_str(), hef_arch_str.c_str()); return HAILO_INVALID_HEF; } - // TODO: MSW-227 check clock rate for mercury as well. + // TODO: MSW-227 check clock rate for hailo15 as well. if ((HAILO_ARCH_HAILO8 == device_arch.value()) || (HAILO_ARCH_HAILO8L == device_arch.value())) { auto extended_device_info_expected = Control::get_extended_device_information(*this); CHECK_EXPECTED_AS_STATUS(extended_device_info_expected, "Can't get device extended info"); @@ -703,15 +713,38 @@ bool DeviceBase::is_hef_compatible(hailo_device_architecture_t device_arch, Prot return (hef_arch == PROTO__HW_ARCH__HAILO8P) || (hef_arch == PROTO__HW_ARCH__HAILO8R) || (hef_arch == PROTO__HW_ARCH__HAILO8L); case HAILO_ARCH_HAILO8L: return (hef_arch == PROTO__HW_ARCH__HAILO8L); - case HAILO_ARCH_MERCURY_CA: - case HAILO_ARCH_MERCURY_VPU: - return (hef_arch == PROTO__HW_ARCH__MERCURY) || (hef_arch == PROTO__HW_ARCH__GINGER) || - (hef_arch == PROTO__HW_ARCH__LAVENDER); + case HAILO_ARCH_HAILO15: + // Compare with HW_ARCH__LAVENDER and HW_ARCH__GINGER to support hefs compiled for them + return (hef_arch == PROTO__HW_ARCH__GINGER) || (hef_arch == PROTO__HW_ARCH__LAVENDER) || + (hef_arch == PROTO__HW_ARCH__HAILO15H); default: return false; } } +hailo_device_architecture_t DeviceBase::hef_arch_to_device_arch(ProtoHEFHwArch hef_arch) +{ + switch (hef_arch) { + case PROTO__HW_ARCH__SAGE_A0: + return HAILO_ARCH_HAILO8_A0; + case PROTO__HW_ARCH__HAILO8: + case PROTO__HW_ARCH__HAILO8P: + case PROTO__HW_ARCH__HAILO8R: + case PROTO__HW_ARCH__SAGE_B0: + case PROTO__HW_ARCH__PAPRIKA_B0: + return HAILO_ARCH_HAILO8; + case PROTO__HW_ARCH__HAILO8L: + return HAILO_ARCH_HAILO8L; + case PROTO__HW_ARCH__HAILO15H: + case PROTO__HW_ARCH__GINGER: + case PROTO__HW_ARCH__LAVENDER: + return HAILO_ARCH_HAILO15; + + default: + return HAILO_ARCH_MAX_ENUM; + } +} + void DeviceBase::check_clock_rate_for_hailo8(uint32_t clock_rate, ProtoHEFHwArch hef_hw_arch) { uint32_t expected_clock_rate = (hef_hw_arch == ProtoHEFHwArch::PROTO__HW_ARCH__HAILO8R) ? HAILO8R_CLOCK_RATE : HAILO8_CLOCK_RATE; diff --git a/hailort/libhailort/src/device_internal.hpp b/hailort/libhailort/src/device_common/device_internal.hpp similarity index 97% rename from hailort/libhailort/src/device_internal.hpp rename to hailort/libhailort/src/device_common/device_internal.hpp index 20df820..58a51ad 100644 --- a/hailort/libhailort/src/device_internal.hpp +++ b/hailort/libhailort/src/device_common/device_internal.hpp @@ -12,7 +12,7 @@ * |-- BaseDevice (Base classes) * |-- VdmaDevice * | |-- PcieDevice - * | |-- CoreDevice + * | |-- IntegratedDevice * |-- EthernetDevice **/ @@ -21,15 +21,16 @@ #include "hailo/device.hpp" #include "hailo/hailort.h" + #include "d2h_event_queue.hpp" -#include "hef_internal.hpp" +#include "hef/hef_internal.hpp" + #include "firmware_header.h" #include "firmware_header_utils.h" #include "control_protocol.h" -#include "hef_internal.hpp" - #include + namespace hailort { @@ -80,6 +81,7 @@ public: virtual Expected read_user_config() override; virtual hailo_status write_user_config(const MemoryView &buffer) override; virtual hailo_status erase_user_config() override; + static hailo_device_architecture_t hef_arch_to_device_arch(ProtoHEFHwArch hef_arch); protected: struct NotificationThreadSharedParams { diff --git a/hailort/libhailort/src/eth/CMakeLists.txt b/hailort/libhailort/src/eth/CMakeLists.txt new file mode 100644 index 0000000..86b410f --- /dev/null +++ b/hailort/libhailort/src/eth/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/eth_device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/eth_stream.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/hcp_config_core_op.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/hcp_config_activated_core_op.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/udp.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/network_rate_calculator.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/eth_device.cpp b/hailort/libhailort/src/eth/eth_device.cpp similarity index 81% rename from hailort/libhailort/src/eth_device.cpp rename to hailort/libhailort/src/eth/eth_device.cpp index 2e54e56..32f955d 100644 --- a/hailort/libhailort/src/eth_device.cpp +++ b/hailort/libhailort/src/eth/eth_device.cpp @@ -9,20 +9,24 @@ * TODO: doc **/ -#include "eth_device.hpp" #include "hailo/hailort.h" -#include "common/utils.hpp" #include "hailo/device.hpp" -#include "control.hpp" -#include "udp.hpp" -#include "common/ethernet_utils.hpp" #include "hailo/hef.hpp" +#include "common/utils.hpp" +#include "common/ethernet_utils.hpp" + +#include "eth/eth_device.hpp" +#include "eth/udp.hpp" +#include "device_common/control.hpp" +#include "network_group/network_group_internal.hpp" + #include #include #include #include + namespace hailort { @@ -386,38 +390,31 @@ hailo_status EthernetDevice::disable_notifications() Expected EthernetDevice::add_hef(Hef &hef, const NetworkGroupsParamsMap &configure_params) { - auto device_arch_exp = get_architecture(); - CHECK_EXPECTED(device_arch_exp); - auto device_arch = device_arch_exp.release(); + // Reset FW state_machine status - can be removed? + static const auto REMOVE_NN_CONFIG_DURING_RESET = false; + auto status = Control::reset_context_switch_state_machine(*this, REMOVE_NN_CONFIG_DURING_RESET); + CHECK_SUCCESS_AS_EXPECTED(status); + + auto added_network_groups = create_networks_group_vector(hef, configure_params); + CHECK_EXPECTED(added_network_groups); + + return added_network_groups; +} +Expected EthernetDevice::create_networks_group_vector(Hef &hef, const NetworkGroupsParamsMap &configure_params) +{ auto partial_clusters_layout_bitmap_exp = Control::get_partial_clusters_layout_bitmap(*this); CHECK_EXPECTED(partial_clusters_layout_bitmap_exp); auto partial_clusters_layout_bitmap = partial_clusters_layout_bitmap_exp.release(); auto &hef_network_groups = hef.pimpl->network_groups(); + auto configure_params_copy = configure_params; ConfiguredNetworkGroupVector added_network_groups; // TODO: can be optimized (add another loop the allocate the network group we're adding) added_network_groups.reserve(hef_network_groups.size()); - auto configure_params_copy = configure_params; - auto hef_arch = hef.pimpl->get_device_arch(); - - // Reset FW state_machine status - can be removed? - static const auto REMOVE_NN_CONFIG_DURING_RESET = false; - auto status = Control::reset_context_switch_state_machine(*this, REMOVE_NN_CONFIG_DURING_RESET); - CHECK_SUCCESS_AS_EXPECTED(status); - for (const auto &base_network_group_proto : hef_network_groups) { - const std::string &network_group_name = base_network_group_proto->network_group_metadata().network_group_name(); - auto &hef_core_ops = hef.pimpl->core_ops(network_group_name); - assert(hef_core_ops.size() == 1); - const auto &core_op = hef_core_ops[0]; - - auto expected_partial_core_op = Hef::Impl::get_core_op_per_arch(core_op, hef_arch, device_arch, - partial_clusters_layout_bitmap); - CHECK_EXPECTED(expected_partial_core_op); - auto partial_core_op = expected_partial_core_op.release(); - - // TODO: decide about core_op names - align with the Compiler + for (const auto &hef_net_group : hef_network_groups) { + const std::string &network_group_name = hef_net_group->network_group_metadata().network_group_name(); /* If NG params are present, use them If no configure params are given, use default*/ @@ -426,52 +423,53 @@ Expected EthernetDevice::add_hef(Hef &hef, const N config_params = configure_params_copy.at(network_group_name); configure_params_copy.erase(network_group_name); } else if (configure_params.empty()) { - auto interface = get_default_streams_interface(); - CHECK_EXPECTED(interface); - auto config_params_exp = hef.create_configure_params(interface.value(), network_group_name); + auto config_params_exp = create_configure_params(hef, network_group_name); CHECK_EXPECTED(config_params_exp); config_params = config_params_exp.release(); } else { continue; } - /* Validate that all network_groups are single context */ - CHECK(1 == partial_core_op->contexts.size(), make_unexpected(HAILO_INTERNAL_FAILURE), - "Only single_context network_groups is supported!. Network group {} has {} contexts.", - network_group_name, partial_core_op->contexts.size()); - CHECK_AS_EXPECTED(!(Hef::Impl::contains_ddr_layers(*partial_core_op)), HAILO_INVALID_OPERATION, - "DDR layers are only supported for PCIe device. Network group {} contains DDR layers.", - network_group_name); - status = Hef::Impl::validate_core_op_unique_layer_names(*partial_core_op); - CHECK_SUCCESS_AS_EXPECTED(status); - - /* Update preliminary_config and dynamic_contexts recepies */ - auto &proto_preliminary_config = partial_core_op->preliminary_config; - auto net_group_config = Hef::Impl::create_single_context_network_group_config(proto_preliminary_config); + auto net_group_config = create_core_op_metadata(hef, network_group_name, partial_clusters_layout_bitmap); CHECK_EXPECTED(net_group_config); - auto network_group_metadata = hef.pimpl->get_network_group_metadata(network_group_name); - CHECK_EXPECTED(network_group_metadata); + // TODO: move to func, support multiple core ops + std::vector> core_ops_ptrs; - auto net_flow_ops = hef.pimpl->post_process_ops(network_group_metadata->network_group_name()); + auto core_op_metadata = hef.pimpl->get_core_op_metadata(network_group_name); + CHECK_EXPECTED(core_op_metadata); - auto single_context_app = HcpConfigNetworkGroup(*this, m_active_net_group_holder, net_group_config.release(), - config_params, network_group_metadata.release(), status, std::move(net_flow_ops)); - CHECK_SUCCESS_AS_EXPECTED(status); + auto core_op_metadata_ptr = make_shared_nothrow(core_op_metadata.release()); + CHECK_AS_EXPECTED(nullptr != core_op_metadata_ptr, HAILO_OUT_OF_HOST_MEMORY); + + auto net_flow_ops = hef.pimpl->post_process_ops(core_op_metadata_ptr->core_op_name()); - auto net_group_shared_ptr = make_shared_nothrow(std::move(single_context_app)); - CHECK_AS_EXPECTED(nullptr != net_group_shared_ptr, HAILO_OUT_OF_HOST_MEMORY); - m_network_groups.emplace_back(net_group_shared_ptr); - added_network_groups.emplace_back(std::static_pointer_cast(net_group_shared_ptr)); + auto status = HAILO_UNINITIALIZED; + auto single_context_app = HcpConfigCoreOp(*this, m_active_core_op_holder, net_group_config.release(), + config_params, core_op_metadata_ptr, status); + CHECK_SUCCESS_AS_EXPECTED(status); - // TODO: move this func into HcpConfigNetworkGroup c'tor - status = net_group_shared_ptr->create_streams_from_config_params(*this); + auto core_op_ptr = make_shared_nothrow(std::move(single_context_app)); + CHECK_AS_EXPECTED(nullptr != core_op_ptr, HAILO_OUT_OF_HOST_MEMORY); + // TODO: move this func into HcpConfigCoreOp c'tor + status = core_op_ptr->create_streams_from_config_params(*this); CHECK_SUCCESS_AS_EXPECTED(status); // Check that all boundary streams were created - status = hef.pimpl->validate_boundary_streams_were_created(network_group_name, *net_group_shared_ptr); + status = hef.pimpl->validate_boundary_streams_were_created(network_group_name, core_op_ptr); CHECK_SUCCESS_AS_EXPECTED(status); + + m_core_ops.push_back(core_op_ptr); + core_ops_ptrs.push_back(core_op_ptr); + + auto net_group_expected = ConfiguredNetworkGroupBase::create(config_params, std::move(core_ops_ptrs), std::move(net_flow_ops)); + CHECK_EXPECTED(net_group_expected); + auto net_group_ptr = net_group_expected.release(); + + added_network_groups.emplace_back(net_group_ptr); + m_network_groups.push_back(net_group_ptr); } + std::string unmatched_keys = ""; for (const auto &pair : configure_params_copy) { unmatched_keys.append(" "); @@ -483,4 +481,40 @@ Expected EthernetDevice::add_hef(Hef &hef, const N return added_network_groups; } +Expected> EthernetDevice::create_core_op_metadata(Hef &hef, const std::string &core_op_name, uint32_t partial_clusters_layout_bitmap) +{ + auto device_arch_exp = get_architecture(); + CHECK_EXPECTED(device_arch_exp); + auto device_arch = device_arch_exp.release(); + auto hef_arch = hef.pimpl->get_device_arch(); + + auto &hef_core_ops = hef.pimpl->core_ops(core_op_name); + assert(1 == hef_core_ops.size()); + const auto &core_op = hef_core_ops[0]; + + auto expected_partial_core_op = Hef::Impl::get_core_op_per_arch(core_op, hef_arch, device_arch, + partial_clusters_layout_bitmap); + CHECK_EXPECTED(expected_partial_core_op); + auto partial_core_op = expected_partial_core_op.release(); + + // TODO: decide about core_op names - align with the Compiler + + /* Validate that all core_ops are single context */ + CHECK(1 == partial_core_op->contexts.size(), make_unexpected(HAILO_INTERNAL_FAILURE), + "Only single-context core-ops are supported!. Core-op {} has {} contexts.", + core_op_name, partial_core_op->contexts.size()); + CHECK_AS_EXPECTED(!(Hef::Impl::contains_ddr_layers(*partial_core_op)), HAILO_INVALID_OPERATION, + "DDR layers are only supported for PCIe device. Core-op {} contains DDR layers.", + core_op_name); + auto status = Hef::Impl::validate_core_op_unique_layer_names(*partial_core_op); + CHECK_SUCCESS_AS_EXPECTED(status); + + /* Update preliminary_config and dynamic_contexts recepies */ + auto &proto_preliminary_config = partial_core_op->preliminary_config; + auto core_op_config = Hef::Impl::create_single_context_core_op_config(proto_preliminary_config); + CHECK_EXPECTED(core_op_config); + + return core_op_config; +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/eth_device.hpp b/hailort/libhailort/src/eth/eth_device.hpp similarity index 78% rename from hailort/libhailort/src/eth_device.hpp rename to hailort/libhailort/src/eth/eth_device.hpp index f840136..e79f93b 100644 --- a/hailort/libhailort/src/eth_device.hpp +++ b/hailort/libhailort/src/eth/eth_device.hpp @@ -12,11 +12,13 @@ #ifndef HAILO_ETH_DEVICE_H_ #define HAILO_ETH_DEVICE_H_ -#include "hailo/expected.hpp" #include "hailo/hailort.h" -#include "device_internal.hpp" -#include "udp.hpp" -#include "context_switch/single_context/hcp_config_network_group.hpp" +#include "hailo/expected.hpp" + +#include "device_common/device_internal.hpp" +#include "eth/udp.hpp" +#include "eth/hcp_config_core_op.hpp" + namespace hailort { @@ -35,7 +37,7 @@ public: { switch (stream_interface) { case HAILO_STREAM_INTERFACE_PCIE: - case HAILO_STREAM_INTERFACE_CORE: + case HAILO_STREAM_INTERFACE_INTEGRATED: return false; case HAILO_STREAM_INTERFACE_ETH: case HAILO_STREAM_INTERFACE_MIPI: @@ -65,12 +67,15 @@ protected: private: EthernetDevice(const hailo_eth_device_info_t &device_info, Udp &&control_udp, hailo_status &status); + Expected create_networks_group_vector(Hef &hef, const NetworkGroupsParamsMap &configure_params); + Expected> create_core_op_metadata(Hef &hef, const std::string &core_op_name, uint32_t partial_clusters_layout_bitmap); const hailo_eth_device_info_t m_device_info; std::string m_device_id; Udp m_control_udp; - std::vector> m_network_groups; - ActiveNetGroupHolder m_active_net_group_holder; + std::vector> m_core_ops; + std::vector> m_network_groups; // TODO: HRT-9547 - Remove when ConfiguredNetworkGroup will be kept in global context + ActiveCoreOpHolder m_active_core_op_holder; }; } /* namespace hailort */ diff --git a/hailort/libhailort/src/eth_stream.cpp b/hailort/libhailort/src/eth/eth_stream.cpp similarity index 93% rename from hailort/libhailort/src/eth_stream.cpp rename to hailort/libhailort/src/eth/eth_stream.cpp index a6938ce..5f6919d 100644 --- a/hailort/libhailort/src/eth_stream.cpp +++ b/hailort/libhailort/src/eth/eth_stream.cpp @@ -9,21 +9,26 @@ * TODO: doc **/ -#include -#include -#include -#include -#include -#include "common/utils.hpp" + +#include "hailo/hailort.h" #include "hailo/stream.hpp" #include "hailo/hef.hpp" #include "hailo/hailort_common.hpp" -#include "eth_stream.hpp" -#include "eth_device.hpp" -#include "control.hpp" -#include "token_bucket.hpp" + #include "common/ethernet_utils.hpp" +#include "common/utils.hpp" + +#include "eth/eth_stream.hpp" +#include "eth/eth_device.hpp" +#include "eth/token_bucket.hpp" +#include "device_common/control.hpp" + +#include +#include +#include +#include + namespace hailort { @@ -79,7 +84,7 @@ hailo_status EthernetInputStream::deactivate_stream() } // Note: Ethernet streams don't work with dynamic batch sizes -hailo_status EthernetInputStream::activate_stream(uint16_t /* dynamic_batch_size */) +hailo_status EthernetInputStream::activate_stream(uint16_t /* dynamic_batch_size */, bool /* resume_pending_stream_transfers */) { hailo_status status = HAILO_UNINITIALIZED; CONTROL_PROTOCOL__config_stream_params_t params = {}; @@ -118,7 +123,7 @@ Expected EthernetInputStream::sync_write_raw_buffer(const MemoryView &bu { hailo_status status = HAILO_UNINITIALIZED; - status = get_network_group_activated_event()->wait(std::chrono::milliseconds(0)); + status = get_core_op_activated_event()->wait(std::chrono::milliseconds(0)); CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_NETWORK_GROUP_NOT_ACTIVATED, "Trying to write on stream before its network_group is activated"); CHECK_SUCCESS_AS_EXPECTED(status); @@ -203,28 +208,18 @@ hailo_status EthernetInputStream::eth_stream__write_with_remainder(void *buffer, } EthernetInputStreamRateLimited::EthernetInputStreamRateLimited(Device &device, Udp &&udp, - EventPtr &&network_group_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info, hailo_status &status) : - EthernetInputStream::EthernetInputStream(device, std::move(udp), std::move(network_group_activated_event), layer_info, status), + EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info, hailo_status &status) : + EthernetInputStream::EthernetInputStream(device, std::move(udp), std::move(core_op_activated_event), layer_info, status), rate_bytes_per_sec(rate_bytes_per_sec) {} -EthernetInputStreamRateLimited::EthernetInputStreamRateLimited(EthernetInputStreamRateLimited &&other) : - EthernetInputStream(std::move(other)), - rate_bytes_per_sec(other.rate_bytes_per_sec) -{} - TokenBucketEthernetInputStream::TokenBucketEthernetInputStream(Device &device, Udp &&udp, - EventPtr &&network_group_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info, hailo_status &status) : + EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info, hailo_status &status) : EthernetInputStreamRateLimited::EthernetInputStreamRateLimited(device, std::move(udp), - std::move(network_group_activated_event), rate_bytes_per_sec, layer_info, status), + std::move(core_op_activated_event), rate_bytes_per_sec, layer_info, status), token_bucket() {} -TokenBucketEthernetInputStream::TokenBucketEthernetInputStream(TokenBucketEthernetInputStream &&other) : - EthernetInputStreamRateLimited(std::move(other)), - token_bucket(std::move(other.token_bucket)) -{} - hailo_status TokenBucketEthernetInputStream::eth_stream__write_with_remainder(void *buffer, size_t offset, size_t size, size_t remainder_size) { size_t transfer_size = 0; size_t offset_end_without_remainder = offset + size - remainder_size; @@ -263,7 +258,7 @@ hailo_status TokenBucketEthernetInputStream::eth_stream__write_with_remainder(vo #if defined(__GNUC__) Expected> TrafficControlEthernetInputStream::create( - Device &device, Udp &&udp, EventPtr &&network_group_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info) + Device &device, Udp &&udp, EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info) { auto board_ip = get_interface_address(&udp.m_device_address.sin_addr); CHECK_EXPECTED(board_ip, "get_interface_address failed with status {}", board_ip.status()); @@ -276,7 +271,7 @@ Expected> TrafficControlEther auto status = HAILO_UNINITIALIZED; // Note: we don't use make_unique because TrafficControlEthernetInputStream's ctor is private auto tc_ptr = std::unique_ptr(new (std::nothrow) - TrafficControlEthernetInputStream(device, std::move(udp), std::move(network_group_activated_event), rate_bytes_per_sec, + TrafficControlEthernetInputStream(device, std::move(udp), std::move(core_op_activated_event), rate_bytes_per_sec, tc.release(), layer_info, status)); CHECK_AS_EXPECTED(nullptr != tc_ptr, HAILO_OUT_OF_HOST_MEMORY); CHECK_SUCCESS_AS_EXPECTED(status); @@ -295,8 +290,8 @@ Expected TrafficControlEthernetInputStream::get_interface_address(c } TrafficControlEthernetInputStream::TrafficControlEthernetInputStream(Device &device, Udp &&udp, - EventPtr &&network_group_activated_event, uint32_t rate_bytes_per_sec, TrafficControl &&tc, const LayerInfo &layer_info, hailo_status &status) : - EthernetInputStreamRateLimited(device, std::move(udp), std::move(network_group_activated_event), rate_bytes_per_sec, layer_info, status), + EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, TrafficControl &&tc, const LayerInfo &layer_info, hailo_status &status) : + EthernetInputStreamRateLimited(device, std::move(udp), std::move(core_op_activated_event), rate_bytes_per_sec, layer_info, status), m_tc(std::move(tc)) {} #endif @@ -363,7 +358,7 @@ hailo_status EthernetInputStream::eth_stream__config_input_sync_params(uint32_t } Expected> EthernetInputStream::create(Device &device, - const LayerInfo &edge_layer, const hailo_eth_input_stream_params_t ¶ms, EventPtr network_group_activated_event) + const LayerInfo &edge_layer, const hailo_eth_input_stream_params_t ¶ms, EventPtr core_op_activated_event) { hailo_status status = HAILO_UNINITIALIZED; // TODO: try to avoid cast @@ -376,18 +371,18 @@ Expected> EthernetInputStream::create(Devic if (params.rate_limit_bytes_per_sec == 0) { local_stream = std::unique_ptr( - new (std::nothrow) EthernetInputStream(device, udp.release(), std::move(network_group_activated_event), edge_layer, status)); + new (std::nothrow) EthernetInputStream(device, udp.release(), std::move(core_op_activated_event), edge_layer, status)); CHECK_SUCCESS_AS_EXPECTED(status); } else { #ifdef _MSC_VER // TODO: Add factory class local_stream = std::unique_ptr( new (std::nothrow) TokenBucketEthernetInputStream(device, udp.release(), - std::move(network_group_activated_event), params.rate_limit_bytes_per_sec, edge_layer, status)); + std::move(core_op_activated_event), params.rate_limit_bytes_per_sec, edge_layer, status)); CHECK_SUCCESS_AS_EXPECTED(status); #else auto stream_expected = TrafficControlEthernetInputStream::create(device, udp.release(), - std::move(network_group_activated_event), params.rate_limit_bytes_per_sec, edge_layer); + std::move(core_op_activated_event), params.rate_limit_bytes_per_sec, edge_layer); CHECK_EXPECTED(stream_expected); local_stream = stream_expected.release(); #endif @@ -467,7 +462,7 @@ hailo_status EthernetOutputStream::deactivate_stream() } // Note: Ethernet streams don't work with dynamic batch sizes -hailo_status EthernetOutputStream::activate_stream(uint16_t /* dynamic_batch_size */) +hailo_status EthernetOutputStream::activate_stream(uint16_t /* dynamic_batch_size */, bool /* resume_pending_stream_transfers */) { hailo_status status = HAILO_UNINITIALIZED; CONTROL_PROTOCOL__config_stream_params_t params = {}; @@ -664,7 +659,7 @@ hailo_status EthernetOutputStream::read_all(MemoryView &buffer) Expected EthernetOutputStream::sync_read_raw_buffer(MemoryView &buffer) { - auto status = get_network_group_activated_event()->wait(std::chrono::milliseconds(0)); + auto status = get_core_op_activated_event()->wait(std::chrono::milliseconds(0)); CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_NETWORK_GROUP_NOT_ACTIVATED, "Trying to read on stream before its network_group is activated"); CHECK_SUCCESS_AS_EXPECTED(status); @@ -698,7 +693,7 @@ hailo_status EthernetOutputStream::fill_output_stream_ptr_with_info(const hailo_ } Expected> EthernetOutputStream::create(Device &device, - const LayerInfo &edge_layer, const hailo_eth_output_stream_params_t ¶ms, EventPtr network_group_activated_event) + const LayerInfo &edge_layer, const hailo_eth_output_stream_params_t ¶ms, EventPtr core_op_activated_event) { hailo_status status = HAILO_UNINITIALIZED; std::unique_ptr local_stream = nullptr; @@ -710,7 +705,7 @@ Expected> EthernetOutputStream::create(Dev CHECK_EXPECTED(udp); local_stream = std::unique_ptr(new (std::nothrow) EthernetOutputStream(device, edge_layer, - udp.release(), std::move(network_group_activated_event), status)); + udp.release(), std::move(core_op_activated_event), status)); CHECK((nullptr != local_stream), make_unexpected(HAILO_OUT_OF_HOST_MEMORY)); CHECK_SUCCESS_AS_EXPECTED(status); diff --git a/hailort/libhailort/src/eth_stream.hpp b/hailort/libhailort/src/eth/eth_stream.hpp similarity index 77% rename from hailort/libhailort/src/eth_stream.hpp rename to hailort/libhailort/src/eth/eth_stream.hpp index 1cc7153..0b6c0a9 100644 --- a/hailort/libhailort/src/eth_stream.hpp +++ b/hailort/libhailort/src/eth/eth_stream.hpp @@ -12,18 +12,20 @@ #ifndef HAILO_ETH_STREAM_H_ #define HAILO_ETH_STREAM_H_ -#include "stream_internal.hpp" #include "hailo/hailort.h" -#include "token_bucket.hpp" -#include "udp.hpp" #include "hailo/hef.hpp" #include "hailo/device.hpp" #include "hailo/event.hpp" +#include "eth/token_bucket.hpp" +#include "eth/udp.hpp" +#include "stream_common/stream_internal.hpp" + #if defined(__GNUC__) #include "common/os/posix/traffic_control.hpp" #endif + namespace hailort { @@ -63,23 +65,15 @@ protected: virtual hailo_status sync_write_all_raw_buffer_no_transform_impl(void *buffer, size_t offset, size_t size) override; public: - EthernetInputStream(Device &device, Udp &&udp, EventPtr &&network_group_activated_event, const LayerInfo &layer_info, hailo_status &status) : - InputStreamBase(layer_info, HAILO_STREAM_INTERFACE_ETH, std::move(network_group_activated_event), status), m_udp(std::move(udp)), m_device(device) {} - EthernetInputStream(EthernetInputStream&& other) : - InputStreamBase(std::move(other)), - configuration(std::move(other.configuration)), - m_udp(std::move(other.m_udp)), - m_is_stream_activated(std::exchange(other.m_is_stream_activated, false)), - m_device(other.m_device) - {} - + EthernetInputStream(Device &device, Udp &&udp, EventPtr &&core_op_activated_event, const LayerInfo &layer_info, hailo_status &status) : + InputStreamBase(layer_info, HAILO_STREAM_INTERFACE_ETH, std::move(core_op_activated_event), status), m_udp(std::move(udp)), m_device(device) {} virtual ~EthernetInputStream(); static Expected> create(Device &device, - const LayerInfo &edge_layer, const hailo_eth_input_stream_params_t ¶ms, EventPtr network_group_activated_event); + const LayerInfo &edge_layer, const hailo_eth_input_stream_params_t ¶ms, EventPtr core_op_activated_event); uint16_t get_remote_port(); - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; virtual hailo_status deactivate_stream() override; virtual hailo_stream_interface_t get_interface() const override { return HAILO_STREAM_INTERFACE_ETH; } virtual std::chrono::milliseconds get_timeout() const override; @@ -92,9 +86,8 @@ protected: const uint32_t rate_bytes_per_sec; public: - EthernetInputStreamRateLimited(Device &device, Udp &&udp, EventPtr &&network_group_activated_event, + EthernetInputStreamRateLimited(Device &device, Udp &&udp, EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info, hailo_status &status); - EthernetInputStreamRateLimited(EthernetInputStreamRateLimited &&other); virtual ~EthernetInputStreamRateLimited() = default; }; @@ -113,9 +106,8 @@ protected: virtual hailo_status eth_stream__write_with_remainder(void *buffer, size_t offset, size_t size, size_t remainder_size); public: - TokenBucketEthernetInputStream(Device &device, Udp &&udp, EventPtr &&network_group_activated_event, + TokenBucketEthernetInputStream(Device &device, Udp &&udp, EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info, hailo_status &status); - TokenBucketEthernetInputStream(TokenBucketEthernetInputStream &&other); virtual ~TokenBucketEthernetInputStream() = default; }; @@ -124,12 +116,11 @@ public: class TrafficControlEthernetInputStream : public EthernetInputStreamRateLimited { public: static Expected> create(Device &device, Udp &&udp, - EventPtr &&network_group_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info); - TrafficControlEthernetInputStream(TrafficControlEthernetInputStream&& other) = default; + EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, const LayerInfo &layer_info); virtual ~TrafficControlEthernetInputStream() = default; private: - TrafficControlEthernetInputStream(Device &device, Udp &&udp, EventPtr &&network_group_activated_event, + TrafficControlEthernetInputStream(Device &device, Udp &&udp, EventPtr &&core_op_activated_event, uint32_t rate_bytes_per_sec, TrafficControl &&tc, const LayerInfo &layer_info, hailo_status &status); static Expected get_interface_address(const struct in_addr *addr); @@ -148,8 +139,8 @@ private: bool m_is_stream_activated; Device &m_device; - EthernetOutputStream(Device &device, const LayerInfo &edge_layer, Udp &&udp, EventPtr &&network_group_activated_event, hailo_status &status) : - OutputStreamBase(edge_layer, std::move(network_group_activated_event), status), + EthernetOutputStream(Device &device, const LayerInfo &edge_layer, Udp &&udp, EventPtr &&core_op_activated_event, hailo_status &status) : + OutputStreamBase(edge_layer, std::move(core_op_activated_event), status), leftover_buffer(), leftover_size(0), // Firmware starts sending sync sequence from 0, so treating the first previous as max value (that will be overflowed to 0) @@ -173,28 +164,14 @@ private: static hailo_status fill_output_stream_ptr_with_info(const hailo_eth_output_stream_params_t ¶ms, EthernetOutputStream *stream); public: - EthernetOutputStream(EthernetOutputStream&& other) : - OutputStreamBase(std::move(other)), - leftover_buffer(), - leftover_size(std::move(other.leftover_size)), - last_seen_sync_index(std::move(other.last_seen_sync_index)), - encountered_timeout(std::move(other.encountered_timeout)), - configuration(std::move(other.configuration)), - m_udp(std::move(other.m_udp)), - m_is_stream_activated(std::exchange(other.m_is_stream_activated, false)), - m_device(other.m_device) - { - memcpy(leftover_buffer, other.leftover_buffer, sizeof(leftover_buffer)); - } - virtual ~EthernetOutputStream(); virtual Expected sync_read_raw_buffer(MemoryView &buffer); static Expected> create(Device &device, const LayerInfo &edge_layer, - const hailo_eth_output_stream_params_t ¶ms, EventPtr network_group_activated_event); + const hailo_eth_output_stream_params_t ¶ms, EventPtr core_op_activated_event); - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; virtual hailo_status deactivate_stream() override; virtual hailo_stream_interface_t get_interface() const override { return HAILO_STREAM_INTERFACE_ETH; } virtual std::chrono::milliseconds get_timeout() const override; diff --git a/hailort/libhailort/src/eth/hcp_config_activated_core_op.cpp b/hailort/libhailort/src/eth/hcp_config_activated_core_op.cpp new file mode 100644 index 0000000..9dff027 --- /dev/null +++ b/hailort/libhailort/src/eth/hcp_config_activated_core_op.cpp @@ -0,0 +1,106 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file hcp_config_activated_core_op.cpp + * @brief HcpConfigActivatedCoreOp implementation + **/ + +#include "eth/hcp_config_activated_core_op.hpp" +#include "device_common/control.hpp" + + +namespace hailort +{ + +Expected HcpConfigActivatedCoreOp::create(Device &device, std::vector &config, + const std::string &core_op_name, + // hailo_activate_network_group_params_t is currently an empty holder, if anything will be added to it, + // it will require a check that these params will be relevant for this one core op only. + const hailo_activate_network_group_params_t &network_group_params, + std::map> &input_streams, + std::map> &output_streams, + ActiveCoreOpHolder &active_core_op_holder, + hailo_power_mode_t power_mode, EventPtr core_op_activated_event, + CoreOp &core_op) +{ + CHECK(!active_core_op_holder.is_any_active(), make_unexpected(HAILO_INVALID_OPERATION), + "core-op is currently active. You must deactivate before activating another core-op"); + + // Close older dataflows + auto status = Control::close_all_streams(device); + CHECK_SUCCESS_AS_EXPECTED(status); + + // Reset nn_core before writing configurations + status = device.reset(HAILO_RESET_DEVICE_MODE_NN_CORE); + CHECK_SUCCESS_AS_EXPECTED(status); + + for (auto &m : config) { + status = device.write_memory(m.address, MemoryView(m.data)); + CHECK_SUCCESS_AS_EXPECTED(status); + } + + HcpConfigActivatedCoreOp object(device, active_core_op_holder, core_op_name, network_group_params, input_streams, output_streams, + power_mode, std::move(core_op_activated_event), core_op, status); + CHECK_SUCCESS_AS_EXPECTED(status); + return object; +} + +HcpConfigActivatedCoreOp::HcpConfigActivatedCoreOp( + Device &device, + ActiveCoreOpHolder &active_core_op_holder, + const std::string &core_op_name, + const hailo_activate_network_group_params_t &network_group_params, + std::map> &input_streams, + std::map> &output_streams, + hailo_power_mode_t power_mode, + EventPtr &&core_op_activated_event, + CoreOp &core_op, hailo_status &status) : + ActivatedCoreOp(network_group_params, input_streams, output_streams, + std::move(core_op_activated_event), status), + m_active_core_op_holder(active_core_op_holder), + m_is_active(true), + m_power_mode(power_mode), + m_device(device), + m_core_op_name(core_op_name) +{ + // Validate ActivatedCoreOp status + if (HAILO_SUCCESS != status) { + return; + } + status = core_op.activate_impl(CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to activate core-op"); + return; + } +} + +HcpConfigActivatedCoreOp::~HcpConfigActivatedCoreOp() +{ + if (!m_is_active) { + return; + } + + auto expected_config_network_ref = m_active_core_op_holder.get(); + if (!expected_config_network_ref.has_value()) { + LOGGER__ERROR("Error getting configured core-op"); + return; + } + const auto &config_core_op = expected_config_network_ref.value(); + + const auto status = config_core_op.get().deactivate_impl(); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to deactivate core-op"); + } +} + +// TODO: add get_core_op_name() for better code readability? +const std::string &HcpConfigActivatedCoreOp::get_network_group_name() const +{ + // network_group name is the same as core_op name in this case. + // HcpConfigActivatedCoreOp should be used only for single core ops network groups. + return m_core_op_name; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/eth/hcp_config_activated_core_op.hpp b/hailort/libhailort/src/eth/hcp_config_activated_core_op.hpp new file mode 100644 index 0000000..2b820d8 --- /dev/null +++ b/hailort/libhailort/src/eth/hcp_config_activated_core_op.hpp @@ -0,0 +1,87 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file hcp_config_activated_core_op.hpp + * @brief Represent activated core-op from HEF. + * + * This core-op can be used for control-core-op only (for etherent or pcie) + **/ + +#ifndef _HAILO_CONTEXT_SWITCH_HCP_CONFIG_ACTIVATED_CORE_OP_HPP_ +#define _HAILO_CONTEXT_SWITCH_HCP_CONFIG_ACTIVATED_CORE_OP_HPP_ + +#include "hailo/device.hpp" + +#include "common/utils.hpp" + +#include "core_op/active_core_op_holder.hpp" + +#include +#include + + +namespace hailort +{ + +struct WriteMemoryInfo +{ + uint32_t address; + Buffer data; +}; + +class HcpConfigActivatedCoreOp : public ActivatedCoreOp +{ + public: + static Expected create(Device &device, std::vector &config, + const std::string &core_op_name, + const hailo_activate_network_group_params_t &network_group_params, + std::map> &input_streams, + std::map> &output_streams, + ActiveCoreOpHolder &active_core_op_holder, + hailo_power_mode_t power_mode, EventPtr core_op_activated_event, + CoreOp &core_op); + + virtual ~HcpConfigActivatedCoreOp(); + HcpConfigActivatedCoreOp(const HcpConfigActivatedCoreOp &) = delete; + HcpConfigActivatedCoreOp &operator=(const HcpConfigActivatedCoreOp &) = delete; + HcpConfigActivatedCoreOp &operator=(HcpConfigActivatedCoreOp &&) = delete; + HcpConfigActivatedCoreOp(HcpConfigActivatedCoreOp &&other) noexcept : + ActivatedCoreOp(std::move(other)), m_active_core_op_holder(other.m_active_core_op_holder), + m_is_active(std::exchange(other.m_is_active, false)), m_power_mode(other.m_power_mode), + m_device(other.m_device), m_core_op_name(std::move(other.m_core_op_name)) {}; + + virtual const std::string &get_network_group_name() const override; + + virtual Expected get_intermediate_buffer(const IntermediateBufferKey &/*key*/) override + { + LOGGER__ERROR("get_intermediate_buffer() is not supported on single_context core_ops"); + return make_unexpected(HAILO_INVALID_OPERATION); + } + + virtual hailo_status set_keep_nn_config_during_reset(const bool /* keep_nn_config_during_reset */) override + { + LOGGER__ERROR("set_keep_nn_config_during_reset() is not supported on single_context core_ops"); + return HAILO_INVALID_OPERATION; + } + + private: + HcpConfigActivatedCoreOp(Device &device, ActiveCoreOpHolder &active_core_op_holder, + const std::string &core_op_name, + const hailo_activate_network_group_params_t &network_group_params, + std::map> &input_streams, + std::map> &output_streams, + hailo_power_mode_t power_mode, EventPtr &&core_op_activated_event, + CoreOp &core_op, hailo_status &status); + + ActiveCoreOpHolder &m_active_core_op_holder; + bool m_is_active; + hailo_power_mode_t m_power_mode; + Device &m_device; + std::string m_core_op_name; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_CONTEXT_SWITCH_HCP_CONFIG_ACTIVATED_CORE_OP_HPP_ */ diff --git a/hailort/libhailort/src/eth/hcp_config_core_op.cpp b/hailort/libhailort/src/eth/hcp_config_core_op.cpp new file mode 100644 index 0000000..eb6d8bc --- /dev/null +++ b/hailort/libhailort/src/eth/hcp_config_core_op.cpp @@ -0,0 +1,124 @@ +#include "eth/hcp_config_core_op.hpp" +#include "device_common/control.hpp" + + +#define OUTPUT_CHANNEL_INDEX_OFFSET (16) + + +namespace hailort +{ + +HcpConfigCoreOp::HcpConfigCoreOp(Device &device, ActiveCoreOpHolder &active_core_op_holder, + std::vector &&config, const ConfigureNetworkParams &config_params, std::shared_ptr metadata, + hailo_status &status) + : CoreOp(config_params, metadata, status), + m_config(std::move(config)), m_active_core_op_holder(active_core_op_holder), m_device(device) +{} + +Expected> HcpConfigCoreOp::create_activated_network_group( + const hailo_activate_network_group_params_t &network_group_params, uint16_t /* dynamic_batch_size */, + bool /* resume_pending_stream_transfers */) +{ + auto start_time = std::chrono::steady_clock::now(); + + auto activated_net_group = HcpConfigActivatedCoreOp::create(m_device, m_config, name(), network_group_params, + m_input_streams, m_output_streams, m_active_core_op_holder, m_config_params.power_mode, + m_core_op_activated_event, (*this)); + CHECK_EXPECTED(activated_net_group); + + std::unique_ptr activated_net_group_ptr = make_unique_nothrow(activated_net_group.release()); + CHECK_AS_EXPECTED(nullptr != activated_net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); + + auto elapsed_time_ms = std::chrono::duration(std::chrono::steady_clock::now() - start_time).count(); + LOGGER__INFO("Activating {} took {} milliseconds. Note that the function is asynchronous and thus the network is not fully activated yet.", name(), elapsed_time_ms); + + return activated_net_group_ptr; +} + +Expected HcpConfigCoreOp::get_default_streams_interface() +{ + return m_device.get_default_streams_interface(); +} + +bool HcpConfigCoreOp::is_scheduled() const +{ + // Scheduler not supported on HcpConfigCoreOp + return false; +} + +hailo_status HcpConfigCoreOp::set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) +{ + (void) timeout; + (void) network_name; + return HAILO_INVALID_OPERATION; +} + +hailo_status HcpConfigCoreOp::set_scheduler_threshold(uint32_t threshold, const std::string &network_name) +{ + (void) threshold; + (void) network_name; + return HAILO_INVALID_OPERATION; +} + +hailo_status HcpConfigCoreOp::set_scheduler_priority(uint8_t /*priority*/, const std::string &/*network_name*/) +{ + return HAILO_INVALID_OPERATION; +} + +Expected> HcpConfigCoreOp::get_latency_meters() +{ + /* hcp does not support latnecy. return empty map */ + LatencyMetersMap empty_map; + return make_shared_nothrow(empty_map); +} + +Expected HcpConfigCoreOp::get_boundary_vdma_channel_by_stream_name( + const std::string &stream_name) +{ + LOGGER__ERROR("get_boundary_vdma_channel_by_stream_name function for stream name {} is not supported on ETH core-ops", + stream_name); + return make_unexpected(HAILO_INVALID_OPERATION); +} + +hailo_status HcpConfigCoreOp::activate_impl(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) +{ + m_active_core_op_holder.set(*this); + + auto status = activate_low_level_streams(dynamic_batch_size, resume_pending_stream_transfers); + CHECK_SUCCESS(status, "Failed activating low level streams"); + + status = m_core_op_activated_event->signal(); + CHECK_SUCCESS(status, "Failed to signal network activation event"); + + return HAILO_SUCCESS; +} +hailo_status HcpConfigCoreOp::deactivate_impl(bool /* keep_nn_config_during_reset */) +{ + auto expected_core_op_ref = m_active_core_op_holder.get(); + CHECK(expected_core_op_ref.has_value(), HAILO_INTERNAL_FAILURE, "Error getting configured core-op"); + + const auto &core_op = expected_core_op_ref.value(); + // Make sure the core-op we are deactivating is this object + CHECK(this == std::addressof(core_op.get()), HAILO_INTERNAL_FAILURE, + "Trying to deactivate different core-op"); + + m_active_core_op_holder.clear(); + + if (!m_core_op_activated_event) { + return HAILO_SUCCESS; + } + + m_core_op_activated_event->reset(); + + for (auto &name_pair : m_input_streams) { + const auto status = name_pair.second->flush(); + CHECK_SUCCESS(status, "Failed to flush input stream {}", name_pair.first); + } + + auto status = deactivate_low_level_streams(); + CHECK_SUCCESS(status, "Failed deactivating low level streams"); + + return HAILO_SUCCESS; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/eth/hcp_config_core_op.hpp b/hailort/libhailort/src/eth/hcp_config_core_op.hpp new file mode 100644 index 0000000..710d98c --- /dev/null +++ b/hailort/libhailort/src/eth/hcp_config_core_op.hpp @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file hcp_config_core_op.hpp + * @brief Represent core-op from HEF file that can be activated + * + * This core-op can be used for control-core-op (for etherent or pcie) + **/ + +#ifndef _HAILO_CONTEXT_SWITCH_HCP_CONFIG_CORE_OP_HPP_ +#define _HAILO_CONTEXT_SWITCH_HCP_CONFIG_CORE_OP_HPP_ + +#include "hailo/device.hpp" +#include "hailo/hailort_defaults.hpp" + +#include "common/utils.hpp" + +#include "eth/hcp_config_activated_core_op.hpp" +#include "core_op/active_core_op_holder.hpp" +#include "core_op/core_op.hpp" + +#include +#include + + +namespace hailort +{ + +class HcpConfigCoreOp : public CoreOp +{ +public: + HcpConfigCoreOp( + Device &device, ActiveCoreOpHolder &active_core_op_holder, std::vector &&config, + const ConfigureNetworkParams &config_params, std::shared_ptr metadata, hailo_status &status); + + virtual Expected> create_activated_network_group( + const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size, + bool resume_pending_stream_transfers) override; + virtual Expected get_default_streams_interface() override; + + virtual Expected> get_latency_meters() override; + virtual Expected get_boundary_vdma_channel_by_stream_name( + const std::string &stream_name) override; + virtual bool is_scheduled() const override; + virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) override; + virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) override; + virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name) override; + + virtual hailo_status activate_impl(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; + virtual hailo_status deactivate_impl(bool keep_nn_config_during_reset) override; + + virtual ~HcpConfigCoreOp() = default; + HcpConfigCoreOp(const HcpConfigCoreOp &other) = delete; + HcpConfigCoreOp &operator=(const HcpConfigCoreOp &other) = delete; + HcpConfigCoreOp &operator=(HcpConfigCoreOp &&other) = delete; + HcpConfigCoreOp(HcpConfigCoreOp &&other) noexcept : CoreOp(std::move(other)), + m_config(std::move(other.m_config)), m_active_core_op_holder(other.m_active_core_op_holder), + m_device(other.m_device) {} + +private: + std::vector m_config; + ActiveCoreOpHolder &m_active_core_op_holder; + Device &m_device; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_CONTEXT_SWITCH_HCP_CONFIG_CORE_OP_HPP_ */ diff --git a/hailort/libhailort/src/network_rate_calculator.cpp b/hailort/libhailort/src/eth/network_rate_calculator.cpp similarity index 99% rename from hailort/libhailort/src/network_rate_calculator.cpp rename to hailort/libhailort/src/eth/network_rate_calculator.cpp index 2a4677d..0578d67 100644 --- a/hailort/libhailort/src/network_rate_calculator.cpp +++ b/hailort/libhailort/src/eth/network_rate_calculator.cpp @@ -7,14 +7,18 @@ * @brief: Network rate calculator **/ -#include "hailo/network_rate_calculator.hpp" + #include "hailo/hailort.h" +#include "hailo/network_rate_calculator.hpp" -#include "eth_stream.hpp" #include "common/utils.hpp" + +#include "eth/eth_stream.hpp" + #include #include + namespace hailort { diff --git a/hailort/libhailort/src/token_bucket.hpp b/hailort/libhailort/src/eth/token_bucket.hpp similarity index 100% rename from hailort/libhailort/src/token_bucket.hpp rename to hailort/libhailort/src/eth/token_bucket.hpp diff --git a/hailort/libhailort/src/udp.cpp b/hailort/libhailort/src/eth/udp.cpp similarity index 98% rename from hailort/libhailort/src/udp.cpp rename to hailort/libhailort/src/eth/udp.cpp index b2fc899..af97e7b 100644 --- a/hailort/libhailort/src/udp.cpp +++ b/hailort/libhailort/src/eth/udp.cpp @@ -7,16 +7,18 @@ * @brief Socket wrapper for Unix **/ -#include -#include -#include +#include "hailo/hailort.h" -#include #include "common/utils.hpp" #include "common/logger_macros.hpp" -#include "udp.hpp" #include "common/socket.hpp" -#include "control_protocol.hpp" +#include "eth/udp.hpp" +#include "device_common/control_protocol.hpp" + +#include +#include +#include + namespace hailort { diff --git a/hailort/libhailort/src/udp.hpp b/hailort/libhailort/src/eth/udp.hpp similarity index 98% rename from hailort/libhailort/src/udp.hpp rename to hailort/libhailort/src/eth/udp.hpp index 900bcb0..2dab4e3 100644 --- a/hailort/libhailort/src/udp.hpp +++ b/hailort/libhailort/src/eth/udp.hpp @@ -10,10 +10,11 @@ #ifndef __OS_UDP_H__ #define __OS_UDP_H__ +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + #include "common/socket.hpp" -#include -#include "hailo/expected.hpp" namespace hailort { diff --git a/hailort/libhailort/src/hailort.cpp b/hailort/libhailort/src/hailort.cpp index 5f8b592..e065116 100644 --- a/hailort/libhailort/src/hailort.cpp +++ b/hailort/libhailort/src/hailort.cpp @@ -21,26 +21,32 @@ #include "hailo/event.hpp" #include "hailo/network_rate_calculator.hpp" #include "hailo/inference_pipeline.hpp" -#include "eth_device.hpp" -#include "pcie_device.hpp" -#include "pcie_stream.hpp" -#include "eth_stream.hpp" -#include "control.hpp" -#include "sensor_config_utils.hpp" + #include "common/compiler_extensions_compat.hpp" -#include "hailort_logger.hpp" -#include "shared_resource_manager.hpp" -#include "vdevice_internal.hpp" -#include "tracer_macros.hpp" +#include "common/os_utils.hpp" + +#include "device_common/control.hpp" +#include "eth/eth_device.hpp" +#include "eth/eth_stream.hpp" +#include "vdma/pcie/pcie_device.hpp" +#include "utils/sensor_config_utils.hpp" +#include "utils/hailort_logger.hpp" +#include "utils/shared_resource_manager.hpp" +#include "vdevice/vdevice_internal.hpp" +#include "utils/profiler/tracer_macros.hpp" #include + using namespace hailort; COMPAT__INITIALIZER(hailort__initialize_logger) { // Init logger singleton if compiling only HailoRT (void) HailoRTLogger::get_instance(); +#ifdef HAILO_SUPPORT_MULTI_PROCESS + (void) HailoRTOSLogger::get_instance(); +#endif (void) SharedResourceManager::get_instance(); TRACE(InitTrace); } @@ -225,8 +231,8 @@ hailo_status hailo_device_get_type_by_device_id(const hailo_device_id_t *device_ case Device::Type::ETH: *device_type = HAILO_DEVICE_TYPE_ETH; break; - case Device::Type::CORE: - *device_type = HAILO_DEVICE_TYPE_CORE; + case Device::Type::INTEGRATED: + *device_type = HAILO_DEVICE_TYPE_INTEGRATED; break; default: LOGGER__ERROR("Internal failure, invalid device type returned"); @@ -840,7 +846,7 @@ hailo_status hailo_activate_network_group(hailo_configured_network_group network hailo_activate_network_group_params_t actual_activation_params = (activation_params != nullptr) ? *activation_params : - HailoRTDefaults::get_network_group_params(); + HailoRTDefaults::get_active_network_group_params(); auto net_group_ptr = reinterpret_cast(network_group); auto activated_net_group = net_group_ptr->activate(actual_activation_params); @@ -982,7 +988,7 @@ hailo_status hailo_set_scheduler_timeout(hailo_configured_network_group configur CHECK_ARG_NOT_NULL(configured_network_group); std::string network_name_str = (nullptr == network_name) ? "" : network_name; - return ((ConfiguredNetworkGroup*)configured_network_group)->set_scheduler_timeout(std::chrono::milliseconds(timeout_ms), network_name_str); + return (reinterpret_cast(configured_network_group))->set_scheduler_timeout(std::chrono::milliseconds(timeout_ms), network_name_str); } hailo_status hailo_set_scheduler_threshold(hailo_configured_network_group configured_network_group, @@ -991,7 +997,15 @@ hailo_status hailo_set_scheduler_threshold(hailo_configured_network_group config CHECK_ARG_NOT_NULL(configured_network_group); std::string network_name_str = (nullptr == network_name) ? "" : network_name; - return ((ConfiguredNetworkGroup*)configured_network_group)->set_scheduler_threshold(threshold, network_name_str); + return (reinterpret_cast(configured_network_group))->set_scheduler_threshold(threshold, network_name_str); +} + +hailo_status hailo_set_scheduler_priority(hailo_configured_network_group configured_network_group, uint8_t priority, const char *network_name) +{ + CHECK_ARG_NOT_NULL(configured_network_group); + + std::string network_name_str = (nullptr == network_name) ? "" : network_name; + return (reinterpret_cast(configured_network_group))->set_scheduler_priority(priority, network_name_str); } hailo_status hailo_calculate_eth_input_rate_limits(hailo_hef hef, const char *network_group_name, uint32_t fps, diff --git a/hailort/libhailort/src/hailort_defaults.cpp b/hailort/libhailort/src/hailort_defaults.cpp new file mode 100644 index 0000000..c845d3e --- /dev/null +++ b/hailort/libhailort/src/hailort_defaults.cpp @@ -0,0 +1,385 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file hailort_defaults.cpp + * @brief Implmentation of hailort_defaults + **/ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" +#include "hailo/hailort_defaults.hpp" + +#include "common/logger_macros.hpp" +#include "common/utils.hpp" + + +namespace hailort +{ + +static const hailo_format_order_t DEFAULT_FORMAT_ORDER_MAP[] = { + // Key is device_format_order, value is default user_format_order + HAILO_FORMAT_ORDER_AUTO, // HAILO_FORMAT_ORDER_AUTO, - Should not be used! + HAILO_FORMAT_ORDER_NHWC, // HAILO_FORMAT_ORDER_NHWC, + HAILO_FORMAT_ORDER_NHWC, // HAILO_FORMAT_ORDER_NHCW, + HAILO_FORMAT_ORDER_FCR, // HAILO_FORMAT_ORDER_FCR, + HAILO_FORMAT_ORDER_F8CR, // HAILO_FORMAT_ORDER_F8CR, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NHW, + HAILO_FORMAT_ORDER_NC, // HAILO_FORMAT_ORDER_NC, + HAILO_FORMAT_ORDER_BAYER_RGB, // HAILO_FORMAT_ORDER_BAYER_RGB, + HAILO_FORMAT_ORDER_12_BIT_BAYER_RGB, // HAILO_FORMAT_ORDER_12_BIT_BAYER_RGB + HAILO_FORMAT_ORDER_HAILO_NMS, // HAILO_FORMAT_ORDER_HAILO_NMS, + HAILO_FORMAT_ORDER_NHWC, // HAILO_FORMAT_ORDER_RGB888, + HAILO_FORMAT_ORDER_NCHW, // HAILO_FORMAT_ORDER_NCHW, + HAILO_FORMAT_ORDER_YUY2, // HAILO_FORMAT_ORDER_YUY2, + HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_NV12, + HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_NV21, + HAILO_FORMAT_ORDER_NV12, // HAILO_FORMAT_ORDER_HAILO_YYUV, + HAILO_FORMAT_ORDER_NV21, // HAILO_FORMAT_ORDER_HAILO_YYVU, + HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_RGB4, + HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_I420, + HAILO_FORMAT_ORDER_I420 // HAILO_FORMAT_ORDER_HAILO_YYYYUV, +}; + +static const hailo_format_order_t DEFAULT_FORMAT_ARGMAX_ORDER_MAP[] = { + // Key is device_format_order, value is default user_format_order + HAILO_FORMAT_ORDER_AUTO, // HAILO_FORMAT_ORDER_AUTO, - Should not be used! + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NHWC, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NHCW, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_FCR, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_F8CR, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NHW, + HAILO_FORMAT_ORDER_NC, // HAILO_FORMAT_ORDER_NC, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_BAYER_RGB, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_12_BIT_BAYER_RGB, + HAILO_FORMAT_ORDER_HAILO_NMS, // HAILO_FORMAT_ORDER_HAILO_NMS, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_RGB888, + HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NCHW, + HAILO_FORMAT_ORDER_YUY2, // HAILO_FORMAT_ORDER_YUY2, + HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_NV12, + HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_NV21, + HAILO_FORMAT_ORDER_NV12, // HAILO_FORMAT_ORDER_HAILO_YYUV, + HAILO_FORMAT_ORDER_NV21, // HAILO_FORMAT_ORDER_HAILO_YYVU, + HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_RGB4, + HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_I420, + HAILO_FORMAT_ORDER_I420 // HAILO_FORMAT_ORDER_HAILO_YYYYUV, +}; + +// This func must be aligned to SDK! +Expected HailoRTDefaults::get_device_format_order(uint32_t compiler_format_order) +{ + switch (compiler_format_order) { + case 0: + return std::move(HAILO_FORMAT_ORDER_NHWC); + break; + case 1: + return std::move(HAILO_FORMAT_ORDER_NHCW); + break; + case 2: + return std::move(HAILO_FORMAT_ORDER_NC); + break; + case 3: + return std::move(HAILO_FORMAT_ORDER_FCR); + break; + case 4: + return std::move(HAILO_FORMAT_ORDER_BAYER_RGB); + break; + case 5: + return std::move(HAILO_FORMAT_ORDER_NHW); + break; + case 6: + return std::move(HAILO_FORMAT_ORDER_HAILO_NMS); + break; + case 7: + return std::move(HAILO_FORMAT_ORDER_F8CR); + break; + case 8: + return std::move(HAILO_FORMAT_ORDER_RGB888); + break; + case 11: + return std::move(HAILO_FORMAT_ORDER_YUY2); + break; + case 13: + return std::move(HAILO_FORMAT_ORDER_NHWC); + break; + case 14: + return std::move(HAILO_FORMAT_ORDER_HAILO_YYUV); + break; + case 15: + return std::move(HAILO_FORMAT_ORDER_HAILO_YYVU); + break; + case 16: + return std::move(HAILO_FORMAT_ORDER_HAILO_YYYYUV); + break; + default: + LOGGER__ERROR("Invalid compiler_format_order ({})", compiler_format_order); + return make_unexpected(HAILO_INTERNAL_FAILURE); + } +} + +hailo_format_order_t HailoRTDefaults::get_default_host_format_order(const hailo_format_t &device_format) +{ + const bool is_argmax = (0 != (device_format.flags & HAILO_FORMAT_FLAGS_HOST_ARGMAX)); + if (!is_argmax) { + return DEFAULT_FORMAT_ORDER_MAP[device_format.order]; + } else { + return DEFAULT_FORMAT_ARGMAX_ORDER_MAP[device_format.order]; + } +} + +struct sockaddr_in HailoRTDefaults::get_sockaddr() +{ + struct sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_port = 0; + address.sin_addr.s_addr = INADDR_ANY; + // sin_zero is already zeroed + return address; +} + +hailo_format_t HailoRTDefaults::get_user_buffer_format() +{ + return get_user_buffer_format(true, HAILO_FORMAT_TYPE_AUTO); +} + +hailo_format_t HailoRTDefaults::get_user_buffer_format(bool quantized, hailo_format_type_t format_type) +{ + hailo_format_t user_buffer_format{}; + user_buffer_format.type = format_type; + user_buffer_format.order = HAILO_FORMAT_ORDER_AUTO; + + hailo_format_flags_t flags = HAILO_FORMAT_FLAGS_NONE; + if (quantized) { + flags = static_cast(flags | HAILO_FORMAT_FLAGS_QUANTIZED); + } + + user_buffer_format.flags = flags; + return user_buffer_format; +} + +hailo_transform_params_t HailoRTDefaults::get_transform_params(bool quantized, hailo_format_type_t format_type) +{ + hailo_transform_params_t params{}; + params.transform_mode = HAILO_STREAM_TRANSFORM_COPY; + params.user_buffer_format = get_user_buffer_format(quantized, format_type); + return params; +} + +hailo_transform_params_t HailoRTDefaults::get_transform_params() +{ + return get_transform_params(true, HAILO_FORMAT_TYPE_AUTO); +} + +hailo_vstream_params_t HailoRTDefaults::get_vstreams_params() +{ + return get_vstreams_params(true, HAILO_FORMAT_TYPE_AUTO); +} + +hailo_vstream_params_t HailoRTDefaults::get_vstreams_params(bool quantized, hailo_format_type_t format_type) +{ + hailo_vstream_params_t params{}; + params.user_buffer_format = get_user_buffer_format(quantized, format_type); + params.queue_size = HAILO_DEFAULT_VSTREAM_QUEUE_SIZE; + params.timeout_ms = HAILO_DEFAULT_VSTREAM_TIMEOUT_MS; + params.vstream_stats_flags = HAILO_VSTREAM_STATS_NONE; + params.pipeline_elements_stats_flags = HAILO_PIPELINE_ELEM_STATS_NONE; + return params; +} + +hailo_transform_params_t HailoRTDefaults::get_transform_params(const hailo_stream_info_t &stream_info) +{ + hailo_transform_params_t params{}; + params.transform_mode = HAILO_STREAM_TRANSFORM_COPY; + params.user_buffer_format.type = stream_info.format.type; + params.user_buffer_format.order = get_default_host_format_order(stream_info.format); + params.user_buffer_format.flags = static_cast( + HAILO_FORMAT_FLAGS_QUANTIZED & + ~HAILO_FORMAT_FLAGS_TRANSPOSED); + return params; +} + +hailo_eth_input_stream_params_t HailoRTDefaults::get_eth_input_stream_params() +{ + hailo_eth_input_stream_params_t params{}; + params.host_address = get_sockaddr(); + params.device_port = HAILO_DEFAULT_ETH_DEVICE_PORT; + params.max_payload_size = HAILO_DEFAULT_ETH_MAX_PAYLOAD_SIZE; + params.is_sync_enabled = false; + params.frames_per_sync = 0; + params.rate_limit_bytes_per_sec = 0; + params.buffers_threshold = HAILO_DEFAULT_BUFFERS_THRESHOLD; + return params; +} + +hailo_eth_output_stream_params_t HailoRTDefaults::get_eth_output_stream_params() +{ + hailo_eth_output_stream_params_t params{}; + params.host_address = get_sockaddr(); + params.device_port = HAILO_DEFAULT_ETH_DEVICE_PORT; + params.max_payload_size = HAILO_DEFAULT_ETH_MAX_PAYLOAD_SIZE; + params.is_sync_enabled = true; + return params; +} + +hailo_pcie_input_stream_params_t HailoRTDefaults::get_pcie_input_stream_params() +{ + hailo_pcie_input_stream_params_t params{}; + return params; +} + +hailo_pcie_output_stream_params_t HailoRTDefaults::get_pcie_output_stream_params() +{ + hailo_pcie_output_stream_params_t params{}; + return params; +} + +hailo_integrated_input_stream_params_t HailoRTDefaults::get_integrated_input_stream_params() +{ + hailo_integrated_input_stream_params_t params{}; + return params; +} + +hailo_integrated_output_stream_params_t HailoRTDefaults::get_integrated_output_stream_params() +{ + hailo_integrated_output_stream_params_t params{}; + return params; +} + +hailo_mipi_input_stream_params_t HailoRTDefaults::get_mipi_input_stream_params() +{ + hailo_mipi_input_stream_params_t params = {}; + + params.mipi_rx_id = 0; + params.data_type = HAILO_MIPI_RX_TYPE_RAW_8; + + params.mipi_common_params.img_width_pixels = 1920; + params.mipi_common_params.img_height_pixels = 1080; + params.mipi_common_params.pixels_per_clock = HAILO_MIPI_PIXELS_PER_CLOCK_4; + params.mipi_common_params.number_of_lanes = 2; + params.mipi_common_params.clock_selection = HAILO_MIPI_CLOCK_SELECTION_AUTOMATIC; + params.mipi_common_params.data_rate = 260; + params.mipi_common_params.virtual_channel_index = 0; + + params.isp_enable = false; + params.isp_params.isp_img_in_order = HAILO_MIPI_ISP_IMG_IN_ORDER_GR_FIRST; + params.isp_params.isp_img_out_data_type = HAILO_MIPI_IMG_OUT_DATA_TYPE_RGB_888; + params.isp_params.isp_crop_enable = false; + params.isp_params.isp_crop_output_width_pixels = 1920; + params.isp_params.isp_crop_output_height_pixels = 1080; + params.isp_params.isp_crop_output_width_start_offset_pixels = 0; + params.isp_params.isp_crop_output_height_start_offset_pixels = 0; + params.isp_params.isp_test_pattern_enable = true; + params.isp_params.isp_configuration_bypass = false; + params.isp_params.isp_run_time_ae_enable = true; + params.isp_params.isp_run_time_awb_enable = true; + params.isp_params.isp_run_time_adt_enable = true; + params.isp_params.isp_run_time_af_enable = false; + params.isp_params.isp_run_time_calculations_interval_ms = 0; + params.isp_params.isp_light_frequency = HAILO_MIPI_ISP_LIGHT_FREQUENCY_50HZ; + + return params; +} + +Expected HailoRTDefaults::get_stream_parameters(hailo_stream_interface_t interface, + hailo_stream_direction_t direction) +{ + hailo_stream_parameters_t params = {}; + params.stream_interface = interface; + params.direction = direction; + switch (params.stream_interface) { + case HAILO_STREAM_INTERFACE_PCIE: + if (HAILO_H2D_STREAM == direction) { + params.pcie_input_params = get_pcie_input_stream_params(); + } else { + params.pcie_output_params = get_pcie_output_stream_params(); + } + break; + case HAILO_STREAM_INTERFACE_INTEGRATED: + if (HAILO_H2D_STREAM == direction) { + params.integrated_input_params = get_integrated_input_stream_params(); + } else { + params.integrated_output_params = get_integrated_output_stream_params(); + } + break; + case HAILO_STREAM_INTERFACE_ETH: + if (HAILO_H2D_STREAM == direction) { + params.eth_input_params = get_eth_input_stream_params(); + } else { + params.eth_output_params = get_eth_output_stream_params(); + } + break; + case HAILO_STREAM_INTERFACE_MIPI: + if (HAILO_H2D_STREAM == direction) { + params.mipi_input_params = get_mipi_input_stream_params(); + break; + } else { + LOGGER__ERROR("Invalid stream interface"); + return make_unexpected(HAILO_INVALID_ARGUMENT); + } + default: + LOGGER__ERROR("Invalid stream interface"); + return make_unexpected(HAILO_INVALID_ARGUMENT); + } + return params; +} + +hailo_activate_network_group_params_t HailoRTDefaults::get_active_network_group_params() +{ + hailo_activate_network_group_params_t params = {}; + return params; +} + +ConfigureNetworkParams HailoRTDefaults::get_configure_params(uint16_t batch_size, hailo_power_mode_t power_mode) +{ + ConfigureNetworkParams params = {}; + params.batch_size = batch_size; + if (std::getenv("FORCE_POWER_MODE_ULTRA_PERFORMANCE") != nullptr) { + power_mode = HAILO_POWER_MODE_ULTRA_PERFORMANCE; + } + params.power_mode = power_mode; + params.latency = HAILO_LATENCY_NONE; + return params; +} + +hailo_network_parameters_t HailoRTDefaults::get_network_parameters(uint16_t batch_size) +{ + hailo_network_parameters_t params = {}; + params.batch_size = batch_size; + + return params; +} + +std::string HailoRTDefaults::get_network_name(const std::string &net_group_name) +{ + std::string default_network_name = net_group_name + + HAILO_DEFAULT_NETWORK_NAME_QUALIFIER + + net_group_name; + + return default_network_name; +} + +hailo_format_t HailoRTDefaults::expand_auto_format(const hailo_format_t &host_format, const hailo_format_t &hw_format) +{ + auto host_format_copy = host_format; + if (HAILO_FORMAT_TYPE_AUTO == host_format_copy.type) { + host_format_copy.type = hw_format.type; + } + if (HAILO_FORMAT_ORDER_AUTO == host_format_copy.order) { + host_format_copy.order = get_default_host_format_order(hw_format); + } + return host_format_copy; +} + +hailo_vdevice_params_t HailoRTDefaults::get_vdevice_params() +{ + hailo_vdevice_params_t params = {}; + params.device_count = HAILO_DEFAULT_DEVICE_COUNT; + params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN; + params.device_ids = nullptr; + params.group_id = HAILO_DEFAULT_VDEVICE_GROUP_ID; + params.multi_process_service = false; + return params; +} + +} /* namespace hailort */ \ No newline at end of file diff --git a/hailort/libhailort/src/hailort_defaults.hpp b/hailort/libhailort/src/hailort_defaults.hpp deleted file mode 100644 index 8cbe348..0000000 --- a/hailort/libhailort/src/hailort_defaults.hpp +++ /dev/null @@ -1,375 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file hailort_defaults.hpp - * @brief - **/ -#ifndef HAILORT_DEFAULTS_HPP_ -#define HAILORT_DEFAULTS_HPP_ - -#include -#include "hailo/expected.hpp" -#include "hailo/network_group.hpp" -#include "hailo/hef.hpp" -#include "common/logger_macros.hpp" -#include "common/utils.hpp" - -namespace hailort -{ - -constexpr hailo_format_order_t DEFAULT_FORMAT_ORDER_MAP[] = { - // Key is device_format_order, value is default user_format_order - HAILO_FORMAT_ORDER_AUTO, // HAILO_FORMAT_ORDER_AUTO, - Should not be used! - HAILO_FORMAT_ORDER_NHWC, // HAILO_FORMAT_ORDER_NHWC, - HAILO_FORMAT_ORDER_NHWC, // HAILO_FORMAT_ORDER_NHCW, - HAILO_FORMAT_ORDER_FCR, // HAILO_FORMAT_ORDER_FCR, - HAILO_FORMAT_ORDER_F8CR, // HAILO_FORMAT_ORDER_F8CR, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NHW, - HAILO_FORMAT_ORDER_NC, // HAILO_FORMAT_ORDER_NC, - HAILO_FORMAT_ORDER_BAYER_RGB, // HAILO_FORMAT_ORDER_BAYER_RGB, - HAILO_FORMAT_ORDER_12_BIT_BAYER_RGB, // HAILO_FORMAT_ORDER_12_BIT_BAYER_RGB - HAILO_FORMAT_ORDER_HAILO_NMS, // HAILO_FORMAT_ORDER_HAILO_NMS, - HAILO_FORMAT_ORDER_NHWC, // HAILO_FORMAT_ORDER_RGB888, - HAILO_FORMAT_ORDER_NCHW, // HAILO_FORMAT_ORDER_NCHW, - HAILO_FORMAT_ORDER_YUY2, // HAILO_FORMAT_ORDER_YUY2, - HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_NV12, - HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_NV21, - HAILO_FORMAT_ORDER_NV12, // HAILO_FORMAT_ORDER_HAILO_YYUV, - HAILO_FORMAT_ORDER_NV21, // HAILO_FORMAT_ORDER_HAILO_YYVU, - HAILO_FORMAT_ORDER_MAX_ENUM // Not used in device side - HAILO_FORMAT_ORDER_RGB4 - }; - -constexpr hailo_format_order_t DEFAULT_FORMAT_ARGMAX_ORDER_MAP[] = { - // Key is device_format_order, value is default user_format_order - HAILO_FORMAT_ORDER_AUTO, // HAILO_FORMAT_ORDER_AUTO, - Should not be used! - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NHWC, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NHCW, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_FCR, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_F8CR, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NHW, - HAILO_FORMAT_ORDER_NC, // HAILO_FORMAT_ORDER_NC, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_BAYER_RGB, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_12_BIT_BAYER_RGB - HAILO_FORMAT_ORDER_HAILO_NMS, // HAILO_FORMAT_ORDER_HAILO_NMS, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_RGB888, - HAILO_FORMAT_ORDER_NHW, // HAILO_FORMAT_ORDER_NCHW, - HAILO_FORMAT_ORDER_YUY2, // HAILO_FORMAT_ORDER_YUY2, - HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_NV12, - HAILO_FORMAT_ORDER_MAX_ENUM, // Not used in device side - HAILO_FORMAT_ORDER_NV21, - HAILO_FORMAT_ORDER_NV12, // HAILO_FORMAT_ORDER_HAILO_YYUV, - HAILO_FORMAT_ORDER_NV21, // HAILO_FORMAT_ORDER_HAILO_YYVU, - HAILO_FORMAT_ORDER_MAX_ENUM // Not used in device side - HAILO_FORMAT_ORDER_RGB4 -}; - - -#define HAILO_DEFAULT_NETWORK_NAME_QUALIFIER (std::string("/")) - - -class HailoRTDefaults -{ -public: - HailoRTDefaults() = delete; - - // This func must be aligned to SDK! - static Expected get_device_format_order(uint32_t compiler_format_order) - { - switch (compiler_format_order) { - case 0: - return std::move(HAILO_FORMAT_ORDER_NHWC); - break; - case 1: - return std::move(HAILO_FORMAT_ORDER_NHCW); - break; - case 2: - return std::move(HAILO_FORMAT_ORDER_NC); - break; - case 3: - return std::move(HAILO_FORMAT_ORDER_FCR); - break; - case 4: - return std::move(HAILO_FORMAT_ORDER_BAYER_RGB); - break; - case 5: - return std::move(HAILO_FORMAT_ORDER_NHW); - break; - case 6: - return std::move(HAILO_FORMAT_ORDER_HAILO_NMS); - break; - case 7: - return std::move(HAILO_FORMAT_ORDER_F8CR); - break; - case 8: - return std::move(HAILO_FORMAT_ORDER_RGB888); - break; - case 11: - return std::move(HAILO_FORMAT_ORDER_YUY2); - break; - case 14: - return std::move(HAILO_FORMAT_ORDER_HAILO_YYUV); - break; - case 15: - return std::move(HAILO_FORMAT_ORDER_HAILO_YYVU); - break; - default: - LOGGER__ERROR("Invalid compiler_format_order ({})", compiler_format_order); - return make_unexpected(HAILO_INTERNAL_FAILURE); - } - } - - static constexpr hailo_format_order_t get_default_host_format_order(const hailo_format_t &device_format) - { - const bool is_argmax = (0 != (device_format.flags & HAILO_FORMAT_FLAGS_HOST_ARGMAX)); - if (!is_argmax) { - return DEFAULT_FORMAT_ORDER_MAP[device_format.order]; - } else { - return DEFAULT_FORMAT_ARGMAX_ORDER_MAP[device_format.order]; - } - } - - static constexpr struct sockaddr_in get_sockaddr() { - struct sockaddr_in address{}; - address.sin_family = AF_INET; - address.sin_port = 0; - address.sin_addr.s_addr = INADDR_ANY; - // sin_zero is already zeroed - return address; - } - - static constexpr hailo_format_t get_user_buffer_format() - { - return get_user_buffer_format(true, HAILO_FORMAT_TYPE_AUTO); - } - - static constexpr hailo_format_t get_user_buffer_format(bool quantized, hailo_format_type_t format_type) - { - hailo_format_t user_buffer_format{}; - user_buffer_format.type = format_type; - user_buffer_format.order = HAILO_FORMAT_ORDER_AUTO; - - hailo_format_flags_t flags = HAILO_FORMAT_FLAGS_NONE; - if (quantized) { - flags = static_cast(flags | HAILO_FORMAT_FLAGS_QUANTIZED); - } - - user_buffer_format.flags = flags; - return user_buffer_format; - } - - static constexpr hailo_transform_params_t get_transform_params(bool quantized, hailo_format_type_t format_type) - { - hailo_transform_params_t params{}; - params.transform_mode = HAILO_STREAM_TRANSFORM_COPY; - params.user_buffer_format = get_user_buffer_format(quantized, format_type); - return params; - } - - static constexpr hailo_transform_params_t get_transform_params() - { - return get_transform_params(true, HAILO_FORMAT_TYPE_AUTO); - } - - static constexpr hailo_vstream_params_t get_vstreams_params() - { - return get_vstreams_params(true, HAILO_FORMAT_TYPE_AUTO); - } - - static constexpr hailo_vstream_params_t get_vstreams_params(bool quantized, hailo_format_type_t format_type) - { - hailo_vstream_params_t params{}; - params.user_buffer_format = get_user_buffer_format(quantized, format_type); - params.queue_size = HAILO_DEFAULT_VSTREAM_QUEUE_SIZE; - params.timeout_ms = HAILO_DEFAULT_VSTREAM_TIMEOUT_MS; - params.vstream_stats_flags = HAILO_VSTREAM_STATS_NONE; - params.pipeline_elements_stats_flags = HAILO_PIPELINE_ELEM_STATS_NONE; - return params; - } - - static constexpr hailo_transform_params_t get_transform_params(const hailo_stream_info_t &stream_info) - { - hailo_transform_params_t params{}; - params.transform_mode = HAILO_STREAM_TRANSFORM_COPY; - params.user_buffer_format.type = stream_info.format.type; - params.user_buffer_format.order = get_default_host_format_order(stream_info.format); - params.user_buffer_format.flags = static_cast( - HAILO_FORMAT_FLAGS_QUANTIZED & - ~HAILO_FORMAT_FLAGS_TRANSPOSED); - return params; - } - - static constexpr hailo_eth_input_stream_params_t get_eth_input_stream_params() { - hailo_eth_input_stream_params_t params{}; - params.host_address = get_sockaddr(); - params.device_port = HAILO_DEFAULT_ETH_DEVICE_PORT; - params.max_payload_size = HAILO_DEFAULT_ETH_MAX_PAYLOAD_SIZE; - params.is_sync_enabled = false; - params.frames_per_sync = 0; - params.rate_limit_bytes_per_sec = 0; - params.buffers_threshold = HAILO_DEFAULT_BUFFERS_THRESHOLD; - return params; - } - - static constexpr hailo_eth_output_stream_params_t get_eth_output_stream_params() { - hailo_eth_output_stream_params_t params{}; - params.host_address = get_sockaddr(); - params.device_port = HAILO_DEFAULT_ETH_DEVICE_PORT; - params.max_payload_size = HAILO_DEFAULT_ETH_MAX_PAYLOAD_SIZE; - params.is_sync_enabled = true; - return params; - } - - - static constexpr hailo_pcie_input_stream_params_t get_pcie_input_stream_params() { - hailo_pcie_input_stream_params_t params{}; - return params; - } - - static constexpr hailo_pcie_output_stream_params_t get_pcie_output_stream_params() { - hailo_pcie_output_stream_params_t params{}; - return params; - } - - static constexpr hailo_core_input_stream_params_t get_core_input_stream_params() { - hailo_core_input_stream_params_t params{}; - return params; - } - - static constexpr hailo_core_output_stream_params_t get_core_output_stream_params() { - hailo_core_output_stream_params_t params{}; - return params; - } - - static constexpr hailo_mipi_input_stream_params_t get_mipi_input_stream_params() - { - hailo_mipi_input_stream_params_t params = {}; - - params.mipi_rx_id = 0; - params.data_type = HAILO_MIPI_RX_TYPE_RAW_8; - - params.mipi_common_params.img_width_pixels = 1920; - params.mipi_common_params.img_height_pixels = 1080; - params.mipi_common_params.pixels_per_clock = HAILO_MIPI_PIXELS_PER_CLOCK_4; - params.mipi_common_params.number_of_lanes = 2; - params.mipi_common_params.clock_selection = HAILO_MIPI_CLOCK_SELECTION_AUTOMATIC; - params.mipi_common_params.data_rate = 260; - params.mipi_common_params.virtual_channel_index = 0; - - params.isp_enable = false; - params.isp_params.isp_img_in_order = HAILO_MIPI_ISP_IMG_IN_ORDER_GR_FIRST; - params.isp_params.isp_img_out_data_type = HAILO_MIPI_IMG_OUT_DATA_TYPE_RGB_888; - params.isp_params.isp_crop_enable = false; - params.isp_params.isp_crop_output_width_pixels = 1920; - params.isp_params.isp_crop_output_height_pixels = 1080; - params.isp_params.isp_crop_output_width_start_offset_pixels = 0; - params.isp_params.isp_crop_output_height_start_offset_pixels = 0; - params.isp_params.isp_test_pattern_enable = true; - params.isp_params.isp_configuration_bypass = false; - params.isp_params.isp_run_time_ae_enable = true; - params.isp_params.isp_run_time_awb_enable = true; - params.isp_params.isp_run_time_adt_enable = true; - params.isp_params.isp_run_time_af_enable = false; - params.isp_params.isp_run_time_calculations_interval_ms = 0; - params.isp_params.isp_light_frequency = HAILO_MIPI_ISP_LIGHT_FREQUENCY_50HZ; - - return params; - } - - static Expected get_stream_parameters(hailo_stream_interface_t interface, - hailo_stream_direction_t direction) - { - hailo_stream_parameters_t params = {}; - params.stream_interface = interface; - params.direction = direction; - switch (params.stream_interface) { - case HAILO_STREAM_INTERFACE_PCIE: - if (HAILO_H2D_STREAM == direction) { - params.pcie_input_params = get_pcie_input_stream_params(); - } else { - params.pcie_output_params = get_pcie_output_stream_params(); - } - break; - case HAILO_STREAM_INTERFACE_CORE: - if (HAILO_H2D_STREAM == direction) { - params.core_input_params = get_core_input_stream_params(); - } else { - params.core_output_params = get_core_output_stream_params(); - } - break; - case HAILO_STREAM_INTERFACE_ETH: - if (HAILO_H2D_STREAM == direction) { - params.eth_input_params = get_eth_input_stream_params(); - } else { - params.eth_output_params = get_eth_output_stream_params(); - } - break; - default: - LOGGER__ERROR("Invalid stream interface"); - return make_unexpected(HAILO_INVALID_ARGUMENT); - } - return params; - } - - static hailo_activate_network_group_params_t get_network_group_params() - { - hailo_activate_network_group_params_t params = {}; - return params; - } - - static ConfigureNetworkParams get_configure_params(uint16_t batch_size = HAILO_DEFAULT_BATCH_SIZE, - hailo_power_mode_t power_mode = HAILO_POWER_MODE_PERFORMANCE) - { - ConfigureNetworkParams params = {}; - params.batch_size = batch_size; - if (std::getenv("FORCE_POWER_MODE_ULTRA_PERFORMANCE") != nullptr) { - power_mode = HAILO_POWER_MODE_ULTRA_PERFORMANCE; - } - params.power_mode = power_mode; - params.latency = HAILO_LATENCY_NONE; - return params; - } - - static hailo_network_parameters_t get_network_parameters(uint16_t batch_size = HAILO_DEFAULT_BATCH_SIZE) - { - hailo_network_parameters_t params = {}; - params.batch_size = batch_size; - - return params; - } - - static std::string get_network_name(const std::string &net_group_name) - { - std::string default_network_name = net_group_name + - HAILO_DEFAULT_NETWORK_NAME_QUALIFIER + - net_group_name; - - return default_network_name; - } - - static hailo_format_t expand_auto_format(const hailo_format_t &host_format, const hailo_format_t &hw_format) - { - auto host_format_copy = host_format; - if (HAILO_FORMAT_TYPE_AUTO == host_format_copy.type) { - host_format_copy.type = hw_format.type; - } - if (HAILO_FORMAT_ORDER_AUTO == host_format_copy.order) { - host_format_copy.order = get_default_host_format_order(hw_format); - } - return host_format_copy; - } - - static hailo_vdevice_params_t get_vdevice_params() - { - hailo_vdevice_params_t params = {}; - params.device_count = HAILO_DEFAULT_DEVICE_COUNT; - params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN; - params.device_ids = nullptr; - params.group_id = HAILO_DEFAULT_VDEVICE_GROUP_ID; - params.multi_process_service = false; - return params; - } -}; - -} /* namespace hailort */ - -#endif /* HAILORT_DEFAULTS_HPP_ */ diff --git a/hailort/libhailort/src/hef/CMakeLists.txt b/hailort/libhailort/src/hef/CMakeLists.txt new file mode 100644 index 0000000..48cb509 --- /dev/null +++ b/hailort/libhailort/src/hef/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/hef.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/core_op_metadata.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/context_switch_actions.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/context_switch/context_switch_actions.cpp b/hailort/libhailort/src/hef/context_switch_actions.cpp similarity index 87% rename from hailort/libhailort/src/context_switch/context_switch_actions.cpp rename to hailort/libhailort/src/hef/context_switch_actions.cpp index d9ad378..eda6184 100644 --- a/hailort/libhailort/src/context_switch/context_switch_actions.cpp +++ b/hailort/libhailort/src/hef/context_switch_actions.cpp @@ -9,8 +9,9 @@ **/ #include "context_switch_actions.hpp" +#include "core_op/resource_manager/resource_manager.hpp" + #include "context_switch_defs.h" -#include "context_switch/multi_context/resource_manager.hpp" namespace hailort { @@ -37,13 +38,12 @@ ContextSwitchConfigAction::ContextSwitchConfigAction(Type type, CONTEXT_SWITCH_D m_action_list_type(action_list_type) {} -Expected> ContextSwitchConfigAction::serialize(const ContextResources &context_resources, - bool is_repeated /*=false*/) const +Expected> ContextSwitchConfigAction::serialize(const ContextResources &context_resources) const { CHECK_AS_EXPECTED(m_action_list_type < CONTEXT_SWITCH_DEFS__ACTION_TYPE_COUNT, HAILO_INTERNAL_FAILURE, "Action cannot be serialized"); - auto header = serialize_header(is_repeated); + auto header = serialize_header(); CHECK_EXPECTED(header); auto params = serialize_params(context_resources); @@ -70,16 +70,14 @@ CONTEXT_SWITCH_DEFS__ACTION_TYPE_t ContextSwitchConfigAction::get_action_list_ty return m_action_list_type; } -Expected ContextSwitchConfigAction::serialize_header(bool is_repeated) const +Expected ContextSwitchConfigAction::serialize_header() const { CHECK_AS_EXPECTED(m_action_list_type != CONTEXT_SWITCH_DEFS__ACTION_TYPE_COUNT, HAILO_INTERNAL_FAILURE, "Action cannot be serialized"); - - auto header = Buffer::create(sizeof(CONTROL_PROTOCOL__ACTION_HEADER_t), 0); - CHECK_EXPECTED(header); - header->as_type().action_type = m_action_list_type; - header->as_type().is_repeated = is_repeated; // TODO: prettier - return header.release(); + CONTEXT_SWITCH_DEFS__common_action_header_t header{}; + header.action_type = m_action_list_type; + header.time_stamp = CONTEXT_SWITCH_DEFS__TIMESTAMP_INIT_VALUE; + return Buffer::create(reinterpret_cast(&header), sizeof(header)); } Expected NoneAction::create() @@ -93,7 +91,7 @@ NoneAction::NoneAction() : ContextSwitchConfigAction(Type::None) {} -Expected> NoneAction::serialize(const ContextResources &, bool) const +Expected> NoneAction::serialize(const ContextResources &) const { // Do nothing return std::vector(); @@ -175,24 +173,28 @@ Expected DeactivateConfigChannelAction::serialize_params(const ContextRe } Expected WriteDataCcwAction::create( - Buffer &&data, uint8_t config_stream_index) + Buffer &&data, uint8_t config_stream_index, size_t total_ccw_burst) { + CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(total_ccw_burst), HAILO_INVALID_HEF, + "Too many ccw burst {} (must fit in uint16)", total_ccw_burst); auto result = ContextSwitchConfigActionPtr(new (std::nothrow) WriteDataCcwAction( - std::move(data), config_stream_index)); + std::move(data), config_stream_index, static_cast(total_ccw_burst))); CHECK_AS_EXPECTED((nullptr != result), HAILO_OUT_OF_HOST_MEMORY); return result; } -WriteDataCcwAction::WriteDataCcwAction(Buffer &&data, uint8_t config_stream_index) : +WriteDataCcwAction::WriteDataCcwAction(Buffer &&data, uint8_t config_stream_index, uint16_t total_ccw_burst) : ContextSwitchConfigAction(Type::WriteDataCcw), m_data(std::move(data)), - m_config_stream_index(config_stream_index) + m_config_stream_index(config_stream_index), + m_total_ccw_burst(total_ccw_burst) {} -Expected> WriteDataCcwAction::serialize(const ContextResources &, bool) const +Expected> WriteDataCcwAction::serialize(const ContextResources &) const { - // WriteDataCcwActions aren't written to the FW's action list. Hence the execute will do nothing. - return std::vector(); + // WriteDataCcwActions aren't written to the FW's action list. + LOGGER__ERROR("Can't serialize WriteDataCcwAction"); + return make_unexpected(HAILO_INTERNAL_FAILURE); } bool WriteDataCcwAction::supports_repeated_block() const @@ -233,10 +235,12 @@ bool AddCcwBurstAction::supports_repeated_block() const } Expected FetchCfgChannelDescriptorsAction::create(const vdma::ChannelId &channel_id, - uint16_t desc_count) + size_t desc_count) { + CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(desc_count), HAILO_INVALID_OPERATION, + "On cfg with continuous mode, max descriptors size must fit in uint16_t"); auto result = ContextSwitchConfigActionPtr(new (std::nothrow) FetchCfgChannelDescriptorsAction(channel_id, - desc_count)); + static_cast(desc_count))); CHECK_AS_EXPECTED((nullptr != result), HAILO_OUT_OF_HOST_MEMORY); return result; } @@ -344,10 +348,8 @@ Expected RepeatedAction::serialize_params(const ContextResources &) cons return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } -Expected> RepeatedAction::serialize(const ContextResources &context_resources, - bool is_repeated/*=false*/) const +Expected> RepeatedAction::serialize(const ContextResources &context_resources) const { - CHECK_AS_EXPECTED(!is_repeated, HAILO_INTERNAL_FAILURE, "Cant use recursive repeated"); std::vector buffers; buffers.reserve(m_actions.size() + 1); // Contains the repeated header and all of the actions @@ -358,11 +360,9 @@ Expected> RepeatedAction::serialize(const ContextResources & for (const auto &action : m_actions) { assert(action->get_action_list_type() == m_sub_action_type); - const bool REPEATED = true; - auto action_buffers = action->serialize(context_resources, REPEATED); - CHECK_EXPECTED(action_buffers); - CHECK_AS_EXPECTED(action_buffers->size() == 1, HAILO_INTERNAL_FAILURE, "Sub action should contain one buffer"); - buffers.emplace_back(std::move(action_buffers->at(0))); + auto action_buffer = action->serialize_params(context_resources); + CHECK_EXPECTED(action_buffer); + buffers.emplace_back(action_buffer.release()); } return buffers; @@ -571,37 +571,32 @@ bool AllowInputDataflowAction::supports_repeated_block() const Expected AllowInputDataflowAction::serialize_params(const ContextResources &context_resources) const { - for (const auto &edge_layer : context_resources.get_boundary_layers()) { - if (m_stream_index == edge_layer.layer_info.stream_index) { - CONTEXT_SWITCH_DEFS__fetch_data_action_data_t params{}; - params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id); - params.stream_index = m_stream_index; - params.network_index = edge_layer.layer_info.network_index; - params.credit_type = CONTEXT_SWITCH_DEFS__CREDIT_IN_BYTES; - params.host_buffer_type = CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t(edge_layer.buffer_info.buffer_type); - params.periph_bytes_per_buffer = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer; - params.frame_periph_size = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer * - edge_layer.layer_info.nn_stream_config.periph_buffers_per_frame; - return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); - } - } + const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index); + CHECK_EXPECTED(edge_layer); - for (const auto &edge_layer : context_resources.get_inter_context_layers()) { - if (m_stream_index == edge_layer.layer_info.stream_index) { - CONTEXT_SWITCH_DEFS__fetch_data_action_data_t params{}; - params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer.channel_id); - params.stream_index = m_stream_index; - params.network_index = edge_layer.layer_info.network_index; - params.credit_type = CONTEXT_SWITCH_DEFS__CREDIT_IN_DESCRIPTORS; - params.host_buffer_type = CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t(edge_layer.buffer_info.buffer_type); - params.periph_bytes_per_buffer = edge_layer.layer_info.nn_stream_config.periph_bytes_per_buffer; - params.frame_periph_size = ((edge_layer.buffer_info.bytes_in_pattern - 1) / (edge_layer.buffer_info.desc_page_size)) + 1; - return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); - } + CONTEXT_SWITCH_DEFS__fetch_data_action_data_t params{}; + params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id); + params.stream_index = m_stream_index; + params.network_index = edge_layer->layer_info.network_index; + params.host_buffer_type = static_cast(edge_layer->buffer_info.buffer_type); + params.periph_bytes_per_buffer = edge_layer->layer_info.nn_stream_config.periph_bytes_per_buffer; + + switch (edge_layer->layer_info.type) { + case LayerType::BOUNDARY: + params.credit_type = CONTEXT_SWITCH_DEFS__CREDIT_IN_BYTES; + params.frame_periph_size = edge_layer->layer_info.nn_stream_config.periph_bytes_per_buffer * + edge_layer->layer_info.nn_stream_config.periph_buffers_per_frame; + break; + case LayerType::INTER_CONTEXT: + params.credit_type = CONTEXT_SWITCH_DEFS__CREDIT_IN_DESCRIPTORS; + params.frame_periph_size = ((edge_layer->buffer_info.bytes_in_pattern - 1) / (edge_layer->buffer_info.desc_page_size)) + 1; + break; + default: + LOGGER__ERROR("Invalid layer type {} for stream {}", static_cast(edge_layer->layer_info.type), m_stream_index); + return make_unexpected(HAILO_INTERNAL_FAILURE); } - LOGGER__ERROR("Stream {} not found in edge layers", m_stream_index); - return make_unexpected(HAILO_INTERNAL_FAILURE); + return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } Expected WaitForModuleConfigDoneAction::create(uint8_t module_index) @@ -762,33 +757,14 @@ bool WaitOutputTransferDoneAction::supports_repeated_block() const Expected WaitOutputTransferDoneAction::serialize_params(const ContextResources &context_resources) const { - const auto channel_id = get_layer_channel_id(context_resources); - CHECK_EXPECTED(channel_id); + const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index); + CHECK_EXPECTED(edge_layer); CONTEXT_SWITCH_DEFS__vdma_dataflow_interrupt_data_t params{}; - params.packed_vdma_channel_id = pack_vdma_channel_id(channel_id.value()); + params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id); return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } -Expected WaitOutputTransferDoneAction::get_layer_channel_id(const ContextResources &context_resources) const -{ - // TODO: HRT-8611 use one loop - for (const auto &edge_layer : context_resources.get_boundary_layers()) { - if (m_stream_index == edge_layer.layer_info.stream_index) { - return vdma::ChannelId(edge_layer.channel_id); - } - } - - for (const auto &edge_layer : context_resources.get_inter_context_layers()) { - if (m_stream_index == edge_layer.layer_info.stream_index) { - return vdma::ChannelId(edge_layer.channel_id); - } - } - - LOGGER__ERROR("Stream {} not found in edge layers", m_stream_index); - return make_unexpected(HAILO_INTERNAL_FAILURE); -} - Expected OpenBoundaryInputChannelAction::create(const vdma::ChannelId &channel_id, const CONTROL_PROTOCOL__host_buffer_info_t &host_buffer_info) { @@ -1101,12 +1077,13 @@ Expected ActivateDdrOutputChannelAction::serialize_params(const ContextR return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } -Expected ValidateChannelAction::create(const vdma::ChannelId &channel_id, - hailo_stream_direction_t stream_direction, bool is_inter_context, - CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t host_buffer_type, uint32_t initial_credit_size) +Expected ValidateChannelAction::create(const EdgeLayer &edge_layer) { - auto result = ContextSwitchConfigActionPtr(new (std::nothrow) ValidateChannelAction(channel_id, stream_direction, - is_inter_context, host_buffer_type, initial_credit_size)); + const bool is_inter_context = (LayerType::INTER_CONTEXT == edge_layer.layer_info.type); + auto result = ContextSwitchConfigActionPtr(new (std::nothrow) ValidateChannelAction(edge_layer.channel_id, + edge_layer.layer_info.direction, is_inter_context, + static_cast(edge_layer.buffer_info.buffer_type), + edge_layer.layer_info.max_shmifo_size)); CHECK_AS_EXPECTED((nullptr != result), HAILO_OUT_OF_HOST_MEMORY); return result; } @@ -1142,12 +1119,13 @@ Expected ValidateChannelAction::serialize_params(const ContextResources return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } -Expected DeactivateChannelAction::create(const vdma::ChannelId &channel_id, - hailo_stream_direction_t stream_direction, bool is_inter_context, - CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t host_buffer_type, uint32_t initial_credit_size) +Expected DeactivateChannelAction::create(const EdgeLayer &edge_layer) { - auto result = ContextSwitchConfigActionPtr(new (std::nothrow) DeactivateChannelAction(channel_id, stream_direction, - is_inter_context, host_buffer_type, initial_credit_size)); + const bool is_inter_context = (LayerType::INTER_CONTEXT == edge_layer.layer_info.type); + auto result = ContextSwitchConfigActionPtr(new (std::nothrow) DeactivateChannelAction(edge_layer.channel_id, + edge_layer.layer_info.direction, is_inter_context, + static_cast(edge_layer.buffer_info.buffer_type), + edge_layer.layer_info.max_shmifo_size)); CHECK_AS_EXPECTED((nullptr != result), HAILO_OUT_OF_HOST_MEMORY); return result; } @@ -1203,40 +1181,16 @@ bool WaitDmaIdleAction::supports_repeated_block() const Expected WaitDmaIdleAction::serialize_params(const ContextResources &context_resources) const { - auto channel_and_type = get_layer_channel_id_and_type(context_resources); - CHECK_EXPECTED(channel_and_type); - - const auto channel_id = channel_and_type->first; - assert(LayerType::INTER_CONTEXT == channel_and_type->second || LayerType::BOUNDARY == channel_and_type->second); - const bool is_inter_context = (LayerType::INTER_CONTEXT == channel_and_type->second); + const auto edge_layer = context_resources.get_edge_layer_by_stream_index(m_stream_index); + CHECK_EXPECTED(edge_layer); CONTEXT_SWITCH_DEFS__wait_dma_idle_data_t params{}; - params.packed_vdma_channel_id = pack_vdma_channel_id(channel_id); - params.is_inter_context = static_cast(is_inter_context); + params.packed_vdma_channel_id = pack_vdma_channel_id(edge_layer->channel_id); + params.is_inter_context = static_cast(LayerType::INTER_CONTEXT == edge_layer->layer_info.type); params.stream_index = m_stream_index; return Buffer::create(reinterpret_cast(¶ms), sizeof(params)); } -Expected> WaitDmaIdleAction::get_layer_channel_id_and_type( - const ContextResources &context_resources) const -{ - // TODO: HRT-8611 use one loop - for (const auto &edge_layer : context_resources.get_boundary_layers()) { - if (m_stream_index == edge_layer.layer_info.stream_index) { - return std::make_pair(edge_layer.channel_id, LayerType::BOUNDARY); - } - } - - for (const auto &edge_layer : context_resources.get_inter_context_layers()) { - if (m_stream_index == edge_layer.layer_info.stream_index) { - return std::make_pair(edge_layer.channel_id, LayerType::INTER_CONTEXT); - } - } - - LOGGER__ERROR("Stream {} not found in edge layers (as boundary or inter context)", m_stream_index); - return make_unexpected(HAILO_INTERNAL_FAILURE); -} - Expected WaitNmsIdleAction::create(uint8_t aggregator_index, uint8_t pred_cluster_ob_index, uint8_t pred_cluster_ob_cluster_index, uint8_t pred_cluster_ob_interface, uint8_t succ_prepost_ob_index, uint8_t succ_prepost_ob_interface) @@ -1303,25 +1257,4 @@ bool EnableNmsAction::supports_repeated_block() const return true; } -ContextSwitchOperation::ContextSwitchOperation(std::vector &&actions) : - m_actions(std::move(actions)) -{} - -const std::vector &ContextSwitchOperation::actions() const -{ - return m_actions; -} - -std::vector ContextSwitchOperation::get_actions_of_type( - const std::set &action_types) const -{ - std::vector filtered_actions; - for (const auto &action : m_actions) { - if (action_types.find(action->get_type()) != action_types.end()) { - filtered_actions.push_back(action); - } - } - return filtered_actions; -} - } /* namespace hailort */ diff --git a/hailort/libhailort/src/context_switch/context_switch_actions.hpp b/hailort/libhailort/src/hef/context_switch_actions.hpp similarity index 94% rename from hailort/libhailort/src/context_switch/context_switch_actions.hpp rename to hailort/libhailort/src/hef/context_switch_actions.hpp index 5b60fc7..155958f 100644 --- a/hailort/libhailort/src/context_switch/context_switch_actions.hpp +++ b/hailort/libhailort/src/hef/context_switch_actions.hpp @@ -13,16 +13,20 @@ #include "hailo/expected.hpp" #include "hailo/buffer.hpp" -#include "vdma/channel_id.hpp" -#include "layer_info.hpp" -#include "control_protocol.hpp" + +#include "vdma/channel/channel_id.hpp" +#include "hef/layer_info.hpp" + +#include "device_common/control_protocol.hpp" #include "context_switch_defs.h" + namespace hailort { class ContextResources; +struct EdgeLayer; class ContextSwitchConfigAction; using ContextSwitchConfigActionPtr = std::shared_ptr; @@ -77,8 +81,10 @@ public: // Serialize the action a vector of buffers - each buffer is a chunk that must be sent continuously to the firmware // (For example each chunk can be sub action of RepeatedAction). - virtual Expected> serialize(const ContextResources &context_resources, - bool is_repeated=false) const; + virtual Expected> serialize(const ContextResources &context_resources) const; + + Expected serialize_header() const; + virtual Expected serialize_params(const ContextResources &context_resources) const = 0; virtual bool supports_repeated_block() const = 0; Type get_type() const; @@ -88,9 +94,6 @@ protected: ContextSwitchConfigAction(Type type); ContextSwitchConfigAction(Type type, CONTEXT_SWITCH_DEFS__ACTION_TYPE_t action_list_type); - Expected serialize_header(bool is_repeated) const; - virtual Expected serialize_params(const ContextResources &context_resources) const = 0; - const Type m_type; const CONTEXT_SWITCH_DEFS__ACTION_TYPE_t m_action_list_type; }; @@ -105,8 +108,7 @@ public: NoneAction &operator=(const NoneAction &) = delete; virtual ~NoneAction() = default; - virtual Expected> serialize(const ContextResources &context_resources, - bool is_repeated=false) const override; + virtual Expected> serialize(const ContextResources &context_resources) const override; virtual bool supports_repeated_block() const override; virtual Expected serialize_params(const ContextResources &context_resources) const override; @@ -150,26 +152,29 @@ private: class WriteDataCcwAction : public ContextSwitchConfigAction { public: - static Expected create(Buffer &&data, uint8_t config_stream_index); + static Expected create(Buffer &&data, uint8_t config_stream_index, + size_t total_ccw_burst); WriteDataCcwAction(WriteDataCcwAction &&) = default; WriteDataCcwAction(const WriteDataCcwAction &) = delete; WriteDataCcwAction &operator=(WriteDataCcwAction &&) = delete; WriteDataCcwAction &operator=(const WriteDataCcwAction &) = delete; virtual ~WriteDataCcwAction() = default; - virtual Expected> serialize(const ContextResources &context_resources, - bool is_repeated=false) const override; + virtual Expected> serialize(const ContextResources &context_resources) const override; virtual bool supports_repeated_block() const override; virtual Expected serialize_params(const ContextResources &context_resources) const override; - uint8_t config_stream_index() const { return m_config_stream_index; } const MemoryView data() const { return MemoryView::create_const(m_data.data(), m_data.size()); } + uint8_t config_stream_index() const { return m_config_stream_index; } + uint16_t total_ccw_burst() const { return m_total_ccw_burst; } private: - WriteDataCcwAction(Buffer &&data, uint8_t config_stream_index); + WriteDataCcwAction(Buffer &&data, uint8_t config_stream_index, + uint16_t total_ccw_burst); Buffer m_data; const uint8_t m_config_stream_index; + const uint16_t m_total_ccw_burst; }; class AddCcwBurstAction : public ContextSwitchConfigAction @@ -189,7 +194,7 @@ private: class FetchCfgChannelDescriptorsAction : public ContextSwitchConfigAction { public: - static Expected create(const vdma::ChannelId &channel_id, uint16_t desc_count); + static Expected create(const vdma::ChannelId &channel_id, size_t desc_count); FetchCfgChannelDescriptorsAction(FetchCfgChannelDescriptorsAction &&) = default; FetchCfgChannelDescriptorsAction(const FetchCfgChannelDescriptorsAction &) = delete; @@ -247,8 +252,7 @@ public: virtual bool supports_repeated_block() const override; virtual Expected serialize_params(const ContextResources &context_resources) const override; - virtual Expected> serialize(const ContextResources &context_resources, - bool is_repeated=false) const override; + virtual Expected> serialize(const ContextResources &context_resources) const override; private: RepeatedAction(std::vector &&actions); @@ -490,8 +494,6 @@ public: private: explicit WaitOutputTransferDoneAction(uint8_t stream_index); - Expected get_layer_channel_id(const ContextResources &context_resources) const; - uint8_t m_stream_index; }; @@ -668,9 +670,7 @@ private: class ValidateChannelAction : public ContextSwitchConfigAction { public: - static Expected create(const vdma::ChannelId &channel_id, - hailo_stream_direction_t stream_direction, bool is_inter_context, - CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t host_buffer_type, uint32_t initial_credit_size); + static Expected create(const EdgeLayer &edge_layer); virtual bool supports_repeated_block() const override; virtual Expected serialize_params(const ContextResources &context_resources) const override; @@ -689,9 +689,7 @@ private: class DeactivateChannelAction : public ContextSwitchConfigAction { public: - static Expected create(const vdma::ChannelId &channel_id, - hailo_stream_direction_t stream_direction, bool is_inter_context, - CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t host_buffer_type, uint32_t initial_credit_size); + static Expected create(const EdgeLayer &edge_layer); virtual bool supports_repeated_block() const override; virtual Expected serialize_params(const ContextResources &context_resources) const override; @@ -718,9 +716,6 @@ public: private: explicit WaitDmaIdleAction(uint8_t stream_index); - Expected> get_layer_channel_id_and_type( - const ContextResources &context_resources) const; - uint8_t m_stream_index; }; @@ -765,17 +760,6 @@ private: const uint8_t m_network_index; }; -class ContextSwitchOperation final { -public: - ContextSwitchOperation(std::vector &&actions); - - const std::vector &actions() const; - std::vector get_actions_of_type(const std::set &action_types) const; - -private: - std::vector m_actions; -}; - } /* namespace hailort */ #endif /* _HAILO_CONTEXT_SWITCH_ACTIONS_HPP_ */ diff --git a/hailort/libhailort/src/network_group_metadata.cpp b/hailort/libhailort/src/hef/core_op_metadata.cpp similarity index 67% rename from hailort/libhailort/src/network_group_metadata.cpp rename to hailort/libhailort/src/hef/core_op_metadata.cpp index 32a61c9..8d148d9 100644 --- a/hailort/libhailort/src/network_group_metadata.cpp +++ b/hailort/libhailort/src/hef/core_op_metadata.cpp @@ -3,11 +3,12 @@ * Distributed under the MIT license (https://opensource.org/licenses/MIT) **/ /** - * @file network_group_metadata.cpp - * @brief Contains all relevant information about a network group from the hef. + * @file core_op_metadata.cpp + * @brief Contains all relevant information about a core-op from the hef. **/ -#include "network_group_metadata.hpp" +#include "core_op_metadata.hpp" +#include namespace hailort { @@ -49,37 +50,32 @@ static bool is_edge_under_mux(const LayerInfo &info, const std::string &edge_nam return false; } - -PreliminaryContextMetadata::PreliminaryContextMetadata(std::vector &&operations, +ContextMetadata::ContextMetadata(std::vector &&actions, ConfigBufferInfoMap&& config_buffers_info) : - m_operations(std::move(operations)), + m_actions(std::move(actions)), m_config_buffers_info(std::move(config_buffers_info)) {} -const std::vector &PreliminaryContextMetadata::get_operations() const -{ - return m_operations; -} - -const ConfigBufferInfoMap &PreliminaryContextMetadata::config_buffers_info() const +const ConfigBufferInfoMap &ContextMetadata::config_buffers_info() const { return m_config_buffers_info; } -ContextMetadata::ContextMetadata(std::vector &&operations, - ConfigBufferInfoMap&& config_buffers_info) : - m_operations(std::move(operations)), - m_config_buffers_info(std::move(config_buffers_info)) -{} - -const std::vector &ContextMetadata::get_operations() const +const std::vector &ContextMetadata::get_actions() const { - return m_operations; + return m_actions; } -const ConfigBufferInfoMap &ContextMetadata::config_buffers_info() const +std::vector ContextMetadata::get_actions_of_type( + const std::set &action_types) const { - return m_config_buffers_info; + std::vector filtered_actions; + for (const auto &action : m_actions) { + if (action_types.find(action->get_type()) != action_types.end()) { + filtered_actions.emplace_back(action); + } + } + return filtered_actions; } void ContextMetadata::add_boundary_layer(const LayerInfo &layer_info) @@ -139,19 +135,62 @@ const std::vector &ContextMetadata::get_ddr_output_layers() const return m_ddr_output_layers; } -NetworkGroupMetadata::NetworkGroupMetadata(const std::string &network_group_name, - PreliminaryContextMetadata &&preliminary_context, +Expected ContextMetadata::get_layers_transfer_size(const std::vector &layer_infos) const +{ + size_t total_transfer_size = 0; + for (const auto &layer_info : layer_infos) { + auto transfer_size = LayerInfoUtils::get_transfer_size(layer_info); + CHECK_EXPECTED(transfer_size); + total_transfer_size += transfer_size.release(); + } + return total_transfer_size; +} + +Expected ContextMetadata::get_context_transfer_size() const +{ + size_t total_transfer_size = 0; + + // Calc config buffers + for (const auto &config_buffer_sizes : m_config_buffers_info) { + total_transfer_size += std::accumulate(config_buffer_sizes.second.begin(), config_buffer_sizes.second.end(), 0); + } + + // Calc all edge layers + auto boundary_input_transfer_size = get_layers_transfer_size(m_boundary_input_layers); + CHECK_EXPECTED(boundary_input_transfer_size); + auto boundary_output_transfer_size = get_layers_transfer_size(m_boundary_output_layers); + CHECK_EXPECTED(boundary_output_transfer_size); + auto ddr_input_transfer_size = get_layers_transfer_size(m_ddr_input_layers); + CHECK_EXPECTED(ddr_input_transfer_size); + auto ddr_output_transfer_size = get_layers_transfer_size(m_ddr_output_layers); + CHECK_EXPECTED(ddr_output_transfer_size); + auto inter_context_input_transfer_size = get_layers_transfer_size(m_inter_context_input_layers); + CHECK_EXPECTED(inter_context_input_transfer_size); + auto inter_context_output_transfer_size = get_layers_transfer_size(m_inter_context_output_layers); + CHECK_EXPECTED(inter_context_output_transfer_size); + + total_transfer_size += + boundary_input_transfer_size.release() + boundary_output_transfer_size.release() + + ddr_input_transfer_size.release() + ddr_output_transfer_size.release() + + inter_context_input_transfer_size.release() + inter_context_output_transfer_size.release(); + + return total_transfer_size; +} + +CoreOpMetadata::CoreOpMetadata(const std::string &core_op_name, + ContextMetadata &&preliminary_context, std::vector &&dynamic_contexts, std::vector &&config_channels_info, std::vector &&sorted_output_names, - SupportedFeatures &supported_features, const std::vector &sorted_network_names) + SupportedFeatures &supported_features, + const std::vector &sorted_network_names) : m_preliminary_context(std::move(preliminary_context)), m_dynamic_contexts(std::move(dynamic_contexts)), m_config_channels_info(std::move(config_channels_info)), - m_network_group_name(network_group_name), m_sorted_output_names(std::move(sorted_output_names)), + m_core_op_name(core_op_name), m_sorted_output_names(std::move(sorted_output_names)), m_supported_features(supported_features), m_sorted_network_names(sorted_network_names) {} -Expected NetworkGroupMetadata::get_layer_info_by_stream_name(const std::string &stream_name) const +Expected CoreOpMetadata::get_layer_info_by_stream_name(const std::string &stream_name) const { for (auto layer_info : get_all_layer_infos()) { if (layer_info.name == stream_name) { @@ -162,7 +201,7 @@ Expected NetworkGroupMetadata::get_layer_info_by_stream_name(const st return make_unexpected(HAILO_NOT_FOUND); } -std::vector NetworkGroupMetadata::get_input_layer_infos() const +std::vector CoreOpMetadata::get_input_layer_infos() const { std::vector res; // Edge layers exists only in the dynamic context. @@ -174,7 +213,7 @@ std::vector NetworkGroupMetadata::get_input_layer_infos() const return res; } -std::vector NetworkGroupMetadata::get_output_layer_infos() const +std::vector CoreOpMetadata::get_output_layer_infos() const { std::vector res; // Edge layers exists only in the dynamic context. @@ -186,7 +225,7 @@ std::vector NetworkGroupMetadata::get_output_layer_infos() const return res; } -std::vector NetworkGroupMetadata::get_all_layer_infos() const +std::vector CoreOpMetadata::get_all_layer_infos() const { const auto input_layer_infos = get_input_layer_infos(); const auto output_layer_infos = get_output_layer_infos(); @@ -199,7 +238,7 @@ std::vector NetworkGroupMetadata::get_all_layer_infos() const return res; } -Expected> NetworkGroupMetadata::get_input_layer_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_input_layer_infos(const std::string &network_name) const { std::vector res; // Edge layers exists only in the dynamic context. @@ -214,7 +253,7 @@ Expected> NetworkGroupMetadata::get_input_layer_infos(con return res; } -Expected> NetworkGroupMetadata::get_output_layer_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_output_layer_infos(const std::string &network_name) const { std::vector res; // Edge layers exists only in the dynamic context. @@ -229,22 +268,22 @@ Expected> NetworkGroupMetadata::get_output_layer_infos(co return res; } -const PreliminaryContextMetadata &NetworkGroupMetadata::preliminary_context() const +const ContextMetadata &CoreOpMetadata::preliminary_context() const { return m_preliminary_context; } -const std::vector &NetworkGroupMetadata::dynamic_contexts() const +const std::vector &CoreOpMetadata::dynamic_contexts() const { return m_dynamic_contexts; } -const std::vector &NetworkGroupMetadata::config_channels_info() const +const std::vector &CoreOpMetadata::config_channels_info() const { return m_config_channels_info; } -Expected> NetworkGroupMetadata::get_all_layer_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_all_layer_infos(const std::string &network_name) const { auto input_layer_infos = get_input_layer_infos(network_name); CHECK_EXPECTED(input_layer_infos); @@ -260,7 +299,7 @@ Expected> NetworkGroupMetadata::get_all_layer_infos(const return res; } -Expected> NetworkGroupMetadata::get_input_stream_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_input_stream_infos(const std::string &network_name) const { auto input_layer_infos = get_input_layer_infos(network_name); CHECK_EXPECTED(input_layer_infos); @@ -268,7 +307,7 @@ Expected> NetworkGroupMetadata::get_input_strea return convert_layer_infos_to_stream_infos(input_layer_infos.value()); } -Expected> NetworkGroupMetadata::get_output_stream_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_output_stream_infos(const std::string &network_name) const { auto output_layer_infos = get_output_layer_infos(network_name); CHECK_EXPECTED(output_layer_infos); @@ -276,7 +315,7 @@ Expected> NetworkGroupMetadata::get_output_stre return convert_layer_infos_to_stream_infos(output_layer_infos.value()); } -Expected> NetworkGroupMetadata::get_all_stream_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_all_stream_infos(const std::string &network_name) const { auto input_stream_infos = get_input_stream_infos(network_name); CHECK_EXPECTED(input_stream_infos); @@ -292,7 +331,7 @@ Expected> NetworkGroupMetadata::get_all_stream_ return res; } -Expected> NetworkGroupMetadata::get_input_vstream_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_input_vstream_infos(const std::string &network_name) const { auto input_layer_infos = get_input_layer_infos(network_name); CHECK_EXPECTED(input_layer_infos); @@ -300,7 +339,7 @@ Expected> NetworkGroupMetadata::get_input_vstr return convert_layer_infos_to_vstream_infos(input_layer_infos.value()); } -Expected> NetworkGroupMetadata::get_output_vstream_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_output_vstream_infos(const std::string &network_name) const { std::vector res; if (m_supported_features.hailo_net_flow) { @@ -339,7 +378,7 @@ Expected> NetworkGroupMetadata::get_output_vst return res; } -Expected> NetworkGroupMetadata::get_all_vstream_infos(const std::string &network_name) const +Expected> CoreOpMetadata::get_all_vstream_infos(const std::string &network_name) const { auto input_vstream_infos = get_input_vstream_infos(network_name); CHECK_EXPECTED(input_vstream_infos); @@ -355,7 +394,7 @@ Expected> NetworkGroupMetadata::get_all_vstrea return res; } -Expected> NetworkGroupMetadata::get_vstream_names_from_stream_name(const std::string &stream_name) const +Expected> CoreOpMetadata::get_vstream_names_from_stream_name(const std::string &stream_name) const { std::vector results; for (auto &layer_info : get_all_layer_infos()) { @@ -372,7 +411,7 @@ Expected> NetworkGroupMetadata::get_vstream_names_from_ return make_unexpected(HAILO_NOT_FOUND); } -Expected> NetworkGroupMetadata::get_stream_names_from_vstream_name(const std::string &vstream_name) const +Expected> CoreOpMetadata::get_stream_names_from_vstream_name(const std::string &vstream_name) const { std::vector results; for (auto &layer_info : get_all_layer_infos()) { @@ -397,7 +436,7 @@ Expected> NetworkGroupMetadata::get_stream_names_from_v return results; } -std::vector NetworkGroupMetadata::convert_layer_infos_to_stream_infos(const std::vector &layer_infos) const +std::vector CoreOpMetadata::convert_layer_infos_to_stream_infos(const std::vector &layer_infos) const { std::vector res; for (auto &layer_info : layer_infos) { @@ -406,7 +445,7 @@ std::vector NetworkGroupMetadata::convert_layer_infos_to_st return res; } -std::vector NetworkGroupMetadata::convert_layer_infos_to_vstream_infos(const std::vector &layer_infos) const +std::vector CoreOpMetadata::convert_layer_infos_to_vstream_infos(const std::vector &layer_infos) const { std::vector res; for (auto &layer_info : layer_infos) { @@ -421,10 +460,9 @@ std::vector NetworkGroupMetadata::convert_layer_infos_to_v return res; } -Expected> NetworkGroupMetadata::get_network_infos() const +Expected> CoreOpMetadata::get_network_infos() const { std::vector network_infos; - auto net_group_name = network_group_name(); network_infos.reserve(m_sorted_network_names.size()); for (auto const &network_name : m_sorted_network_names) { hailo_network_info_t network_info = {}; @@ -438,8 +476,23 @@ Expected> NetworkGroupMetadata::get_network_in return network_infos; } +size_t CoreOpMetadata::get_contexts_count() +{ + return (m_dynamic_contexts.size() + CONTROL_PROTOCOL__CONTEXT_SWITCH_NUMBER_OF_NON_DYNAMIC_CONTEXTS); +} + +Expected CoreOpMetadata::get_total_transfer_size() +{ + size_t total_transfer_size = 0; + for (const auto &dynamic_context : m_dynamic_contexts) { + auto context_size = dynamic_context.get_context_transfer_size(); + CHECK_EXPECTED(context_size); + total_transfer_size += context_size.release(); + } + return total_transfer_size; +} -Expected NetworkGroupMetadataPerArch::get_metadata(uint32_t partial_clusters_layout_bitmap) +Expected CoreOpMetadataPerArch::get_metadata(uint32_t partial_clusters_layout_bitmap) { if (PARTIAL_CLUSTERS_LAYOUT_IGNORE == partial_clusters_layout_bitmap) { // Passing PARTIAL_CLUSTERS_LAYOUT_IGNORE is magic for getting one of the metadata @@ -451,11 +504,11 @@ Expected NetworkGroupMetadataPerArch::get_metadata(uint32_ auto result = m_metadata_per_arch[partial_clusters_layout_bitmap]; return result; } - LOGGER__ERROR("NetworkGroupMetadataPerArch does not contain metadata for partial_clusters_layout_bitmap {}", partial_clusters_layout_bitmap); + LOGGER__ERROR("CoreOpPerArch does not contain metadata for partial_clusters_layout_bitmap {}", partial_clusters_layout_bitmap); return make_unexpected(HAILO_INTERNAL_FAILURE); } -void NetworkGroupMetadataPerArch::add_metadata(const NetworkGroupMetadata &metadata, uint32_t partial_clusters_layout_bitmap) +void CoreOpMetadataPerArch::add_metadata(const CoreOpMetadata &metadata, uint32_t partial_clusters_layout_bitmap) { m_metadata_per_arch[partial_clusters_layout_bitmap] = metadata; } diff --git a/hailort/libhailort/src/network_group_metadata.hpp b/hailort/libhailort/src/hef/core_op_metadata.hpp similarity index 72% rename from hailort/libhailort/src/network_group_metadata.hpp rename to hailort/libhailort/src/hef/core_op_metadata.hpp index 813dbd7..d725524 100644 --- a/hailort/libhailort/src/network_group_metadata.hpp +++ b/hailort/libhailort/src/hef/core_op_metadata.hpp @@ -3,15 +3,16 @@ * Distributed under the MIT license (https://opensource.org/licenses/MIT) **/ /** - * @file network_group_metadata.hpp - * @brief Contains all relevant information about a network group from the hef. + * @file core_op_metadata.hpp + * @brief Contains all relevant information about a core-op from the hef. **/ -#ifndef _HAILO_NETWORK_GROUP_METADATA_HPP_ -#define _HAILO_NETWORK_GROUP_METADATA_HPP_ +#ifndef _HAILO_CORE_OP_METADATA_HPP_ +#define _HAILO_CORE_OP_METADATA_HPP_ + +#include "hef/layer_info.hpp" +#include "hef/context_switch_actions.hpp" -#include "layer_info.hpp" -#include "context_switch/context_switch_actions.hpp" namespace hailort { @@ -29,25 +30,17 @@ struct SupportedFeatures { // For each config_stream_index we store vector of all ccw write length. The vector is used to build the config buffer.g using ConfigBufferInfoMap = std::unordered_map>; -class PreliminaryContextMetadata final { -public: - PreliminaryContextMetadata() = default; // TODO HRT-8478: remove - PreliminaryContextMetadata(std::vector &&operations, - ConfigBufferInfoMap&& config_buffers_info); - const std::vector &get_operations() const; - const ConfigBufferInfoMap &config_buffers_info() const; - -private: - std::vector m_operations; - ConfigBufferInfoMap m_config_buffers_info; -}; class ContextMetadata final { public: - explicit ContextMetadata(std::vector &&operations, + ContextMetadata() = default; // TODO HRT-8478: remove + ContextMetadata(std::vector &&actions, ConfigBufferInfoMap&& config_buffers_info); - const std::vector &get_operations() const; + const std::vector &get_actions() const; + std::vector get_actions_of_type( + const std::set &action_types) const; + const ConfigBufferInfoMap &config_buffers_info() const; void add_boundary_layer(const LayerInfo &layer_info); @@ -61,8 +54,10 @@ public: const std::vector &get_ddr_input_layers() const; const std::vector &get_ddr_output_layers() const; + Expected get_layers_transfer_size(const std::vector &layer_infos) const; + Expected get_context_transfer_size() const; private: - std::vector m_operations; + std::vector m_actions; ConfigBufferInfoMap m_config_buffers_info; std::vector m_boundary_input_layers; @@ -77,11 +72,11 @@ struct ConfigChannelInfo { uint8_t engine_index; }; -class NetworkGroupMetadata final { +class CoreOpMetadata final { public: - NetworkGroupMetadata() = default; // TODO HRT-8478: remove - NetworkGroupMetadata(const std::string &network_group_name, - PreliminaryContextMetadata &&preliminary_context, + CoreOpMetadata() = default; // TODO HRT-8478: remove + CoreOpMetadata(const std::string &core_op_name, + ContextMetadata &&preliminary_context, std::vector &&dynamic_contexts, std::vector &&config_channels_info, std::vector &&sorted_output_names, @@ -97,7 +92,7 @@ public: Expected> get_all_layer_infos(const std::string &network_name) const; Expected get_layer_info_by_stream_name(const std::string &stream_name) const; - const PreliminaryContextMetadata &preliminary_context() const; + const ContextMetadata &preliminary_context() const; const std::vector &dynamic_contexts() const; const std::vector &config_channels_info() const; @@ -106,23 +101,27 @@ public: Expected> get_output_stream_infos(const std::string &network_name = "") const; Expected> get_all_stream_infos(const std::string &network_name = "") const; + // TODO: HRT-9546 - Remove, should only be in CNG Expected> get_input_vstream_infos(const std::string &network_name = "") const; Expected> get_output_vstream_infos(const std::string &network_name = "") const; Expected> get_all_vstream_infos(const std::string &network_name = "") const; + // TODO: HRT-9546 - Remove, should only be in CNG - need to decide if relevant only for one CoreOp case. Expected> get_vstream_names_from_stream_name(const std::string &stream_name) const; Expected> get_stream_names_from_vstream_name(const std::string &vstream_name) const; Expected> get_network_infos() const; - const std::string &network_group_name() const + size_t get_contexts_count(); + + const std::string &core_op_name() const { - return m_network_group_name; + return m_core_op_name; } const std::string default_network_name() const { - return HailoRTDefaults::get_network_name(m_network_group_name); + return HailoRTDefaults::get_network_name(m_core_op_name); } const std::vector get_sorted_output_names() const @@ -130,6 +129,7 @@ public: return m_sorted_output_names; } + // duplicated for each CoreOp const SupportedFeatures &supported_features() const { return m_supported_features; @@ -140,39 +140,43 @@ public: return m_sorted_network_names; } + // TODO: HRT-9546 - Move to CNG void add_output_vstream_info(const hailo_vstream_info_t &output_vstream_info) { m_output_vstreams_infos.push_back(output_vstream_info); } + Expected get_total_transfer_size(); + private: std::vector convert_layer_infos_to_stream_infos(const std::vector &layer_infos) const; std::vector convert_layer_infos_to_vstream_infos(const std::vector &layer_infos) const; - PreliminaryContextMetadata m_preliminary_context; + ContextMetadata m_preliminary_context; std::vector m_dynamic_contexts; std::vector m_config_channels_info; - std::string m_network_group_name; + std::string m_core_op_name; std::vector m_sorted_output_names; SupportedFeatures m_supported_features; std::vector m_sorted_network_names; - // TODO: remove this from here! NetworkGroupMetadata should be CoreOpMetadata and contain no net_flow information! (HRT-8639) + + // TODO: remove this from here! NetworkGroupMetadata should be CoreOpMetadata and contain no net_flow information! (HRT-9546) // To add insult to injury, this is being constructed lazyly by add_output_layer_info std::vector m_output_vstreams_infos; // Valid only in case of post process }; -class NetworkGroupMetadataPerArch final +class CoreOpMetadataPerArch final { public: - NetworkGroupMetadataPerArch() = default; + CoreOpMetadataPerArch() = default; - Expected get_metadata(uint32_t partial_clusters_layout_bitmap); - void add_metadata(const NetworkGroupMetadata &metadata, uint32_t partial_clusters_layout_bitmap); + Expected get_metadata(uint32_t partial_clusters_layout_bitmap); + void add_metadata(const CoreOpMetadata &metadata, uint32_t partial_clusters_layout_bitmap); private: - std::map m_metadata_per_arch; + std::map m_metadata_per_arch; }; } /* namespace hailort */ -#endif /* _HAILO_NETWORK_GROUP_METADATA_HPP_ */ +#endif /* _HAILO_CORE_OP_METADATA_HPP_ */ diff --git a/hailort/libhailort/src/hef.cpp b/hailort/libhailort/src/hef/hef.cpp similarity index 78% rename from hailort/libhailort/src/hef.cpp rename to hailort/libhailort/src/hef/hef.cpp index aa0ee84..cd543a0 100644 --- a/hailort/libhailort/src/hef.cpp +++ b/hailort/libhailort/src/hef/hef.cpp @@ -11,22 +11,27 @@ #include "hailo/hailort.h" #include "hailo/hef.hpp" -#include "hef_internal.hpp" #include "hailo/stream.hpp" #include "hailo/device.hpp" -#include "common/utils.hpp" #include "hailo/hailort_common.hpp" -#include "hailort_defaults.hpp" -#include "common/string_utils.hpp" +#include "hailo/hailort_defaults.hpp" -#include "pcie_device.hpp" -#include "context_switch/multi_context/vdma_config_manager.hpp" -#include "context_switch/single_context/hcp_config_network_group.hpp" -#include "byte_order.h" +#include "common/string_utils.hpp" +#include "common/utils.hpp" #include "common/logger_macros.hpp" #include "common/file_utils.hpp" -#include "layer_info.hpp" -#include "control.hpp" + +#include "net_flow/ops/nms_post_process.hpp" +#include "net_flow/ops/yolo_post_process.hpp" +#include "net_flow/ops/ssd_post_process.hpp" +#include "hef/hef_internal.hpp" +#include "vdma/pcie/pcie_device.hpp" +#include "vdma/vdma_config_manager.hpp" +#include "eth/hcp_config_core_op.hpp" +#include "hef/layer_info.hpp" +#include "device_common/control.hpp" + +#include "byte_order.h" #include "context_switch_defs.h" #include @@ -39,14 +44,70 @@ #include #include + namespace hailort { #define HEF__MD5_BUFFER_SIZE (1024) #define DEFAULT_BATCH_SIZE (1) +#define SKIP_SPACE_COMMA_CHARACTERS (2) static const uint8_t ENABLE_LCU_CONTROL_WORD[4] = {1, 0, 0, 0}; +#define TAB (" ") + +static std::string add_tabs(uint8_t count) +{ + // Each TAB counts as 4 spaces + std::string res = ""; + for (uint8_t i = 0; i < count; i++) { + res = res + TAB; + } + return res; +} + +static std::string get_shape_str(const hailo_stream_info_t &stream_info) +{ + switch (stream_info.format.order) + { + case HAILO_FORMAT_ORDER_HAILO_NMS: + return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) + + "(number of classes: " + std::to_string(stream_info.nms_info.number_of_classes) + + ", max_bboxes_per_class: "+ std::to_string(stream_info.nms_info.max_bboxes_per_class) + ")"; + case HAILO_FORMAT_ORDER_NC: + return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) + + "(" + std::to_string(stream_info.hw_shape.features) + ")"; + case HAILO_FORMAT_ORDER_NHW: + return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) + + "(" + std::to_string(stream_info.hw_shape.height) + "x" + std::to_string(stream_info.hw_shape.width) + ")"; + default: + return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) + + "(" + std::to_string(stream_info.hw_shape.height) + "x" + std::to_string(stream_info.hw_shape.width) + + "x" + std::to_string(stream_info.hw_shape.features) + ")"; + } +} + +static std::string get_shape_str(const hailo_vstream_info_t &vstream_info) +{ + switch (vstream_info.format.order) + { + case HAILO_FORMAT_ORDER_HAILO_NMS: + return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) + + "(number of classes: " + std::to_string(vstream_info.nms_shape.number_of_classes) + + ", max_bboxes_per_class: " + std::to_string(vstream_info.nms_shape.max_bboxes_per_class) + ")"; + case HAILO_FORMAT_ORDER_NC: + return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) + + "(" + std::to_string(vstream_info.shape.features) + ")"; + case HAILO_FORMAT_ORDER_NHW: + return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) + + "(" +std::to_string(vstream_info.shape.height) + "x" + std::to_string(vstream_info.shape.width) + ")"; + default: + return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) + + "(" + std::to_string(vstream_info.shape.height) + "x" + std::to_string(vstream_info.shape.width) + "x" + + std::to_string(vstream_info.shape.features) + ")"; + } +} + #pragma pack(push, 1) typedef struct { uint32_t words_count; @@ -205,6 +266,17 @@ Expected Hef::get_bottleneck_fps(const std::string &net_group_name) return pimpl->get_bottleneck_fps(net_group_name); } + +Expected Hef::get_hef_device_arch() +{ + return DeviceBase::hef_arch_to_device_arch(pimpl->get_device_arch()); +} + +Expected Hef::device_arch_to_string(const hailo_device_architecture_t arch) +{ + return HailoRTCommon::get_device_arch_str(arch); +} + Expected Hef::get_vstream_name_from_original_name(const std::string &original_name, const std::string &net_group_name) { @@ -415,12 +487,12 @@ hailo_status Hef::Impl::fill_networks_metadata() { fill_extensions_bitset(); - NetworkGroupMetadataPerArch metadata; + CoreOpMetadataPerArch metadata; uint32_t partial_clusters_layout_bitmap = 0; for (auto &network_group : m_groups) { auto network_group_name = HefUtils::get_network_group_name(*network_group, m_supported_features); - // TODO: keep metadata per core_op (HRT-8639) + // TODO: keep metadata per core_op (HRT-9551) const auto &core_ops = m_core_ops_per_group[network_group_name]; assert(core_ops.size() == 1); const auto &core_op = core_ops[0]; @@ -431,9 +503,9 @@ hailo_status Hef::Impl::fill_networks_metadata() auto metadata_per_arch = create_metadata_per_arch(*(partial_core_op->core_op)); CHECK_EXPECTED_AS_STATUS(metadata_per_arch); auto &&arch_metadata = metadata_per_arch.release(); - auto expected_net_flow_ops = create_network_group_ops(*network_group, arch_metadata); + auto expected_net_flow_ops = create_net_flow_ops(*network_group, arch_metadata); CHECK_EXPECTED_AS_STATUS(expected_net_flow_ops); - m_post_process_ops_per_group.insert({arch_metadata.network_group_name(), expected_net_flow_ops.value()}); + m_post_process_ops_per_group.insert({arch_metadata.core_op_name(), expected_net_flow_ops.value()}); metadata.add_metadata(arch_metadata, partial_clusters_layout_bitmap); } } else { @@ -451,8 +523,8 @@ hailo_status Hef::Impl::fill_networks_metadata() auto metadata_per_arch = create_metadata_per_arch(partial_core_op); CHECK_EXPECTED_AS_STATUS(metadata_per_arch); auto &&arch_metadata = metadata_per_arch.release(); - std::vector> empty_ops; - m_post_process_ops_per_group.insert({arch_metadata.network_group_name(), empty_ops}); + std::vector> empty_ops; + m_post_process_ops_per_group.insert({arch_metadata.core_op_name(), empty_ops}); metadata.add_metadata(arch_metadata, partial_clusters_layout_bitmap); } } @@ -461,14 +533,14 @@ hailo_status Hef::Impl::fill_networks_metadata() auto metadata_per_arch = create_metadata_per_arch(core_op); CHECK_EXPECTED_AS_STATUS(metadata_per_arch); auto &&arch_metadata = metadata_per_arch.release(); - auto expected_net_flow_ops = create_network_group_ops(*network_group, arch_metadata); + auto expected_net_flow_ops = create_net_flow_ops(*network_group, arch_metadata); CHECK_EXPECTED_AS_STATUS(expected_net_flow_ops); - m_post_process_ops_per_group.insert({arch_metadata.network_group_name(), expected_net_flow_ops.value()}); + m_post_process_ops_per_group.insert({arch_metadata.core_op_name(), expected_net_flow_ops.value()}); metadata.add_metadata(arch_metadata, partial_clusters_layout_bitmap); } - CHECK(!contains(m_network_group_metadata_per_arch, network_group_name), + CHECK(!contains(m_core_op_per_arch, network_group_name), HAILO_INVALID_OPERATION, "Network group with the name {} is already configured on the device", network_group_name); - m_network_group_metadata_per_arch.emplace(network_group_name, metadata); + m_core_op_per_arch.emplace(network_group_name, metadata); } return HAILO_SUCCESS; } @@ -506,7 +578,7 @@ static Expected> parse_config_channels_info(const return config_channels_info; } -Expected Hef::Impl::create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op) +Expected Hef::Impl::create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op) { auto preliminary_context = HefUtils::parse_preliminary_context(core_op.preliminary_config, m_supported_features); CHECK_EXPECTED(preliminary_context); @@ -531,7 +603,9 @@ Expected Hef::Impl::create_metadata_per_arch(const ProtoHE sorted_network_names.push_back(HailoRTDefaults::get_network_name(core_op.network_group_metadata.network_group_name())); } - NetworkGroupMetadata metadata_per_arch(core_op.network_group_metadata.network_group_name(), + // Currently, CoreOp name is the same as network_group_name, thats why we init it with it. + // TODO: HRT-9551 - Change it when supporting multi core ops. + CoreOpMetadata metadata_per_arch(core_op.network_group_metadata.network_group_name(), preliminary_context.release(), dynamic_contexts.release(), config_channels_info.release(), sorted_output_names.release(), m_supported_features, sorted_network_names); return metadata_per_arch; @@ -702,14 +776,163 @@ SupportedFeatures Hef::Impl::get_supported_features(const ProtoHEFHeader &header return supported_features; } -Expected>> Hef::Impl::create_network_group_ops(const ProtoHEFNetworkGroup &network_group_proto, - NetworkGroupMetadata &network_group_meta_data) const +net_flow::NmsPostProcessConfig create_nms_config(const ProtoHEFOp &op_proto) +{ + net_flow::NmsPostProcessConfig nms_config{}; + nms_config.nms_score_th = (float32_t)op_proto.nms_op().nms_score_th(); + nms_config.nms_iou_th = (float32_t)op_proto.nms_op().nms_iou_th(); + nms_config.max_proposals_per_class = op_proto.nms_op().max_proposals_per_class(); + nms_config.classes = op_proto.nms_op().classes(); + nms_config.background_removal = op_proto.nms_op().background_removal(); + nms_config.background_removal_index = op_proto.nms_op().background_removal_index(); + + return nms_config; +} + +Expected> create_yolov5_op(const ProtoHEFOp &op_proto, hailo_format_t output_format, + const std::map &pad_index_to_streams_info, const std::map &input_to_output_pads) +{ + auto nms_config = create_nms_config(op_proto); + net_flow::YoloPostProcessConfig yolo_config{}; + yolo_config.image_height = (float32_t)op_proto.nms_op().yolo_nms_op().image_height(); + yolo_config.image_width = (float32_t)op_proto.nms_op().yolo_nms_op().image_width(); + for (auto &bbox_proto : op_proto.nms_op().yolo_nms_op().bbox_decoders()) { + std::vector bbox_anchors; + CHECK_AS_EXPECTED((bbox_proto.h().size() == bbox_proto.w().size()), HAILO_INVALID_HEF, + "YOLOv5 height anchors count {} doesn't mach the width anchors count {}", bbox_proto.h().size(), bbox_proto.w().size()); + for (int i = 0; i < bbox_proto.h().size(); ++i) { + bbox_anchors.push_back(bbox_proto.w()[i]); + bbox_anchors.push_back(bbox_proto.h()[i]); + } + assert(contains(pad_index_to_streams_info, static_cast(bbox_proto.pad_index()))); + yolo_config.anchors.insert({pad_index_to_streams_info.at(bbox_proto.pad_index()).name, bbox_anchors}); + } + + std::map inputs_metadata; + std::map outputs_metadata; + net_flow::BufferMetaData output_metadata{}; + output_metadata.format = output_format; + outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); + + for (auto &input_pad : op_proto.input_pads()) { + CHECK_AS_EXPECTED(contains(input_to_output_pads, static_cast(input_pad.index())), HAILO_INVALID_HEF, + "NMS op is not connected to core op"); + auto output_pad_index = input_to_output_pads.at(input_pad.index()); + CHECK_AS_EXPECTED(contains(pad_index_to_streams_info, output_pad_index), HAILO_INVALID_HEF, + "Pad {} of post-process {} is not connected to any core output stream", + input_pad.index(), op_proto.name()); + const auto &op_input_stream = pad_index_to_streams_info.at(output_pad_index); + net_flow::BufferMetaData input_metadata{}; + input_metadata.format = op_input_stream.format; + input_metadata.quant_info = op_input_stream.quant_info; + input_metadata.shape = op_input_stream.shape; + input_metadata.padded_shape = op_input_stream.hw_shape; + inputs_metadata.insert({op_input_stream.name, input_metadata}); + } + return net_flow::YOLOv5PostProcessOp::create(inputs_metadata, outputs_metadata, nms_config, yolo_config); +} + +Expected> create_yolox_op(const ProtoHEFOp &op_proto, hailo_format_t output_format, + const std::map &pad_index_to_streams_info, const std::map &input_to_output_pads) +{ + auto nms_config = create_nms_config(op_proto); + net_flow::YoloPostProcessConfig yolo_config{}; + yolo_config.image_height = (float32_t)op_proto.nms_op().yolo_nms_op().image_height(); + yolo_config.image_width = (float32_t)op_proto.nms_op().yolo_nms_op().image_width(); + + std::map inputs_metadata; + std::map outputs_metadata; + net_flow::BufferMetaData output_metadata{}; + output_metadata.format = output_format; + outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); + + for (auto &input_pad : op_proto.input_pads()) { + CHECK_AS_EXPECTED(contains(input_to_output_pads, static_cast(input_pad.index())), HAILO_INVALID_HEF, + "NMS op is not connected to core op"); + auto output_pad_index = input_to_output_pads.at(input_pad.index()); + CHECK_AS_EXPECTED(contains(pad_index_to_streams_info, output_pad_index), HAILO_INVALID_HEF, + "Pad {} of post-process {} is not connected to any core output stream", + input_pad.index(), op_proto.name()); + const auto &op_input_stream = pad_index_to_streams_info.at(output_pad_index); + net_flow::BufferMetaData input_metadata{}; + input_metadata.format = op_input_stream.format; + input_metadata.quant_info = op_input_stream.quant_info; + input_metadata.shape = op_input_stream.shape; + input_metadata.padded_shape = op_input_stream.hw_shape; + inputs_metadata.insert({op_input_stream.name, input_metadata}); + } + return net_flow::YOLOXPostProcessOp::create(inputs_metadata, outputs_metadata, nms_config, yolo_config); +} + +Expected> create_ssd_op(const ProtoHEFOp &op_proto, hailo_format_t output_format, + const std::map &pad_index_to_streams_info, const std::map &input_to_output_pads) +{ + auto nms_config = create_nms_config(op_proto); + net_flow::SSDPostProcessConfig ssd_config{}; + ssd_config.image_height = (float32_t)op_proto.nms_op().ssd_nms_op().image_height(); + ssd_config.image_width = (float32_t)op_proto.nms_op().ssd_nms_op().image_width(); + ssd_config.centers_scale_factor = op_proto.nms_op().ssd_nms_op().centers_scale_factor(); + ssd_config.bbox_dimensions_scale_factor = op_proto.nms_op().ssd_nms_op().bbox_dimensions_scale_factor(); + ssd_config.ty_index = op_proto.nms_op().ssd_nms_op().ty(); + ssd_config.tx_index = op_proto.nms_op().ssd_nms_op().tx(); + ssd_config.th_index = op_proto.nms_op().ssd_nms_op().th(); + ssd_config.tw_index = op_proto.nms_op().ssd_nms_op().tw(); + + if ((ssd_config.ty_index == 0) && (ssd_config.tx_index == 0) && (ssd_config.th_index == 0) && (ssd_config.tw_index == 0)) { + ssd_config.ty_index = net_flow::SSDPostProcessOp::DEFAULT_Y_OFFSET_IDX; + ssd_config.tx_index = net_flow::SSDPostProcessOp::DEFAULT_X_OFFSET_IDX; + ssd_config.th_index = net_flow::SSDPostProcessOp::DEFAULT_H_OFFSET_IDX; + ssd_config.tw_index = net_flow::SSDPostProcessOp::DEFAULT_W_OFFSET_IDX; + } + + for (auto &bbox_proto : op_proto.nms_op().ssd_nms_op().bbox_decoders()) { + std::vector bbox_anchors; + assert(bbox_proto.h().size() == bbox_proto.w().size()); + for (int i = 0; i < bbox_proto.h().size(); ++i) { + bbox_anchors.push_back(bbox_proto.w()[i]); + bbox_anchors.push_back(bbox_proto.h()[i]); + } + assert(contains(pad_index_to_streams_info, static_cast(bbox_proto.reg_pad_index()))); + auto reg_name = pad_index_to_streams_info.at(bbox_proto.reg_pad_index()).name; + ssd_config.anchors.insert({reg_name, bbox_anchors}); + assert(contains(pad_index_to_streams_info, static_cast(bbox_proto.cls_pad_index()))); + auto cls_name = pad_index_to_streams_info.at(bbox_proto.cls_pad_index()).name; + ssd_config.anchors.insert({pad_index_to_streams_info.at(bbox_proto.cls_pad_index()).name, bbox_anchors}); + ssd_config.reg_to_cls_inputs.insert({reg_name, cls_name}); + } + + std::map inputs_metadata; + std::map outputs_metadata; + net_flow::BufferMetaData output_metadata{}; + output_metadata.format = output_format; + outputs_metadata.insert({op_proto.output_pads()[0].name(), output_metadata}); + + for (auto &input_pad : op_proto.input_pads()) { + CHECK_AS_EXPECTED(contains(input_to_output_pads, static_cast(input_pad.index())), HAILO_INVALID_HEF, + "NMS op is not connected to core op"); + auto output_pad_index = input_to_output_pads.at(input_pad.index()); + CHECK_AS_EXPECTED(contains(pad_index_to_streams_info, output_pad_index), HAILO_INVALID_HEF, + "Pad {} of post-process {} is not connected to any core output stream", + input_pad.index(), op_proto.name()); + const auto &op_input_stream = pad_index_to_streams_info.at(output_pad_index); + net_flow::BufferMetaData input_metadata{}; + input_metadata.format = op_input_stream.format; + input_metadata.quant_info = op_input_stream.quant_info; + input_metadata.shape = op_input_stream.shape; + input_metadata.padded_shape = op_input_stream.hw_shape; + inputs_metadata.insert({op_input_stream.name, input_metadata}); + } + return net_flow::SSDPostProcessOp::create(inputs_metadata, outputs_metadata, nms_config, ssd_config); +} + +Expected>> Hef::Impl::create_net_flow_ops(const ProtoHEFNetworkGroup &network_group_proto, + CoreOpMetadata &core_op_metadata) const { std::vector> result; if (!m_supported_features.hailo_net_flow) { return result; } - auto output_layer_infos = network_group_meta_data.get_output_layer_infos(); + auto output_layer_infos = core_op_metadata.get_output_layer_infos(); std::map pad_index_to_streams_info; for (auto &output_layer_info : output_layer_infos) { if (output_layer_info.pad_index != INVALID_PAD_INDEX) { @@ -726,70 +949,86 @@ Expected>> Hef::Impl::create_network break; } case ProtoHEFOp::kNmsOp: { - NetFlowYoloNmsElement nms_op{}; - nms_op.type = NetFlowElement::Type::YoloNmsOp; - nms_op.name = "YOLO_NMS"; - nms_op.nms_score_th = (float32_t)op_proto.nms_op().nms_score_th(); - nms_op.nms_iou_th = (float32_t)op_proto.nms_op().nms_iou_th(); - nms_op.max_proposals_per_class = op_proto.nms_op().max_proposals_per_class(); - nms_op.classes = op_proto.nms_op().classes(); - nms_op.background_removal = op_proto.nms_op().background_removal(); - nms_op.background_removal_index = op_proto.nms_op().background_removal_index(); - nms_op.image_height = (float32_t)op_proto.nms_op().yolo_nms_op().image_height(); - nms_op.image_width = (float32_t)op_proto.nms_op().yolo_nms_op().image_width(); - nms_op.input_division_factor = op_proto.nms_op().yolo_nms_op().input_division_factor(); - if (!nms_op.input_division_factor) { - nms_op.input_division_factor = 1; - } - nms_op.bbox_decoders.reserve(op_proto.nms_op().yolo_nms_op().bbox_decoders().size()); - for (auto &bbox_proto : op_proto.nms_op().yolo_nms_op().bbox_decoders()) { - YoloBboxDecoder yolo_bbox_decoder; - for (auto h : bbox_proto.h()) { - yolo_bbox_decoder.h.push_back(h); - } - for (auto w : bbox_proto.w()) { - yolo_bbox_decoder.w.push_back(w); - } - yolo_bbox_decoder.stride = bbox_proto.stride(); - yolo_bbox_decoder.stream_name = pad_index_to_streams_info[input_to_output_pads[bbox_proto.pad_index()]].name; - nms_op.bbox_decoders.push_back(yolo_bbox_decoder); - } - std::set input_pads; - std::transform(op_proto.input_pads().begin(), op_proto.input_pads().end(), std::inserter(input_pads, input_pads.begin()), - [](auto &pad) { - return pad.index(); - }); + hailo_format_t output_format{}; + output_format.type = HAILO_FORMAT_TYPE_FLOAT32; + output_format.order = HAILO_FORMAT_ORDER_HAILO_NMS; + output_format.flags = HAILO_FORMAT_FLAGS_QUANTIZED; + NetFlowElement net_flow_element{}; + + // TODO: HRT-9902 - Move nms_info to be an op member instead of NetFlowElement + net_flow_element.nms_info = { + op_proto.nms_op().classes(), + op_proto.nms_op().max_proposals_per_class(), + sizeof(hailo_bbox_float32_t), + 1, // input_division_factor + false, + hailo_nms_defuse_info_t() + }; for (auto &input_pad : op_proto.input_pads()) { - CHECK_AS_EXPECTED(input_to_output_pads.count(input_pad.index()), HAILO_INVALID_HEF, - "NMS op is not connected to core op"); - auto output_pad_index = input_to_output_pads[input_pad.index()]; - CHECK_AS_EXPECTED(pad_index_to_streams_info.count(output_pad_index), HAILO_INVALID_HEF, + CHECK_AS_EXPECTED(contains(input_to_output_pads, static_cast(input_pad.index())), HAILO_INVALID_HEF, + "NMS op is not connected to core-op"); + auto output_pad_index = input_to_output_pads.at(input_pad.index()); + CHECK_AS_EXPECTED(contains(pad_index_to_streams_info, output_pad_index), HAILO_INVALID_HEF, "Pad {} of post-process {} is not connected to any core output stream", input_pad.index(), op_proto.name()); - const auto &op_input_stream = pad_index_to_streams_info[output_pad_index]; - nms_op.input_pads.push_back(NetFlowPad{input_pad.name(), op_input_stream.format, op_input_stream.quant_info, 0}); - nms_op.input_streams.insert(op_input_stream.name); + const auto &op_input_stream = pad_index_to_streams_info.at(output_pad_index); + net_flow_element.input_streams.insert(op_input_stream.name); } - hailo_format_t format; - format.type = HAILO_FORMAT_TYPE_FLOAT32; - format.order = HAILO_FORMAT_ORDER_HAILO_NMS; - format.flags = HAILO_FORMAT_FLAGS_QUANTIZED; - assert(op_proto.output_pads().size() == 1); - auto proto_output_pad = op_proto.output_pads()[0]; - nms_op.output_pads.push_back(NetFlowPad{proto_output_pad.name(), format, hailo_quant_info_t(), nms_op.classes}); - result.push_back(std::shared_ptr(std::make_shared(nms_op))); + std::shared_ptr post_process_op; + switch (op_proto.nms_op().nms_op_case()) { + case ProtoHEFNmsOp::kYoloNmsOp: { + net_flow_element.name = "YOLO-Post-Process"; + auto expected_post_process_op = create_yolov5_op(op_proto, output_format, pad_index_to_streams_info, input_to_output_pads); + CHECK_EXPECTED(expected_post_process_op); + post_process_op = expected_post_process_op.release(); + break; + } + case ProtoHEFNmsOp::kYoloxNmsOp: { + net_flow_element.name = "YOLOX-Post-Process"; + auto expected_post_process_op = create_yolox_op(op_proto, output_format, pad_index_to_streams_info, input_to_output_pads); + CHECK_EXPECTED(expected_post_process_op); + post_process_op = expected_post_process_op.release(); + break; + } + case ProtoHEFNmsOp::kSsdNmsOp: { + net_flow_element.name = "SSD-Post-Process"; + auto expected_post_process_op = create_ssd_op(op_proto, output_format, pad_index_to_streams_info, input_to_output_pads); + CHECK_EXPECTED(expected_post_process_op); + post_process_op = expected_post_process_op.release(); + break; + } + case ProtoHEFNmsOp::kIouOp: { + // TODO (HRT-8827) + break; + } + default: { + LOGGER__ERROR("Unsupported Net-Flow NMS-Op"); + return make_unexpected(HAILO_INTERNAL_FAILURE); + } + } + net_flow_element.op = post_process_op; // Fill meta-data output vstream info auto net_group_name = HefUtils::get_network_group_name(network_group_proto, m_supported_features); auto network_name = HailoRTDefaults::get_network_name(net_group_name); hailo_vstream_info_t net_flow_output_vstream_info{}; + assert(op_proto.output_pads().size() == 1); + auto proto_output_pad = op_proto.output_pads()[0]; strncpy(net_flow_output_vstream_info.name, proto_output_pad.name().c_str(), proto_output_pad.name().length() + 1); strncpy(net_flow_output_vstream_info.network_name, network_name.c_str(), network_name.length() + 1); net_flow_output_vstream_info.direction = HAILO_D2H_STREAM; - net_flow_output_vstream_info.format = format; - net_flow_output_vstream_info.nms_shape.number_of_classes = nms_op.classes; - net_flow_output_vstream_info.nms_shape.max_bboxes_per_class = nms_op.max_proposals_per_class; - network_group_meta_data.add_output_vstream_info(net_flow_output_vstream_info); + net_flow_output_vstream_info.format = output_format; + net_flow_output_vstream_info.nms_shape.max_bboxes_per_class = op_proto.nms_op().max_proposals_per_class(); + net_flow_output_vstream_info.nms_shape.number_of_classes = op_proto.nms_op().classes(); + if (op_proto.nms_op().background_removal()) { + net_flow_output_vstream_info.nms_shape.number_of_classes--; + net_flow_element.nms_info.number_of_classes--; + } + + result.push_back(std::make_shared(net_flow_element)); + + // TODO: HRT-9546 - Move vstreams out of core op + core_op_metadata.add_output_vstream_info(net_flow_output_vstream_info); break; } default: { @@ -801,6 +1040,32 @@ Expected>> Hef::Impl::create_network return result; } +Expected Hef::Impl::get_core_op_metadata(const std::string &network_group_name, uint32_t partial_clusters_layout_bitmap) +{ + CHECK_AS_EXPECTED(contains(m_core_op_per_arch, network_group_name), HAILO_NOT_FOUND, + "Network group with name {} wasn't found", network_group_name); + auto metadata_per_arch = m_core_op_per_arch.at(network_group_name); + auto metadata = metadata_per_arch.get_metadata(partial_clusters_layout_bitmap); + return metadata; +} + +hailo_status Hef::Impl::validate_boundary_streams_were_created(const std::string &network_group_name, std::shared_ptr core_op) +{ + auto number_of_inputs = get_number_of_input_streams(network_group_name); + CHECK_EXPECTED_AS_STATUS(number_of_inputs); + + auto size = core_op->get_input_streams().size(); + CHECK((number_of_inputs.value() == size), + HAILO_INVALID_ARGUMENT, "passed configure_params for network group {} did not contain all input streams", network_group_name); + + auto number_of_outputs = get_number_of_output_streams(network_group_name); + CHECK_EXPECTED_AS_STATUS(number_of_inputs); + CHECK((number_of_outputs.value() == core_op->get_output_streams().size()), + HAILO_INVALID_ARGUMENT, "passed configure_params for network group {} did not contain all output streams", network_group_name); + + return HAILO_SUCCESS; +} + hailo_status get_hw_padding_params(hailo_format_order_t format_order, uint32_t width, uint32_t features, uint32_t hw_data_bytes, uint16_t &feature_padding_payload, uint16_t &periph_bytes_per_buffer) { @@ -979,7 +1244,7 @@ bool HefConfigurator::is_hw_padding_supported(const ProtoHEFEdgeLayer &edge_laye Expected> Hef::Impl::get_input_stream_infos(const std::string &net_group_name, const std::string &network_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); + auto network_group_metadata = get_core_op_metadata(net_group_name); CHECK_EXPECTED(network_group_metadata); return network_group_metadata->get_input_stream_infos(network_name); } @@ -987,7 +1252,7 @@ Expected> Hef::Impl::get_input_stream_infos(con Expected> Hef::Impl::get_output_stream_infos(const std::string &net_group_name, const std::string &network_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); + auto network_group_metadata = get_core_op_metadata(net_group_name); CHECK_EXPECTED(network_group_metadata); return network_group_metadata->get_output_stream_infos(network_name); } @@ -995,14 +1260,14 @@ Expected> Hef::Impl::get_output_stream_infos(co Expected> Hef::Impl::get_all_stream_infos(const std::string &net_group_name, const std::string &network_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); + auto network_group_metadata = get_core_op_metadata(net_group_name); CHECK_EXPECTED(network_group_metadata); return network_group_metadata->get_all_stream_infos(network_name); } Expected> Hef::Impl::get_network_infos(const std::string &net_group_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); + auto network_group_metadata = get_core_op_metadata(net_group_name); CHECK_EXPECTED(network_group_metadata); return network_group_metadata->get_network_infos(); } @@ -1010,7 +1275,7 @@ Expected> Hef::Impl::get_network_infos(const s Expected Hef::Impl::get_stream_info_by_name(const std::string &stream_name, hailo_stream_direction_t stream_direction, const std::string &net_group_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); + auto network_group_metadata = get_core_op_metadata(net_group_name); CHECK_EXPECTED(network_group_metadata); if (HAILO_H2D_STREAM == stream_direction) { @@ -1037,7 +1302,7 @@ Expected Hef::Impl::get_stream_info_by_name(const std::stri Expected> Hef::Impl::get_input_vstream_infos(const std::string &net_group_name, const std::string &network_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); + auto network_group_metadata = get_core_op_metadata(net_group_name); CHECK_EXPECTED(network_group_metadata); return network_group_metadata->get_input_vstream_infos(network_name); } @@ -1045,7 +1310,7 @@ Expected> Hef::Impl::get_input_vstream_infos(c Expected> Hef::Impl::get_output_vstream_infos(const std::string &net_group_name, const std::string &network_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); + auto network_group_metadata = get_core_op_metadata(net_group_name); CHECK_EXPECTED(network_group_metadata); return network_group_metadata->get_output_vstream_infos(network_name); } @@ -1053,7 +1318,7 @@ Expected> Hef::Impl::get_output_vstream_infos( Expected> Hef::Impl::get_all_vstream_infos(const std::string &net_group_name, const std::string &network_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); + auto network_group_metadata = get_core_op_metadata(net_group_name); CHECK_EXPECTED(network_group_metadata); return network_group_metadata->get_all_vstream_infos(network_name); } @@ -1069,7 +1334,7 @@ const std::vector& Hef::Impl::core_ops(const std::string &ne return m_core_ops_per_group.at(net_group_name); }; -const std::vector> Hef::Impl::post_process_ops(const std::string &net_group_name) const +const std::vector> Hef::Impl::post_process_ops(const std::string &net_group_name) const { assert(contains(m_post_process_ops_per_group, net_group_name)); return m_post_process_ops_per_group.at(net_group_name); @@ -1188,19 +1453,19 @@ Expected> Hef::Impl::get_core_op_by_net_grou Expected Hef::Impl::get_number_of_input_streams(const std::string &net_group_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED(network_group_metadata); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED(core_op_metadata); - auto input_layer_infos = network_group_metadata->get_input_layer_infos(); + auto input_layer_infos = core_op_metadata->get_input_layer_infos(); return input_layer_infos.size(); } Expected Hef::Impl::get_number_of_output_streams(const std::string &net_group_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED(network_group_metadata); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED(core_op_metadata); - auto output_layer_infos = network_group_metadata->get_output_layer_infos(); + auto output_layer_infos = core_op_metadata->get_output_layer_infos(); return output_layer_infos.size(); } @@ -1220,7 +1485,7 @@ static Expected get_layer_type(const ProtoHEFEdgeConnectionType &edge } hailo_status HefUtils::fill_layer_info_with_base_info(const ProtoHEFEdgeLayerBase &base_info, - const ProtoHEFEdgeConnectionType &edge_connection_type, const ProtoHEFNetworkGroupMetadata &network_group_proto, + const ProtoHEFEdgeConnectionType &edge_connection_type, const ProtoHEFNetworkGroupMetadata &network_group_proto, bool hw_padding_supported, bool transposed, const uint8_t context_index, const uint8_t network_index, LayerInfo &layer_info) { @@ -1626,19 +1891,6 @@ static Expected parse_action(const ProtoHEFAction const SupportedFeatures &supported_features) { switch (proto_action.action_case()) { - case ProtoHEFAction::kWriteDataCcw:\ - { - CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(proto_action.write_data_ccw().cfg_channel_index()), HAILO_INVALID_HEF, - "Invalid cfg channel index"); - const auto config_stream_index = static_cast(proto_action.write_data_ccw().cfg_channel_index()); - - auto data = Buffer::create( - reinterpret_cast(proto_action.write_data_ccw().data().data()), - proto_action.write_data_ccw().data().length()); - CHECK_EXPECTED(data); - - return WriteDataCcwAction::create(data.release(), config_stream_index); - } case ProtoHEFAction::kDisableLcu: CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(proto_action.disable_lcu().cluster_index()), HAILO_INVALID_HEF, "Failed to parse HEF. Invalid cluster_index: {}.", proto_action.disable_lcu().cluster_index()); @@ -1740,7 +1992,7 @@ static Expected parse_action(const ProtoHEFAction const auto nms_unit_index = static_cast(proto_action.enable_nms().nms_unit_index()); - return EnableNmsAction::create(network_index, nms_unit_index); + return EnableNmsAction::create(nms_unit_index, network_index); } default: @@ -1752,100 +2004,128 @@ static Expected parse_action(const ProtoHEFAction return make_unexpected(HAILO_INTERNAL_FAILURE); } -static Expected parse_operation(const ProtoHEFOperation &operation_proto, - const SupportedFeatures &supported_features) +static Expected build_config_buffer(const std::vector &ccw_buffers) { - std::vector actions; - actions.reserve(operation_proto.actions_size() + 1); // +1 for the trigger action + size_t buffer_size = 0; + for (const auto &ccw_buffer : ccw_buffers) { + buffer_size += ccw_buffer.size(); + } - auto trigger_action = parse_trigger_action(operation_proto.trigger()); - CHECK_EXPECTED(trigger_action); - actions.emplace_back(trigger_action.release()); + auto config_buffer = Buffer::create(buffer_size); + CHECK_EXPECTED(config_buffer); - actions.reserve(operation_proto.actions_size()); - for (const auto &proto_action : operation_proto.actions()) { - auto action = parse_action(proto_action, supported_features); - CHECK_EXPECTED(action); - actions.emplace_back(action.release()); + size_t current_offset = 0; + for (const auto &ccw_buffer : ccw_buffers) { + assert(current_offset + ccw_buffer.size() <= config_buffer->size()); + memcpy(config_buffer->data() + current_offset, ccw_buffer.data(), ccw_buffer.size()); + current_offset += ccw_buffer.size(); } - return ContextSwitchOperation(std::move(actions)); + return config_buffer.release(); } -static Expected> parse_operations( - const google::protobuf::RepeatedPtrField &operations_proto, - const SupportedFeatures &supported_features) +static hailo_status merge_write_ccw_actions( + std::vector &actions, + ConfigBufferInfoMap &config_buffer_infos, + const std::vector &write_ccw_actions) { - CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(operations_proto.size()), HAILO_INVALID_HEF, - "Failed to parse HEF. Invalid operations_count: {}.", operations_proto.size()); - std::vector operations; - operations.reserve(operations_proto.size()); - for (const auto &operation_proto : operations_proto) { - auto operation = parse_operation(operation_proto, supported_features); - CHECK_EXPECTED(operation); - operations.emplace_back(operation.release()); + // Map between config stream index and vector of config buffers. + std::map> ccw_buffers_per_config_streams; + for (const auto *write_ccw_action : write_ccw_actions) { + CHECK(IS_FIT_IN_UINT8(write_ccw_action->cfg_channel_index()), HAILO_INVALID_HEF, + "Invalid cfg channel index"); + const auto config_stream_index = static_cast(write_ccw_action->cfg_channel_index()); + const auto write_ccw_buffer = MemoryView::create_const(write_ccw_action->data().data(), write_ccw_action->data().size()); + ccw_buffers_per_config_streams[config_stream_index].emplace_back(write_ccw_buffer); } - return operations; -} -static hailo_status update_parsing_info(uint8_t cfg_index, uint32_t data_length, ConfigBufferInfoMap &results) -{ - CHECK(cfg_index < CONTROL_PROTOCOL__MAX_CFG_CHANNELS, HAILO_INVALID_HEF, "Invalid cfg_index"); + for (const auto &ccw_buffers_per_config_stream : ccw_buffers_per_config_streams) { + const auto config_stream_index = ccw_buffers_per_config_stream.first; + const auto &ccw_buffers = ccw_buffers_per_config_stream.second; + auto config_buffer = build_config_buffer(ccw_buffers); + CHECK_EXPECTED_AS_STATUS(config_buffer); + + assert(config_buffer->size() < std::numeric_limits::max()); + config_buffer_infos[config_stream_index].emplace_back(static_cast(config_buffer->size())); - if (contains(results, cfg_index)) { - results.at(cfg_index).push_back(data_length); - return HAILO_SUCCESS; + const size_t total_ccw_burst = ccw_buffers.size(); + auto action = WriteDataCcwAction::create(config_buffer.release(), config_stream_index, total_ccw_burst); + + CHECK_EXPECTED_AS_STATUS(action); + actions.emplace_back(action.release()); } - // If we got here, the current cfg_index's info is parsed for the first time - results.emplace(cfg_index, std::vector(1, data_length)); return HAILO_SUCCESS; } -static Expected get_config_buffer_info( - const google::protobuf::RepeatedPtrField &operations) +static hailo_status parse_operation(std::vector &actions, + ConfigBufferInfoMap &config_buffer_infos, + const ProtoHEFOperation &operation_proto, + const SupportedFeatures &supported_features) { - auto status = HAILO_UNINITIALIZED; - ConfigBufferInfoMap results; + auto trigger_action = parse_trigger_action(operation_proto.trigger()); + CHECK_EXPECTED_AS_STATUS(trigger_action); + actions.emplace_back(trigger_action.release()); - for (const auto &operation : operations) { - for (const auto &action : operation.actions()) { - if (ProtoHEFAction::kWriteDataCcw == action.action_case()) { - CHECK_AS_EXPECTED(IS_FIT_IN_UINT8(action.write_data_ccw().cfg_channel_index()), HAILO_INVALID_HEF, - "Invalid cfg index {}", action.write_data_ccw().cfg_channel_index()); - status = update_parsing_info(static_cast(action.write_data_ccw().cfg_channel_index()), - static_cast(action.write_data_ccw().data().length()), results); - CHECK_SUCCESS_AS_EXPECTED(status); + // If current_write_ccw_actions isn't empty, means we currently parsing a group of consecutive write ccw actions. + // we will merge those actions into one write ccw per config channel. + std::vector current_write_ccw_actions; + + for (int action_index = 0; action_index < operation_proto.actions_size(); action_index++) { + const auto &proto_action = operation_proto.actions(action_index); + if (proto_action.action_case() == ProtoHEFAction::kWriteDataCcw) { + // Keep in vector, parse later + current_write_ccw_actions.push_back(&proto_action.write_data_ccw()); + + const auto next_action_index = action_index + 1; + const bool is_last_ccw = + (next_action_index == operation_proto.actions_size()) || + (operation_proto.actions(next_action_index).action_case() != ProtoHEFAction::kWriteDataCcw); + if (is_last_ccw) { + auto status = merge_write_ccw_actions(actions, config_buffer_infos, current_write_ccw_actions); + CHECK_SUCCESS(status); + current_write_ccw_actions.clear(); } + } else { + auto action = parse_action(proto_action, supported_features); + CHECK_EXPECTED_AS_STATUS(action); + actions.emplace_back(action.release()); } } - return results; + assert(current_write_ccw_actions.empty()); + + return HAILO_SUCCESS; } -Expected HefUtils::parse_preliminary_context(const ProtoHEFPreliminaryConfig &preliminary_proto, +static Expected parse_operations( + const google::protobuf::RepeatedPtrField &operations_proto, const SupportedFeatures &supported_features) { - auto operations = parse_operations(preliminary_proto.operation(), supported_features); - CHECK_EXPECTED(operations); + std::vector actions; + ConfigBufferInfoMap config_buffer_infos; + + for (const auto &operation_proto : operations_proto) { + auto status = parse_operation(actions, config_buffer_infos, operation_proto, supported_features); + CHECK_SUCCESS_AS_EXPECTED(status); + } - auto config_buffer_infos = get_config_buffer_info(preliminary_proto.operation()); - CHECK_EXPECTED(config_buffer_infos); + return ContextMetadata(std::move(actions), std::move(config_buffer_infos)); +} - return PreliminaryContextMetadata(operations.release(), config_buffer_infos.release()); +Expected HefUtils::parse_preliminary_context(const ProtoHEFPreliminaryConfig &preliminary_proto, + const SupportedFeatures &supported_features) +{ + return parse_operations(preliminary_proto.operation(), supported_features); } Expected HefUtils::parse_single_dynamic_context(const ProtoHEFCoreOpMock &core_op, const ProtoHEFContext &context_proto, uint8_t context_index, const SupportedFeatures &supported_features) { - auto operations = parse_operations(context_proto.operations(), supported_features); - CHECK_EXPECTED(operations); - - auto config_buffer_infos = get_config_buffer_info(context_proto.operations()); - CHECK_EXPECTED(config_buffer_infos); - - ContextMetadata context_metadata(operations.release(), config_buffer_infos.release()); + auto context_metadata_exp = parse_operations(context_proto.operations(), supported_features); + CHECK_EXPECTED(context_metadata_exp); + ContextMetadata context_metadata = context_metadata_exp.release(); - for (const auto &edge_layer : context_proto.metadata().edge_layers()) { + for (const auto &edge_layer : context_proto.metadata().edge_layers()) { if (ProtoHEFEdgeConnectionType::PROTO__EDGE_CONNECTION_TYPE__BOUNDARY == edge_layer.context_switch_info().edge_connection_type()) { auto status = fill_boundary_layers_info(core_op, context_index, edge_layer, @@ -1868,7 +2148,6 @@ Expected HefUtils::parse_single_dynamic_context(const ProtoHEFC context_index); CHECK_SUCCESS_AS_EXPECTED(status); - return context_metadata; } @@ -2172,10 +2451,10 @@ Expected> Hef::Impl::get_sorted_output_names(const std: LOGGER__ERROR("Did not find network group of name {}", net_group_name); return make_unexpected(HAILO_INVALID_HEF); } - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED(network_group_metadata); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED(core_op_metadata); - auto res = network_group_metadata->get_sorted_output_names(); + auto res = core_op_metadata->get_sorted_output_names(); return res; } @@ -2204,9 +2483,9 @@ static Expected parse_ccw_buffer(const std::string &ccw_buffer) return write_memory_info; } -/* HcpConfigNetworkGroup funcs */ +/* HcpConfigCoreOp funcs */ -Expected> Hef::Impl::create_single_context_network_group_config(const ProtoHEFPreliminaryConfig& proto_config) +Expected> Hef::Impl::create_single_context_core_op_config(const ProtoHEFPreliminaryConfig& proto_config) { std::vector config_buffers; @@ -2308,19 +2587,19 @@ bool Hef::Impl::contains_ddr_layers(const ProtoHEFCoreOpMock& core_op) Expected> Hef::Impl::get_stream_names_from_vstream_name(const std::string &vstream_name, const std::string &net_group_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED(network_group_metadata); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED(core_op_metadata); - return network_group_metadata->get_stream_names_from_vstream_name(vstream_name); + return core_op_metadata->get_stream_names_from_vstream_name(vstream_name); } Expected> Hef::Impl::get_vstream_names_from_stream_name(const std::string &stream_name, const std::string &net_group_name) { - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED(network_group_metadata); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED(core_op_metadata); - return network_group_metadata->get_vstream_names_from_stream_name(stream_name); + return core_op_metadata->get_vstream_names_from_stream_name(stream_name); } Expected Hef::Impl::get_vstream_name_from_original_name_mux(const std::string &original_name, const ProtoHefEdge &layer) @@ -2530,6 +2809,147 @@ Expected> Hef::get_network_groups_infos( return pimpl->get_network_groups_infos(); } +Expected> Hef::Impl::get_stream_infos_description(const std::string &network_group_name, const std::string &network_name) +{ + std::vector infos_strings; + auto input_stream_infos = get_input_stream_infos(network_group_name, network_name); + CHECK_EXPECTED(input_stream_infos, "Failed to parse input stream infos"); + auto output_stream_infos = get_output_stream_infos(network_group_name, network_name); + CHECK_EXPECTED(output_stream_infos, "Failed to parse output stream infos"); + infos_strings.reserve(input_stream_infos.value().size() + output_stream_infos.value().size()); + std::string infos_string; + + for (const auto &stream_info : input_stream_infos.value()) { + auto shape_str = get_shape_str(stream_info); + infos_string = "Input " + std::string(stream_info.name) + " " + shape_str + "\n"; + infos_strings.emplace_back(infos_string); + } + + for (const auto &stream_info : output_stream_infos.value()) { + auto shape_str = get_shape_str(stream_info); + infos_string = "Output " + std::string(stream_info.name) + " " + shape_str + "\n"; + infos_strings.emplace_back(infos_string); + } + + return infos_strings; +} + +Expected> Hef::Impl::get_vstream_infos_description(const std::string &network_group_name, const std::string &network_name) +{ + std::vector infos_strings; + auto input_vstream_infos = get_input_vstream_infos(network_group_name, network_name); + CHECK_EXPECTED(input_vstream_infos, "Failed to parse input vstream infos"); + auto output_vstream_infos = get_output_vstream_infos(network_group_name, network_name); + CHECK_EXPECTED(output_vstream_infos, "Failed to parse output stream infos"); + infos_strings.reserve(input_vstream_infos.value().size() + output_vstream_infos.value().size()); + std::string infos_string; + + for (const auto &vstream_info : input_vstream_infos.value()) { + auto shape_str = get_shape_str(vstream_info); + infos_string = "Input " + std::string(vstream_info.name) + " " + shape_str + "\n"; + infos_strings.emplace_back(infos_string); + } + + for (const auto &vstream_info : output_vstream_infos.value()) { + auto shape_str = get_shape_str(vstream_info); + infos_string = "Output " + std::string(vstream_info.name) + " " + shape_str + "\n"; + infos_strings.emplace_back(infos_string); + } + + return infos_strings; +} + +Expected> Hef::Impl::get_post_processes_infos_description(const std::string &network_group_name) +{ + std::vector infos_strings; + std::string infos_string; + + auto post_process = post_process_ops(network_group_name); + for (const auto &post_process_info : post_process) { + infos_string = post_process_info->op->get_op_description(); + infos_string += ", Bbox size: " + std::to_string(post_process_info->nms_info.bbox_size) + + ", Max bboxes per class: " + std::to_string(post_process_info->nms_info.max_bboxes_per_class); + } + /* If the string is empty there is no need to continue. */ + if (infos_string.empty()) { + return infos_strings; + } + + /* Splitting the info string and assembling the vector from each token. */ + std::string token; + size_t pos; + while ((pos = infos_string.find(",")) != std::string::npos) { + token.assign(infos_string.begin(), infos_string.begin() + pos); + token += "\n"; + infos_strings.push_back(token); + /* Assuming each token is separated with ", " */ + infos_string.erase(0, pos + SKIP_SPACE_COMMA_CHARACTERS); + } + infos_strings.push_back(infos_string + "\n"); + + return infos_strings; +} + +Expected Hef::get_hef_description(bool stream_infos, bool vstream_infos) +{ + auto arch = get_hef_device_arch(); + CHECK_EXPECTED(arch); + return pimpl->get_hef_description(stream_infos, vstream_infos, arch.value()); +} + +Expected Hef::Impl::get_hef_description(bool stream_infos, bool vstream_infos, hailo_device_architecture_t device_arch) +{ + std::string hef_infos; + auto hef_arch_str = HailoRTCommon::get_device_arch_str(device_arch); + hef_infos += "Architecture HEF was compiled for: " + hef_arch_str + "\n"; + + auto network_group_infos = get_network_groups_infos(); + CHECK_EXPECTED(network_group_infos); + for (const auto &network_group_info : network_group_infos.release()) { + auto core_op_meta_data = get_core_op_metadata(network_group_info.name); + CHECK_EXPECTED(core_op_meta_data); + auto number_of_contexts = core_op_meta_data->get_contexts_count(); + auto contexts_str = (network_group_info.is_multi_context ? "Multi Context - Number of contexts: " + std::to_string(number_of_contexts) : "Single Context"); + hef_infos += "Network group name: " + std::string(network_group_info.name) + ", " + contexts_str + "\n"; + + auto network_infos = get_network_infos(network_group_info.name); + CHECK_EXPECTED(network_infos, "Failed to parse networks infos"); + + for (const auto &network_info : network_infos.value()) { + hef_infos += add_tabs(1) + "Network name: " + network_info.name + "\n"; + if (stream_infos) { + auto stream_infos_strings = get_stream_infos_description(network_group_info.name, network_info.name); + CHECK_EXPECTED(stream_infos_strings); + hef_infos += add_tabs(2) + "Stream infos:" + "\n"; + for (auto stream_info_string : stream_infos_strings.value()) { + hef_infos += add_tabs(3) + stream_info_string; + } + } + if (vstream_infos) { + auto vstream_infos_strings = get_vstream_infos_description(network_group_info.name, network_info.name); + CHECK_EXPECTED(vstream_infos_strings); + hef_infos += add_tabs(2) + "VStream infos:" + "\n"; + for (auto vstream_info_string : vstream_infos_strings.value()) { + hef_infos += add_tabs(3) + vstream_info_string; + } + + auto post_processes_infos_strings = get_post_processes_infos_description(network_group_info.name); + CHECK_EXPECTED(post_processes_infos_strings); + /* Validating that there is a postprocess info. */ + if (post_processes_infos_strings->size() <= 0) { + continue; + } + hef_infos += add_tabs(3) + "Operation:" + "\n"; + for (auto post_process_info_string : post_processes_infos_strings.value()) { + hef_infos += add_tabs(4) + post_process_info_string; + } + } + } + } + + return hef_infos; +} + Expected> Hef::Impl::get_network_groups_infos() { std::vector results; @@ -2600,9 +3020,9 @@ hailo_status Hef::Impl::fill_missing_input_vstream_params_with_default(const std const std::string &network_name, std::map &input_vstreams_params, bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) { - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED_AS_STATUS(network_group_metadata); - auto input_vstream_infos = network_group_metadata->get_input_vstream_infos(network_name); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED_AS_STATUS(core_op_metadata); + auto input_vstream_infos = core_op_metadata->get_input_vstream_infos(network_name); CHECK_EXPECTED_AS_STATUS(input_vstream_infos); return fill_missing_vstream_params_with_default(input_vstreams_params, input_vstream_infos.value(), @@ -2613,9 +3033,9 @@ hailo_status Hef::Impl::fill_missing_output_vstream_params_with_default(const st const std::string &network_name, std::map &output_vstream_params, bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) { - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED_AS_STATUS(network_group_metadata); - auto output_vstream_infos = network_group_metadata->get_output_vstream_infos(network_name); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED_AS_STATUS(core_op_metadata); + auto output_vstream_infos = core_op_metadata->get_output_vstream_infos(network_name); CHECK_EXPECTED_AS_STATUS(output_vstream_infos); return fill_missing_vstream_params_with_default(output_vstream_params, output_vstream_infos.value(), @@ -2686,16 +3106,16 @@ Expected> Hef::create_stream_pa Expected> Hef::Impl::create_stream_parameters_by_name( const std::string &net_group_name, hailo_stream_interface_t stream_interface) { - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED(network_group_metadata); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED(core_op_metadata); std::map results; - for (auto &input_layer : network_group_metadata->get_input_layer_infos()) { + for (auto &input_layer : core_op_metadata->get_input_layer_infos()) { auto params = HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_H2D_STREAM); CHECK_EXPECTED(params); results.emplace(std::make_pair(input_layer.name, params.release())); } - for (auto &output_layer : network_group_metadata->get_output_layer_infos()) { + for (auto &output_layer : core_op_metadata->get_output_layer_infos()) { auto params = HailoRTDefaults::get_stream_parameters(stream_interface, HAILO_D2H_STREAM); CHECK_EXPECTED(params); results.emplace(std::make_pair(output_layer.name, params.release())); @@ -2716,12 +3136,12 @@ Expected> Hef::Impl::create_ne auto core_op = get_core_op_by_net_group_name(net_group_name); CHECK_EXPECTED(core_op); - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED(network_group_metadata); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED(core_op_metadata); std::map results; - if (network_group_metadata->supported_features().multi_network_support) { + if (core_op_metadata->supported_features().multi_network_support) { CHECK_AS_EXPECTED((core_op.value()->networks_names.size() != 0), HAILO_INTERNAL_FAILURE, "Hef support multiple networks, but no networks found in the proto"); for (const auto &partial_network_name : core_op.value()->networks_names) { @@ -2754,18 +3174,18 @@ Expected> Hef::Impl::create_str const std::string &net_group_name, hailo_stream_interface_t output_interface, const hailo_mipi_input_stream_params_t &mipi_params) { - auto network_group_metadata = get_network_group_metadata(net_group_name); - CHECK_EXPECTED(network_group_metadata); + auto core_op_metadata = get_core_op_metadata(net_group_name); + CHECK_EXPECTED(core_op_metadata); std::map results; - for (auto &input_layer : network_group_metadata->get_input_layer_infos()) { + for (auto &input_layer : core_op_metadata->get_input_layer_infos()) { hailo_stream_parameters_t params = {}; params.direction = HAILO_H2D_STREAM; params.stream_interface = HAILO_STREAM_INTERFACE_MIPI; params.mipi_input_params = mipi_params; results.emplace(std::make_pair(input_layer.name, params)); } - for (auto &output_layer : network_group_metadata->get_output_layer_infos()) { + for (auto &output_layer : core_op_metadata->get_output_layer_infos()) { auto params = HailoRTDefaults::get_stream_parameters(output_interface, HAILO_D2H_STREAM); CHECK_EXPECTED(params); results.emplace(std::make_pair(output_layer.name, params.release())); diff --git a/hailort/libhailort/src/hef_internal.hpp b/hailort/libhailort/src/hef/hef_internal.hpp similarity index 88% rename from hailort/libhailort/src/hef_internal.hpp rename to hailort/libhailort/src/hef/hef_internal.hpp index f73882b..f2a3b53 100644 --- a/hailort/libhailort/src/hef_internal.hpp +++ b/hailort/libhailort/src/hef/hef_internal.hpp @@ -29,15 +29,17 @@ #include "hailo/expected.hpp" #include "hailo/hef.hpp" #include "hailo/network_group.hpp" -#include "context_switch/context_switch_actions.hpp" -#include "layer_info.hpp" -#include "network_group_metadata.hpp" -#include "hailort_defaults.hpp" -#include "control_protocol.h" -#include "pipeline.hpp" +#include "hailo/hailort_defaults.hpp" -#include "control_protocol.hpp" +#include "hef/core_op_metadata.hpp" +#include "hef/layer_info.hpp" +#include "hef/context_switch_actions.hpp" +#include "net_flow/ops/op.hpp" +#include "net_flow/pipeline/pipeline.hpp" +#include "core_op/core_op.hpp" +#include "device_common/control_protocol.hpp" +#include "control_protocol.h" #include #include #include @@ -46,11 +48,12 @@ extern "C" { #include "md5.h" } + namespace hailort { -class ResourcesManager; -class ConfigBuffer; +class CoreOpMetadata; +class CoreOp; using ProtoHEFNetworkGroupPtr = std::shared_ptr; struct ProtoHEFCoreOpMock; @@ -127,6 +130,13 @@ typedef enum { HEF__FORMAT__F8CR, } HEF__net_io_formatter_type_t; +struct NetFlowElement +{ + std::string name; + std::shared_ptr op; + std::set input_streams; + hailo_nms_info_t nms_info; +}; const static uint32_t SUPPORTED_EXTENSIONS_BITSET_SIZE = 1000; static const std::vector SUPPORTED_EXTENSIONS = { @@ -183,7 +193,7 @@ class OutputStreamBase; // Forward declerations struct WriteMemoryInfo; class Device; -class VdmaConfigNetworkGroup; +class VdmaConfigCoreOp; class VdmaDevice; class HailoRTDriver; @@ -199,7 +209,7 @@ public: const std::vector& network_groups() const; const std::vector& core_ops(const std::string &net_group_name) const; - const std::vector> post_process_ops(const std::string &net_group_name) const; + const std::vector> post_process_ops(const std::string &net_group_name) const; Expected> get_network_group_and_network_name(const std::string &name); @@ -226,7 +236,7 @@ public: Expected get_number_of_output_streams(const std::string &net_group_name=""); ProtoHEFHwArch get_device_arch(); Expected get_bottleneck_fps(const std::string &net_group_name=""); - static bool contains_ddr_layers(const ProtoHEFCoreOpMock &net_group); + static bool contains_ddr_layers(const ProtoHEFCoreOpMock &core_op); static hailo_status validate_core_op_unique_layer_names(const ProtoHEFCoreOpMock &core_op); Expected> get_network_input_vstream_infos(const std::string &net_group_name="", const std::string &network_name=""); @@ -244,14 +254,14 @@ public: std::vector get_network_groups_names(); Expected> get_network_groups_infos(); - Expected create_configure_params(hailo_stream_interface_t stream_interface, const std::string &network_gorup_name); + Expected create_configure_params(hailo_stream_interface_t stream_interface, const std::string &network_group_name); Expected create_configure_params_mipi_input(hailo_stream_interface_t output_interface, - const hailo_mipi_input_stream_params_t &mipi_params, const std::string &network_gorup_name); + const hailo_mipi_input_stream_params_t &mipi_params, const std::string &network_group_name); - static Expected> create_single_context_network_group_config( + static Expected> create_single_context_core_op_config( const ProtoHEFPreliminaryConfig& proto_config); - static Expected> get_core_op_per_arch(const ProtoHEFCoreOpMock &base_net_group, + static Expected> get_core_op_per_arch(const ProtoHEFCoreOpMock &core_op, ProtoHEFHwArch hef_arch, hailo_device_architecture_t device_arch, uint32_t partial_clusters_layout_bitmap); Expected> create_stream_parameters_by_name( @@ -279,19 +289,15 @@ public: static hailo_status fill_missing_vstream_params_with_default(std::map &vstream_params, std::vector &name_to_format_info, bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size); - // Also adds information to NetworkGroupMetadata - Expected>> create_network_group_ops(const ProtoHEFNetworkGroup &network_group_proto, - NetworkGroupMetadata &network_group_meta_data) const; + // Also adds information to CoreOpMetadata + // TODO: When supporting multiple core ops in same netflow - Change metadata param to a map of core_ops_metadata. + Expected>> create_net_flow_ops(const ProtoHEFNetworkGroup &network_group_proto, + CoreOpMetadata &core_op_metadata) const; + // TODO: Should return map of NG's core_ops metadata? + Expected get_core_op_metadata(const std::string &network_group_name, uint32_t partial_clusters_layout_bitmap = PARTIAL_CLUSTERS_LAYOUT_IGNORE); - Expected get_network_group_metadata(const std::string &network_group_name, uint32_t partial_clusters_layout_bitmap = PARTIAL_CLUSTERS_LAYOUT_IGNORE) - { - CHECK_AS_EXPECTED(contains(m_network_group_metadata_per_arch, network_group_name), HAILO_NOT_FOUND, - "Network group with name {} wasn't found", network_group_name); - auto metadata_per_arch = m_network_group_metadata_per_arch.at(network_group_name); - auto metadata = metadata_per_arch.get_metadata(partial_clusters_layout_bitmap); - return metadata; - } + Expected get_hef_description(bool stream_infos, bool vstream_infos, hailo_device_architecture_t device_arch); const MD5_SUM_t &md5() const { @@ -326,20 +332,8 @@ public: return HAILO_SUCCESS; } - hailo_status validate_boundary_streams_were_created(const std::string &network_group_name, ConfiguredNetworkGroup &network_group) - { - auto number_of_inputs = get_number_of_input_streams(network_group_name); - CHECK_EXPECTED_AS_STATUS(number_of_inputs); - CHECK((number_of_inputs.value() == network_group.get_input_streams().size()), - HAILO_INVALID_ARGUMENT, "passed configure_params for network group {} did not contain all input streams", network_group_name); - - auto number_of_outputs = get_number_of_output_streams(network_group_name); - CHECK_EXPECTED_AS_STATUS(number_of_inputs); - CHECK((number_of_outputs.value() == network_group.get_output_streams().size()), - HAILO_INVALID_ARGUMENT, "passed configure_params for network group {} did not contain all output streams", network_group_name); - - return HAILO_SUCCESS; - } + // TODO: HRT-8875 - Change to validate all core ops under same ng or use this func in the new configure API and use this to validate each op when configured. + hailo_status validate_boundary_streams_were_created(const std::string &network_group_name, std::shared_ptr core_op); #ifdef HAILO_SUPPORT_MULTI_PROCESS const MemoryView get_hef_memview(); @@ -377,7 +371,10 @@ private: static Expected get_vstream_name_from_original_name_mux(const std::string &original_name, const ProtoHefEdge &layer); static Expected> get_original_names_from_vstream_name_mux(const std::string &vstream_name, const ProtoHefEdge &layer); - Expected create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op); + Expected create_metadata_per_arch(const ProtoHEFCoreOpMock &core_op); + Expected> get_stream_infos_description(const std::string &network_group_name, const std::string &network_name); + Expected> get_vstream_infos_description(const std::string &network_group_name, const std::string &network_name); + Expected> get_post_processes_infos_description(const std::string &network_group_name); // Hef information ProtoHEFHeader m_header; @@ -385,7 +382,7 @@ private: SupportedFeatures m_supported_features; std::vector m_groups; std::map> m_core_ops_per_group; - std::map>> m_post_process_ops_per_group; + std::map>> m_post_process_ops_per_group; std::vector m_hef_extensions; std::vector m_hef_optional_extensions; std::bitset m_supported_extensions_bitset; @@ -395,8 +392,8 @@ private: Buffer m_hef_buffer; #endif // HAILO_SUPPORT_MULTI_PROCESS - // NetworkGroups information - std::map m_network_group_metadata_per_arch; // TODO: keep meta data per core_op (HRT-8639) + // CoreOps information - TODO: Should be a map of map, mapping network_groups to it's core ops (second map is mapping core op name to its metadata). + std::map m_core_op_per_arch; }; // TODO: Make this part of a namespace? (HRT-2881) @@ -456,7 +453,7 @@ public: const std::vector &context_ddr_input_layers, const std::vector &context_ddr_output_layers, const uint8_t context_index); - static Expected parse_preliminary_context(const ProtoHEFPreliminaryConfig &preliminary_proto, + static Expected parse_preliminary_context(const ProtoHEFPreliminaryConfig &preliminary_proto, const SupportedFeatures &supported_features); static Expected parse_single_dynamic_context(const ProtoHEFCoreOpMock &core_op, const ProtoHEFContext &context_proto, uint8_t context_index, const SupportedFeatures &supported_features); @@ -473,7 +470,7 @@ public: const std::string &net_group_name, const SupportedFeatures &supported_features); static std::string get_network_group_name(const ProtoHEFNetworkGroup &net_group, const SupportedFeatures &supported_features); - static std::string get_network_name(const ProtoHEFCoreOpMock &net_group, const std::string &partial_network_name); + static std::string get_network_name(const ProtoHEFCoreOpMock &core_op, const std::string &partial_network_name); static std::string get_network_name(const std::string &net_group_name, const std::string &partial_network_name); private: diff --git a/hailort/libhailort/src/layer_info.hpp b/hailort/libhailort/src/hef/layer_info.hpp similarity index 91% rename from hailort/libhailort/src/layer_info.hpp rename to hailort/libhailort/src/hef/layer_info.hpp index 140fdc2..af1ed07 100644 --- a/hailort/libhailort/src/layer_info.hpp +++ b/hailort/libhailort/src/hef/layer_info.hpp @@ -12,14 +12,16 @@ #include "hailo/hailort.h" #include "hailo/hailort_common.hpp" -#include "hailort_defaults.hpp" -#include "control_protocol.h" +#include "hailo/hailort_defaults.hpp" + #include "os/hailort_driver.hpp" +#include "control_protocol.h" #include #include #include + namespace hailort { @@ -166,6 +168,18 @@ public: return res; } + static Expected get_transfer_size(const LayerInfo &layer_info) { + switch (layer_info.type) { + case LayerType::BOUNDARY: + case LayerType::INTER_CONTEXT: + return layer_info.nn_stream_config.periph_bytes_per_buffer * layer_info.nn_stream_config.periph_buffers_per_frame; + case LayerType::DDR: + return layer_info.nn_stream_config.periph_bytes_per_buffer * layer_info.ddr_info.total_buffers_per_frame; + default: + return make_unexpected(HAILO_NOT_IMPLEMENTED); + } + } + private: static hailo_vstream_info_t get_vstream_info_from_layer_info_impl(const LayerInfo &layer_info) { diff --git a/hailort/libhailort/src/hw_consts.hpp b/hailort/libhailort/src/hw_consts.hpp index c3598e1..4faec77 100644 --- a/hailort/libhailort/src/hw_consts.hpp +++ b/hailort/libhailort/src/hw_consts.hpp @@ -28,7 +28,6 @@ #define VDMA_CHANNEL_CONTROL_OFFSET (0x00) #define VDMA_CHANNEL_NUM_AVAIL_OFFSET (0x02) #define VDMA_CHANNEL_NUM_PROC_OFFSET (0x04) -#define VDMA_CHANNEL_ERROR_OFFSET (0x08) #endif /* _HAILO_HW_CONSTS_HPP_ */ diff --git a/hailort/libhailort/src/mipi/CMakeLists.txt b/hailort/libhailort/src/mipi/CMakeLists.txt new file mode 100644 index 0000000..bd101a2 --- /dev/null +++ b/hailort/libhailort/src/mipi/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/mipi_stream.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/mipi_stream.cpp b/hailort/libhailort/src/mipi/mipi_stream.cpp similarity index 94% rename from hailort/libhailort/src/mipi_stream.cpp rename to hailort/libhailort/src/mipi/mipi_stream.cpp index 4a9115c..e7c92fe 100644 --- a/hailort/libhailort/src/mipi_stream.cpp +++ b/hailort/libhailort/src/mipi/mipi_stream.cpp @@ -12,17 +12,20 @@ * implemented. **/ -#include +#include "hailo/hailort.h" + #include "common/utils.hpp" -#include -#include "mipi_stream.hpp" + +#include "device_common/control.hpp" +#include "mipi/mipi_stream.hpp" + namespace hailort { MipiInputStream::MipiInputStream(Device &device, const CONTROL_PROTOCOL__mipi_input_config_params_t &mipi_params, - EventPtr &&network_group_activated_event, const LayerInfo &layer_info, hailo_status &status) : - InputStreamBase(layer_info, HAILO_STREAM_INTERFACE_MIPI, std::move(network_group_activated_event), status), + EventPtr &&core_op_activated_event, const LayerInfo &layer_info, hailo_status &status) : + InputStreamBase(layer_info, HAILO_STREAM_INTERFACE_MIPI, std::move(core_op_activated_event), status), m_device(device), m_is_stream_activated(false), m_mipi_input_params(mipi_params) @@ -101,7 +104,7 @@ CONTROL_PROTOCOL__mipi_input_config_params_t MipiInputStream::hailo_mipi_params_ } // Note: Mipi streams don't work with dynamic batch sizes -hailo_status MipiInputStream::activate_stream(uint16_t /* dynamic_batch_size */) +hailo_status MipiInputStream::activate_stream(uint16_t /* dynamic_batch_size */, bool /* resume_pending_stream_transfers */) { hailo_status status = HAILO_UNINITIALIZED; CONTROL_PROTOCOL__config_stream_params_t params = {}; @@ -141,12 +144,12 @@ hailo_status MipiInputStream::sync_write_all_raw_buffer_no_transform_impl(void * Expected> MipiInputStream::create(Device &device, const LayerInfo &edge_layer, const hailo_mipi_input_stream_params_t ¶ms, - EventPtr network_group_activated_event) + EventPtr core_op_activated_event) { auto mipi_params = MipiInputStream::hailo_mipi_params_to_control_mipi_params(params); auto status = HAILO_UNINITIALIZED; std::unique_ptr stream(new (std::nothrow) MipiInputStream(device, mipi_params, - std::move(network_group_activated_event), edge_layer, status)); + std::move(core_op_activated_event), edge_layer, status)); CHECK_AS_EXPECTED(stream != nullptr, HAILO_OUT_OF_HOST_MEMORY); CHECK_SUCCESS_AS_EXPECTED(status); diff --git a/hailort/libhailort/src/mipi_stream.hpp b/hailort/libhailort/src/mipi/mipi_stream.hpp similarity index 80% rename from hailort/libhailort/src/mipi_stream.hpp rename to hailort/libhailort/src/mipi/mipi_stream.hpp index 3ca6e43..b52597f 100644 --- a/hailort/libhailort/src/mipi_stream.hpp +++ b/hailort/libhailort/src/mipi/mipi_stream.hpp @@ -12,18 +12,20 @@ #ifndef HAILO_MIPI_STREAM_H_ #define HAILO_MIPI_STREAM_H_ -#include "stream_internal.hpp" #include "hailo/hailort.h" #include "hailo/expected.hpp" #include "hailo/event.hpp" +#include "stream_common/stream_internal.hpp" + + namespace hailort { class MipiInputStream : public InputStreamBase { private: MipiInputStream(Device &device, const CONTROL_PROTOCOL__mipi_input_config_params_t &mipi_params, - EventPtr &&network_group_activated_event, const LayerInfo &layer_info, hailo_status &status); + EventPtr &&core_op_activated_event, const LayerInfo &layer_info, hailo_status &status); static CONTROL_PROTOCOL__mipi_input_config_params_t hailo_mipi_params_to_control_mipi_params( const hailo_mipi_input_stream_params_t ¶ms); @@ -40,18 +42,10 @@ protected: public: static Expected> create(Device &device, const LayerInfo &edge_layer, const hailo_mipi_input_stream_params_t ¶ms, - EventPtr network_group_activated_event); - - MipiInputStream(MipiInputStream&& other) : - InputStreamBase(std::move(other)), - m_device(other.m_device), - m_is_stream_activated(std::exchange(other.m_is_stream_activated, false)), - m_mipi_input_params(std::move(other.m_mipi_input_params)) - {} - + EventPtr core_op_activated_event); virtual ~MipiInputStream(); - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; virtual hailo_status deactivate_stream() override; virtual hailo_stream_interface_t get_interface() const override { return HAILO_STREAM_INTERFACE_MIPI; } virtual std::chrono::milliseconds get_timeout() const override; diff --git a/hailort/libhailort/src/net_flow/CMakeLists.txt b/hailort/libhailort/src/net_flow/CMakeLists.txt index 430284e..ece9aa3 100644 --- a/hailort/libhailort/src/net_flow/CMakeLists.txt +++ b/hailort/libhailort/src/net_flow/CMakeLists.txt @@ -1 +1,16 @@ -cmake_minimum_required(VERSION 3.0.0) \ No newline at end of file +cmake_minimum_required(VERSION 3.0.0) + +set(HAILORT_OPS_CPP_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/ops/nms_post_process.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ops/yolo_post_process.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ops/ssd_post_process.cpp +) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/pipeline.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/inference_pipeline.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pipeline/vstream.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} ${HAILORT_OPS_CPP_SOURCES} PARENT_SCOPE) +set(HAILORT_OPS_CPP_SOURCES ${HAILORT_OPS_CPP_SOURCES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/net_flow/ops/nms_post_process.cpp b/hailort/libhailort/src/net_flow/ops/nms_post_process.cpp new file mode 100644 index 0000000..1bf1859 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/nms_post_process.cpp @@ -0,0 +1,133 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file nms_post_process.cpp + * @brief NMS post process + * + * Reference code: https://github.com/winfredsu/ssd_postprocessing/blob/master/ssd_postprocessing.py + **/ + +#include "net_flow/ops/nms_post_process.hpp" + +namespace hailort +{ +namespace net_flow +{ + float NmsPostProcessOp::compute_iou(const hailo_bbox_float32_t &box_1, const hailo_bbox_float32_t &box_2) + { + const float overlap_area_width = std::min(box_1.x_max, box_2.x_max) - std::max(box_1.x_min, box_2.x_min); + const float overlap_area_height = std::min(box_1.y_max, box_2.y_max) - std::max(box_1.y_min, box_2.y_min); + if (overlap_area_width <= 0.0f || overlap_area_height <= 0.0f) { + return 0.0f; + } + const float intersection = overlap_area_width * overlap_area_height; + const float box_1_area = (box_1.y_max - box_1.y_min) * (box_1.x_max - box_1.x_min); + const float box_2_area = (box_2.y_max - box_2.y_min) * (box_2.x_max - box_2.x_min); + const float union_area = (box_1_area + box_2_area - intersection); + + return (intersection / union_area); + } + + void NmsPostProcessOp::remove_overlapping_boxes(std::vector &detections, std::vector &classes_detections_count) + { + std::sort(detections.begin(), detections.end(), + [](DetectionBbox a, DetectionBbox b) + { return a.m_bbox.score > b.m_bbox.score; }); + + for (size_t i = 0; i < detections.size(); i++) { + if (detections[i].m_bbox.score == REMOVED_CLASS_SCORE) { + // Detection overlapped with a higher score detection + continue; + } + + for (size_t j = i + 1; j < detections.size(); j++) { + if (detections[j].m_bbox.score == REMOVED_CLASS_SCORE) { + // Detection overlapped with a higher score detection + continue; + } + + if (detections[i].m_class_id == detections[j].m_class_id + && (compute_iou(detections[i].m_bbox, detections[j].m_bbox) >= m_nms_config.nms_iou_th)) { + // Remove detections[j] if the iou is higher then the threshold + detections[j].m_bbox.score = REMOVED_CLASS_SCORE; + assert(detections[i].m_class_id < classes_detections_count.size()); + assert(classes_detections_count[detections[j].m_class_id] > 0); + classes_detections_count[detections[j].m_class_id]--; + } + } + } + } + + void NmsPostProcessOp::fill_nms_format_buffer(MemoryView &buffer, const std::vector &detections, + std::vector &classes_detections_count) + { + // Calculate the number of detections before each class, to help us later calculate the buffer_offset for it's detections. + std::vector num_of_detections_before; + num_of_detections_before.reserve(m_nms_config.classes); + uint32_t ignored_detections_count = 0; + for (size_t class_idx = 0; class_idx < m_nms_config.classes; class_idx++) { + if (classes_detections_count[class_idx] > m_nms_config.max_proposals_per_class) { + ignored_detections_count += (classes_detections_count[class_idx] - m_nms_config.max_proposals_per_class); + classes_detections_count[class_idx] = m_nms_config.max_proposals_per_class; + } + + if (0 == class_idx) { + num_of_detections_before[class_idx] = 0; + } + else { + num_of_detections_before[class_idx] = num_of_detections_before[class_idx - 1] + classes_detections_count[class_idx - 1]; + } + + // Fill `bbox_count` value for class_idx in the result buffer + float32_t bbox_count_casted = static_cast(classes_detections_count[class_idx]); + auto buffer_offset = (class_idx * sizeof(bbox_count_casted)) + (num_of_detections_before[class_idx] * sizeof(hailo_bbox_float32_t)); + memcpy((buffer.data() + buffer_offset), &bbox_count_casted, sizeof(bbox_count_casted)); + } + + for (auto &detection : detections) { + if (REMOVED_CLASS_SCORE == detection.m_bbox.score) { + // Detection overlapped with a higher score detection and removed in remove_overlapping_boxes() + continue; + } + if (0 == classes_detections_count[detection.m_class_id]) { + // This class' detections count is higher then m_nms_config.max_proposals_per_class. + // This detection is ignored due to having lower score (detections vector is sorted by score). + continue; + } + + auto buffer_offset = ((detection.m_class_id + 1) * sizeof(float32_t)) + + (num_of_detections_before[detection.m_class_id] * sizeof(hailo_bbox_float32_t)); + + assert((buffer_offset + sizeof(hailo_bbox_float32_t)) <= buffer.size()); + memcpy((hailo_bbox_float32_t*)(buffer.data() + buffer_offset), &detection.m_bbox, sizeof(hailo_bbox_float32_t)); + num_of_detections_before[detection.m_class_id]++; + classes_detections_count[detection.m_class_id]--; + } + + if (0 != ignored_detections_count) { + LOGGER__INFO("{} Detections were ignored, due to `max_bboxes_per_class` defined as {}.", + ignored_detections_count, m_nms_config.max_proposals_per_class); + } + } + + hailo_status NmsPostProcessOp::hailo_nms_format(std::vector &&detections, + MemoryView dst_view, std::vector &classes_detections_count) + { + remove_overlapping_boxes(detections, classes_detections_count); + fill_nms_format_buffer(dst_view, detections, classes_detections_count); + return HAILO_SUCCESS; + } + + std::string NmsPostProcessOp::get_nms_config_description() + { + auto config_info = fmt::format("Score threshold: {:.3f}, Iou threshold: {:.2f}, Classes: {}, Cross classes: {}", + m_nms_config.nms_score_th, m_nms_config.nms_iou_th, m_nms_config.classes, m_nms_config.cross_classes); + if (m_nms_config.background_removal) { + config_info += fmt::format(", Background removal index: {}", m_nms_config.background_removal_index); + } + return config_info; + } +} +} \ No newline at end of file diff --git a/hailort/libhailort/src/net_flow/ops/nms_post_process.hpp b/hailort/libhailort/src/net_flow/ops/nms_post_process.hpp new file mode 100644 index 0000000..8b95a84 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/nms_post_process.hpp @@ -0,0 +1,166 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file op.hpp + * @brief Net-Flow op + * + * https://learnopencv.com/object-detection-using-yolov5-and-opencv-dnn-in-c-and-python : + * The headline '4.3.5 POST-PROCESSING YOLOv5 Prediction Output' contains explanations on the YOLOv5 post-process. + **/ + +#ifndef _HAILO_NET_FLOW_NMS_POST_PROCESS_HPP_ +#define _HAILO_NET_FLOW_NMS_POST_PROCESS_HPP_ + +#include "hailo/hailort.h" +#include "hailo/quantization.hpp" +#include "hailo/buffer.hpp" + +#include "common/utils.hpp" +#include "common/logger_macros.hpp" + +#include "net_flow/ops/op.hpp" + + +namespace hailort +{ +namespace net_flow +{ + +#define INVALID_BBOX_DIM (std::numeric_limits::max()) +#define INVALID_NMS_DETECTION (std::numeric_limits::max()) +#define INVALID_NMS_SCORE (std::numeric_limits::max()) + +struct DetectionBbox +{ + DetectionBbox(float32_t x_min, float32_t y_min, float32_t width, float32_t height, float32_t score, uint32_t class_id) + : m_class_id(class_id), m_bbox{y_min, x_min, (y_min + height), (x_min + width), score} {} + + DetectionBbox(const hailo_bbox_float32_t &bbox, uint32_t class_id) + : m_class_id(class_id), m_bbox(bbox) {} + + DetectionBbox() : DetectionBbox(hailo_bbox_float32_t{ + INVALID_BBOX_DIM, + INVALID_BBOX_DIM, + INVALID_BBOX_DIM, + INVALID_BBOX_DIM, + INVALID_BBOX_DIM + }, INVALID_NMS_DETECTION) {} + + uint32_t m_class_id; + hailo_bbox_float32_t m_bbox; +}; + +struct NmsPostProcessConfig +{ + // User given confidence threshold for a bbox. A bbox will be consider as detection if the + // (objectness * class_score) is higher then the confidence_threshold. + double nms_score_th = 0; + + // User given IOU threshold (intersection over union). This threshold is for performing + // Non-maximum suppression (Removing overlapping boxes). + double nms_iou_th = 0; + + // Maximum amount of bboxes per nms class. + uint32_t max_proposals_per_class = 0; + + // The model's number of classes. (This depends on the dataset that the model trained on). + uint32_t classes = 0; + + // Toggle background class removal from results + bool background_removal = false; + + // Index of background class for background removal + uint32_t background_removal_index = 0; + + // Indicates whether or not NMS performs IOU over different classes for the same box. + // If set to false - NMS won't intersect different classes, and a box could have multiple labels. + bool cross_classes = false; +}; + +static const float32_t REMOVED_CLASS_SCORE = 0.0f; + +class NmsPostProcessOp : public Op +{ +public: + virtual ~NmsPostProcessOp() = default; + + /** + * Computes the IOU ratio of @a box_1 and @a box_2 + */ + static float compute_iou(const hailo_bbox_float32_t &box_1, const hailo_bbox_float32_t &box_2); + +protected: + NmsPostProcessOp(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const std::string &name) + : Op(inputs_metadata, outputs_metadata, name) + , m_nms_config(nms_post_process_config) + {} + + NmsPostProcessConfig m_nms_config; + + template + std::pair get_max_class(const DeviceType *data, uint32_t entry_idx, uint32_t classes_start_index, + float32_t objectness, hailo_quant_info_t quant_info, uint32_t width) + { + std::pair max_id_score_pair; + for (uint32_t class_index = 0; class_index < m_nms_config.classes; class_index++) { + auto class_id = class_index; + if (m_nms_config.background_removal) { + if (m_nms_config.background_removal_index == class_index) { + // Ignore if class_index is background_removal_index + continue; + } + else if (0 == m_nms_config.background_removal_index) { + // background_removal_index will always be the first or last index. + // If it is the first one we need to reduce all classes id's in 1. + // If it is the last one we just ignore it in the previous if case. + class_id--; + } + } + + auto class_entry_idx = entry_idx + ((classes_start_index + class_index) * width); + auto class_confidence = Quantization::dequantize_output(data[class_entry_idx], quant_info); + auto class_score = class_confidence * objectness; + if (class_score > max_id_score_pair.second) { + max_id_score_pair.first = class_id; + max_id_score_pair.second = class_score; + } + } + return max_id_score_pair; + } + + /** + * Removes overlapping boxes in @a detections by setting the class confidence to zero. + * + * @param[in] detections A vector of @a DetectionBbox containing the detections boxes after ::extract_detections() function. + * + */ + void remove_overlapping_boxes(std::vector &detections, std::vector &classes_detections_count); + + /* + * For each class the layout is + * \code + * struct (packed) { + * uint16_t/float32_t bbox_count; + * hailo_bbox_t/hailo_bbox_float32_t bbox[bbox_count]; + * }; + * \endcode + */ + void fill_nms_format_buffer(MemoryView &buffer, const std::vector &detections, + std::vector &classes_detections_count); + + hailo_status hailo_nms_format(std::vector &&detections, + MemoryView dst_view, std::vector &classes_detections_count); + + std::string get_nms_config_description(); + +}; + +} +} + +#endif // _HAILO_NET_FLOW_NMS_POST_PROCESS_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/net_flow/ops/op.hpp b/hailort/libhailort/src/net_flow/ops/op.hpp new file mode 100644 index 0000000..d6a02b3 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/op.hpp @@ -0,0 +1,86 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file op.hpp + * @brief Net-Flow op + * + * https://learnopencv.com/object-detection-using-yolov5-and-opencv-dnn-in-c-and-python : + * The headline '4.3.5 POST-PROCESSING YOLOv5 Prediction Output' contains explanations on the YOLOv5 post-process. + **/ + +#ifndef _HAILO_NET_FLOW_OP_HPP_ +#define _HAILO_NET_FLOW_OP_HPP_ + +#include "hailo/hailort.h" +#include "hailo/buffer.hpp" + +#include "common/utils.hpp" +#include "common/logger_macros.hpp" + + +namespace hailort +{ +namespace net_flow +{ + +struct BufferMetaData +{ + hailo_3d_image_shape_t shape; + hailo_3d_image_shape_t padded_shape; + hailo_format_t format; + hailo_quant_info_t quant_info; +}; + + +class Op +{ +public: + virtual ~Op() = default; + + /** + * Executes operation on inferred data. + * + * @param[in] inputs A map between input names to input buffers. + * @param[in] outputs A map between outputs names and their pre-allocated buffers. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + * + */ + virtual hailo_status execute(const std::map &inputs, std::map &outputs) = 0; + + const std::map &inputs_metadata() const + { + return m_inputs_metadata; + } + + const std::map &outputs_metadata() const + { + return m_outputs_metadata; + } + + std::string get_name() { + return m_name; + } + + virtual std::string get_op_description() = 0; + +protected: + Op(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const std::string &name) + : m_inputs_metadata(inputs_metadata) + , m_outputs_metadata(outputs_metadata) + , m_name(name) + {} + + std::map m_inputs_metadata; + std::map m_outputs_metadata; + const std::string m_name; +}; + +} +} + +#endif // _HAILO_NET_FLOW_OP_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/net_flow/ops/ssd_post_process.cpp b/hailort/libhailort/src/net_flow/ops/ssd_post_process.cpp new file mode 100644 index 0000000..12816c1 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/ssd_post_process.cpp @@ -0,0 +1,202 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file ssd_post_process.cpp + * @brief SSD post process + * + * Reference code: https://github.com/winfredsu/ssd_postprocessing/blob/master/ssd_postprocessing.py + **/ + +#include "net_flow/ops/ssd_post_process.hpp" + +namespace hailort +{ +namespace net_flow +{ + +Expected> SSDPostProcessOp::create(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const SSDPostProcessConfig &ssd_post_process_config) +{ + for (auto &name_to_inputs_metadata : inputs_metadata) { + CHECK_AS_EXPECTED(name_to_inputs_metadata.second.format.order == HAILO_FORMAT_ORDER_NHCW, HAILO_INVALID_ARGUMENT, + "SSDPostProcessOp: Unexpected input format {}", name_to_inputs_metadata.second.format.order); + } + + // Validate each anchor is mapped by reg and cls inputs + for (const auto ®_to_cls_name : ssd_post_process_config.reg_to_cls_inputs) { + CHECK_AS_EXPECTED(ssd_post_process_config.anchors.count(reg_to_cls_name.first), HAILO_INVALID_ARGUMENT, + "SSDPostProcessOp: anchors does not contain reg layer {}", reg_to_cls_name.first); + CHECK_AS_EXPECTED(ssd_post_process_config.anchors.count(reg_to_cls_name.second), HAILO_INVALID_ARGUMENT, + "SSDPostProcessOp: anchors does not contain cls layer {}", reg_to_cls_name.second); + const auto ®_anchors = ssd_post_process_config.anchors.at(reg_to_cls_name.first); + const auto &cls_anchors = ssd_post_process_config.anchors.at(reg_to_cls_name.second); + CHECK_AS_EXPECTED(reg_anchors.size() == cls_anchors.size(), HAILO_INVALID_ARGUMENT, + "SSDPostProcessOp: reg and cls layers have different number of anchors. reg: #{}, cls: #{}", + reg_anchors.size(), cls_anchors.size()); + for (size_t i = 0; i < reg_anchors.size(); ++i) { + auto reg_anchor = reg_anchors[i]; + auto cls_anchor = cls_anchors[i]; + CHECK_AS_EXPECTED(reg_anchor == cls_anchor, HAILO_INVALID_ARGUMENT, + "SSDPostProcessOp: reg and cls layers have differenet anchors. reg: {}, cls: {}", + reg_anchor, cls_anchor); + } + } + + // Validate regs and clss pairs have same shapes + for (const auto ®_to_cls_name : ssd_post_process_config.reg_to_cls_inputs) { + CHECK_AS_EXPECTED(inputs_metadata.count(reg_to_cls_name.first), HAILO_INVALID_ARGUMENT, + "SSDPostProcessOp: inputs_metadata does not contain reg layer {}", reg_to_cls_name.first); + CHECK_AS_EXPECTED(inputs_metadata.count(reg_to_cls_name.second), HAILO_INVALID_ARGUMENT, + "SSDPostProcessOp: inputs_metadata does not contain cls layer {}", reg_to_cls_name.second); + const auto ®_input_metadata = inputs_metadata.at(reg_to_cls_name.first); + const auto &cls_input_metadata = inputs_metadata.at(reg_to_cls_name.second); + // NOTE: padded shape might be different because features might be different, + // and padding is added when width*features % 8 != 0 + CHECK_AS_EXPECTED((reg_input_metadata.shape.height == cls_input_metadata.shape.height) + && (reg_input_metadata.shape.width == cls_input_metadata.shape.width), + HAILO_INVALID_ARGUMENT, "SSDPostProcessOp: reg input {} has different shape than cls input {}", + reg_to_cls_name.first, reg_to_cls_name.second); + } + auto op = std::shared_ptr(new (std::nothrow) SSDPostProcessOp(inputs_metadata, outputs_metadata, nms_post_process_config, ssd_post_process_config)); + CHECK_AS_EXPECTED(op != nullptr, HAILO_OUT_OF_HOST_MEMORY); + return std::shared_ptr(std::move(op)); +} + +hailo_status SSDPostProcessOp::execute(const std::map &inputs, std::map &outputs) +{ + CHECK(inputs.size() == m_ssd_config.anchors.size(), HAILO_INVALID_ARGUMENT, + "Anchors vector count must be equal to data vector count. Anchors size is {}, data size is {}", + m_ssd_config.anchors.size(), inputs.size()); + + std::vector detections; + std::vector classes_detections_count(m_nms_config.classes, 0); + detections.reserve(m_nms_config.max_proposals_per_class * m_nms_config.classes); + for (const auto ®_to_cls : m_ssd_config.reg_to_cls_inputs) { + assert(inputs.count(reg_to_cls.first)); + assert(inputs.count(reg_to_cls.second)); + auto status = extract_detections(reg_to_cls.first, reg_to_cls.second, + inputs.at(reg_to_cls.first), inputs.at(reg_to_cls.second), + detections, classes_detections_count); + CHECK_SUCCESS(status); + } + + // TODO: Add support for TF_FORMAT_ORDER + return hailo_nms_format(std::move(detections), outputs.begin()->second, classes_detections_count); +} + +hailo_status SSDPostProcessOp::extract_detections(const std::string ®_input_name, const std::string &cls_input_name, + const MemoryView ®_buffer, const MemoryView &cls_buffer, + std::vector &detections, std::vector &classes_detections_count) +{ + const auto ®_shape = m_inputs_metadata[reg_input_name].shape; + const auto ®_padded_shape = m_inputs_metadata[reg_input_name].padded_shape; + const auto &cls_padded_shape = m_inputs_metadata[cls_input_name].padded_shape; + + const uint32_t X_INDEX = m_ssd_config.tx_index; + const uint32_t Y_INDEX = m_ssd_config.ty_index; + const uint32_t W_INDEX = m_ssd_config.tw_index; + const uint32_t H_INDEX = m_ssd_config.th_index; + + const uint32_t X_OFFSET = X_INDEX * reg_padded_shape.width; + const uint32_t Y_OFFSET = Y_INDEX * reg_padded_shape.width; + const uint32_t W_OFFSET = W_INDEX * reg_padded_shape.width; + const uint32_t H_OFFSET = H_INDEX * reg_padded_shape.width; + + // Each layer anchors vector is structured as {w,h} pairs. + // For example, if we have a vector of size 6 (default SSD vector) then we have 3 anchors for this layer. + assert(m_ssd_config.anchors.count(reg_input_name)); + assert(m_ssd_config.anchors.count(cls_input_name)); + const auto &layer_anchors = m_ssd_config.anchors[reg_input_name]; + assert(layer_anchors.size() % 2 == 0); + const size_t num_of_anchors = (layer_anchors.size() / 2); + + // Validate reg buffer size + static const uint32_t reg_entry_size = 4; + auto number_of_entries = reg_padded_shape.height * reg_padded_shape.width * num_of_anchors; + auto buffer_size = number_of_entries * reg_entry_size; + CHECK(buffer_size == reg_buffer.size(), HAILO_INVALID_ARGUMENT, + "Failed to extract_detections, reg {} buffer_size should be {}, but is {}", reg_input_name, buffer_size, reg_buffer.size()); + + // Validate cls buffer size + const uint32_t cls_entry_size = m_nms_config.classes; + number_of_entries = cls_padded_shape.height * cls_padded_shape.width * num_of_anchors; + buffer_size = number_of_entries * cls_entry_size; + CHECK(buffer_size == cls_buffer.size(), HAILO_INVALID_ARGUMENT, + "Failed to extract_detections, cls {} buffer_size should be {}, but is {}", cls_input_name, buffer_size, cls_buffer.size()); + + auto reg_row_size = reg_padded_shape.width * reg_padded_shape.features; + auto cls_row_size = cls_padded_shape.width * cls_padded_shape.features; + for (uint32_t row = 0; row < reg_shape.height; row++) { + for (uint32_t col = 0; col < reg_shape.width; col++) { + for (uint32_t anchor = 0; anchor < num_of_anchors; anchor++) { + auto reg_idx = (reg_row_size * row) + col + ((anchor * reg_entry_size) * reg_padded_shape.width); + auto cls_idx = (cls_row_size * row) + col + ((anchor * cls_entry_size) * cls_padded_shape.width); + const auto &wa = layer_anchors[anchor * 2]; + const auto &ha = layer_anchors[anchor * 2 + 1]; + auto anchor_w_stride = 1.0f / static_cast(reg_shape.width); + auto anchor_h_stride = 1.0f / static_cast(reg_shape.height); + auto anchor_w_offset = 0.5f * anchor_w_stride; + auto anchor_h_offset = 0.5f * anchor_h_stride; + auto xcenter_a = static_cast(col) * anchor_w_stride + anchor_w_offset; + auto ycenter_a = static_cast(row) * anchor_h_stride + anchor_h_offset; + // Decode bboxes + if (m_inputs_metadata[reg_input_name].format.type == HAILO_FORMAT_TYPE_UINT8) { + auto status = extract_bbox_detections( + reg_input_name, cls_input_name, + reg_buffer, cls_buffer, + reg_idx + X_OFFSET, + reg_idx + Y_OFFSET, + reg_idx + W_OFFSET, + reg_idx + H_OFFSET, + cls_idx, wa, ha, xcenter_a, ycenter_a, + detections, classes_detections_count); + CHECK_SUCCESS(status); + } else if (m_inputs_metadata[reg_input_name].format.type == HAILO_FORMAT_TYPE_UINT16) { + auto status = extract_bbox_detections( + reg_input_name, cls_input_name, + reg_buffer, cls_buffer, + reg_idx + X_OFFSET, + reg_idx + Y_OFFSET, + reg_idx + W_OFFSET, + reg_idx + H_OFFSET, + cls_idx, wa, ha, xcenter_a, ycenter_a, + detections, classes_detections_count); + CHECK_SUCCESS(status); + } else if (m_inputs_metadata[reg_input_name].format.type == HAILO_FORMAT_TYPE_FLOAT32) { + // For testing - TODO: Remove after generator tests are in, and return error. + auto status = extract_bbox_detections( + reg_input_name, cls_input_name, + reg_buffer, cls_buffer, + reg_idx + X_OFFSET, + reg_idx + Y_OFFSET, + reg_idx + W_OFFSET, + reg_idx + H_OFFSET, + cls_idx, wa, ha, xcenter_a, ycenter_a, + detections, classes_detections_count); + CHECK_SUCCESS(status); + } else { + CHECK_SUCCESS(HAILO_INVALID_ARGUMENT, "SSD post-process received invalid reg input type: {}", + m_inputs_metadata[reg_input_name].format.type); + } + } + } + } + + return HAILO_SUCCESS; +} + +std::string SSDPostProcessOp::get_op_description() +{ + auto nms_config_info = get_nms_config_description(); + auto config_info = fmt::format("Name: {}, {}, Image height: {:.2f}, Image width: {:.2f}, Centers scales factor: {}, " + "Bbox dimension scale factor: {}, Normalize boxes: {}", m_name, nms_config_info, m_ssd_config.image_height, m_ssd_config.image_width, + m_ssd_config.centers_scale_factor, m_ssd_config.bbox_dimensions_scale_factor, m_ssd_config.normalize_boxes); + return config_info; +} + +} +} \ No newline at end of file diff --git a/hailort/libhailort/src/net_flow/ops/ssd_post_process.hpp b/hailort/libhailort/src/net_flow/ops/ssd_post_process.hpp new file mode 100644 index 0000000..5bc9b10 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/ssd_post_process.hpp @@ -0,0 +1,200 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file ssd_post_process.hpp + * @brief SSD post process + * + * Reference code: https://github.com/winfredsu/ssd_postprocessing/blob/master/ssd_postprocessing.py + **/ + +#ifndef _HAILO_SSD_POST_PROCESS_HPP_ +#define _HAILO_SSD_POST_PROCESS_HPP_ + +#include "net_flow/ops/nms_post_process.hpp" + +namespace hailort +{ +namespace net_flow +{ + +struct SSDPostProcessConfig +{ + // The image height. + float32_t image_height = 0; + + // The image width. + float32_t image_width = 0; + + uint32_t centers_scale_factor = 0; + + uint32_t bbox_dimensions_scale_factor = 0; + + uint32_t ty_index = 0; + uint32_t tx_index = 0; + uint32_t th_index = 0; + uint32_t tw_index = 0; + + std::map reg_to_cls_inputs; + + // A vector of anchors, each element in the vector represents the anchors for a specific layer + // Each layer anchors vector is structured as {w,h} pairs. + // Each anchor is mapped by 2 keys: + // 1. reg input + // 2. cls input + std::map> anchors; + + // Indicates whether boxes should be normalized (and clipped) + bool normalize_boxes = false; +}; + +class SSDPostProcessOp : public NmsPostProcessOp +{ + +public: + static Expected> create(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const SSDPostProcessConfig &ssd_post_process_config); + + hailo_status execute(const std::map &inputs, std::map &outputs) override; + std::string get_op_description() override; + + static const uint32_t DEFAULT_Y_OFFSET_IDX = 0; + static const uint32_t DEFAULT_X_OFFSET_IDX = 1; + static const uint32_t DEFAULT_H_OFFSET_IDX = 2; + static const uint32_t DEFAULT_W_OFFSET_IDX = 3; + +private: + SSDPostProcessOp(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const SSDPostProcessConfig &ssd_post_process_config) + : NmsPostProcessOp(inputs_metadata, outputs_metadata, nms_post_process_config, "SSD-Post-Process") + , m_ssd_config(ssd_post_process_config) + {} + + SSDPostProcessConfig m_ssd_config; + + template + void extract_bbox_classes(const hailo_bbox_float32_t &dims_bbox, DeviceType *cls_data, const BufferMetaData &cls_metadata, uint32_t cls_index, + std::vector &detections, std::vector &classes_detections_count) + { + if (m_nms_config.cross_classes) { + // Pre-NMS optimization. If NMS checks IOU over different classes, only the maximum class is relevant + auto max_id_score_pair = get_max_class(cls_data, cls_index, 0, 1, + cls_metadata.quant_info, cls_metadata.padded_shape.width); + auto bbox = dims_bbox; + bbox.score = max_id_score_pair.second; + if (max_id_score_pair.second >= m_nms_config.nms_score_th) { + detections.emplace_back(DetectionBbox(bbox, max_id_score_pair.first)); + classes_detections_count[max_id_score_pair.first]++; + } + } else { + for (uint32_t class_index = 0; class_index < m_nms_config.classes; class_index++) { + auto class_id = class_index; + if (m_nms_config.background_removal) { + if (m_nms_config.background_removal_index == class_index) { + // Ignore if class_index is background_removal_index + continue; + } + else if (0 == m_nms_config.background_removal_index) { + // background_removal_index will always be the first or last index. + // If it is the first one we need to reduce all classes id's in 1. + // If it is the last one we just ignore it in the previous if case. + class_id--; + } + } + + auto class_entry_idx = cls_index + (class_index * cls_metadata.padded_shape.width); + auto class_score = Quantization::dequantize_output(cls_data[class_entry_idx], + cls_metadata.quant_info); + if (class_score < m_nms_config.nms_score_th) { + continue; + } + auto bbox = dims_bbox; + bbox.score = class_score; + detections.emplace_back(bbox, class_id); + classes_detections_count[class_id]++; + } + } + } + + template + hailo_status extract_bbox_detections(const std::string ®_input_name, const std::string &cls_input_name, + const MemoryView ®_buffer, const MemoryView &cls_buffer, + uint64_t x_index, uint64_t y_index, uint64_t w_index, uint64_t h_index, + uint32_t cls_index, float32_t wa, float32_t ha, float32_t xcenter_a, float32_t ycenter_a, + std::vector &detections, std::vector &classes_detections_count) + { + const auto &shape = m_inputs_metadata[reg_input_name].shape; + const auto ®_quant_info = m_inputs_metadata[reg_input_name].quant_info; + DeviceType *reg_data = (DeviceType*)reg_buffer.data(); + auto *cls_data = cls_buffer.data(); + auto tx = Quantization::dequantize_output(reg_data[x_index], reg_quant_info); + auto ty = Quantization::dequantize_output(reg_data[y_index], reg_quant_info); + auto tw = Quantization::dequantize_output(reg_data[w_index], reg_quant_info); + auto th = Quantization::dequantize_output(reg_data[h_index], reg_quant_info); + tx /= static_cast(m_ssd_config.centers_scale_factor); + ty /= static_cast(m_ssd_config.centers_scale_factor); + tw /= static_cast(m_ssd_config.bbox_dimensions_scale_factor); + th /= static_cast(m_ssd_config.bbox_dimensions_scale_factor); + auto w = exp(tw) * wa; + auto h = exp(th) * ha; + auto x_center = tx * wa + xcenter_a; + auto y_center = ty * ha + ycenter_a; + auto x_min = (x_center - (w / 2.0f)); + auto y_min = (y_center - (h / 2.0f)); + auto x_max = (x_center + (w / 2.0f)); + auto y_max = (y_center + (h / 2.0f)); + + // TODO: HRT-10033 - Fix support for clip_boxes and normalize_output + // Currently `normalize_boxes` is always false + if (m_ssd_config.normalize_boxes) { + x_min = Quantization::clip(x_min, 0, static_cast(shape.width-1)); + y_min = Quantization::clip(y_min, 0, static_cast(shape.height-1)); + x_max = Quantization::clip(x_max, 0, static_cast(shape.width-1)); + y_max = Quantization::clip(y_max, 0, static_cast(shape.height-1)); + } + hailo_bbox_float32_t dims_bbox{y_min, x_min, y_max, x_max, 0}; + const auto &cls_metadata = m_inputs_metadata[cls_input_name]; + if (cls_metadata.format.type == HAILO_FORMAT_TYPE_UINT8) { + extract_bbox_classes(dims_bbox, (uint8_t*)cls_data, m_inputs_metadata[cls_input_name], + cls_index, detections, classes_detections_count); + } else if (cls_metadata.format.type == HAILO_FORMAT_TYPE_UINT16) { + extract_bbox_classes(dims_bbox, (uint16_t*)cls_data, m_inputs_metadata[cls_input_name], + cls_index, detections, classes_detections_count); + } else if (cls_metadata.format.type == HAILO_FORMAT_TYPE_FLOAT32) { + extract_bbox_classes(dims_bbox, (float32_t*)cls_data, m_inputs_metadata[cls_input_name], + cls_index, detections, classes_detections_count); + } else { + CHECK_SUCCESS(HAILO_INVALID_ARGUMENT, "SSD post-process received invalid cls input type: {}", + m_inputs_metadata[cls_input_name].format.type); + } + return HAILO_SUCCESS; + } + + /** + * Extract bboxes with confidence level higher then @a confidence_threshold from @a buffer and add them to @a detections. + * + * @param[in] reg_input_name Name of the regression input + * @param[in] cls_input_name Name of the classes input + * @param[in] reg_buffer Buffer containing the boxes data after inference + * @param[in] cls_buffer Buffer containing the classes ids after inference. + * @param[inout] detections A vector of ::DetectionBbox objects, to add the detected bboxes to. + * @param[inout] classes_detections_count A vector of uint32_t, to add count of detections count per class to. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + */ + hailo_status extract_detections(const std::string ®_input_name, const std::string &cls_input_name, + const MemoryView ®_buffer, const MemoryView &cls_buffer, + std::vector &detections, std::vector &classes_detections_count); +}; + + +} + +} + +#endif // _HAILO_SSD_POST_PROCESSING_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/net_flow/ops/yolo_post_process.cpp b/hailort/libhailort/src/net_flow/ops/yolo_post_process.cpp new file mode 100644 index 0000000..2f6a118 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/yolo_post_process.cpp @@ -0,0 +1,117 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file yolo_post_process.cpp + * @brief YOLO post process + * + * https://learnopencv.com/object-detection-using-yolov5-and-opencv-dnn-in-c-and-python : + * The headline '4.3.5 POST-PROCESSING YOLOv5 Prediction Output' contains explanations on the YOLOv5 post-processing. + **/ + +#include "net_flow/ops/yolo_post_process.hpp" + +namespace hailort +{ +namespace net_flow +{ + +Expected> YOLOv5PostProcessOp::create(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const YoloPostProcessConfig &yolo_post_process_config) +{ + for (auto &name_to_inputs_metadata : inputs_metadata) { + CHECK_AS_EXPECTED(name_to_inputs_metadata.second.format.order == HAILO_FORMAT_ORDER_NHCW, HAILO_INVALID_ARGUMENT, + "YOLOv5PostProcessOp: Unexpected input format {}", name_to_inputs_metadata.second.format.order); + } + auto op = std::shared_ptr(new (std::nothrow) YOLOv5PostProcessOp(inputs_metadata, outputs_metadata, nms_post_process_config, yolo_post_process_config)); + CHECK_AS_EXPECTED(op != nullptr, HAILO_OUT_OF_HOST_MEMORY); + return std::shared_ptr(std::move(op)); +} + +Expected> YOLOXPostProcessOp::create(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const YoloPostProcessConfig &yolo_post_process_config) +{ + for (auto &name_to_inputs_metadata : inputs_metadata) { + CHECK_AS_EXPECTED(name_to_inputs_metadata.second.format.order == HAILO_FORMAT_ORDER_NHCW, HAILO_INVALID_ARGUMENT, + "YOLOv5PostProcessOp: Unexpected input format {}", name_to_inputs_metadata.second.format.order); + } + auto modified_yolo_post_process_config = yolo_post_process_config; + for (auto &name_to_meta : inputs_metadata) { + std::vector anchors = {1, 1}; + modified_yolo_post_process_config.anchors.insert({name_to_meta.first, anchors}); + } + auto op = std::shared_ptr(new (std::nothrow) YOLOXPostProcessOp(inputs_metadata, outputs_metadata, nms_post_process_config, + modified_yolo_post_process_config)); + CHECK_AS_EXPECTED(op != nullptr, HAILO_OUT_OF_HOST_MEMORY); + return std::shared_ptr(std::move(op)); +} + +hailo_status YOLOPostProcessOp::execute(const std::map &inputs, std::map &outputs) +{ + CHECK(inputs.size() == m_yolo_config.anchors.size(), HAILO_INVALID_ARGUMENT, + "Anchors vector count must be equal to data vector count. Anchors size is {}, data size is {}", + m_yolo_config.anchors.size(), inputs.size()); + + std::vector detections; + std::vector classes_detections_count(m_nms_config.classes, 0); + detections.reserve(m_nms_config.max_proposals_per_class * m_nms_config.classes); + for (const auto &name_to_input : inputs) { + hailo_status status; + auto &name = name_to_input.first; + auto &input_metadata = m_inputs_metadata[name]; + if (input_metadata.format.type == HAILO_FORMAT_TYPE_UINT8) { + status = extract_detections(name_to_input.second, input_metadata.quant_info, input_metadata.shape, + input_metadata.padded_shape, m_yolo_config.anchors[name], detections, classes_detections_count); + } else if (input_metadata.format.type == HAILO_FORMAT_TYPE_UINT16) { + status = extract_detections(name_to_input.second, input_metadata.quant_info, input_metadata.shape, + input_metadata.padded_shape, m_yolo_config.anchors[name], detections, classes_detections_count); + } else { + CHECK_SUCCESS(HAILO_INVALID_ARGUMENT, "YOLOv5 post-process received invalid input type"); + } + CHECK_SUCCESS(status); + } + + // TODO: Add support for TF_FORMAT_ORDER + return hailo_nms_format(std::move(detections), outputs.begin()->second, classes_detections_count); +} + +std::string YOLOPostProcessOp::get_op_description() +{ + auto nms_config_info = get_nms_config_description(); + auto config_info = fmt::format("Name: {}, {}, Image height: {:.2f}, Image width: {:.2f}", + m_name, nms_config_info, m_yolo_config.image_height, m_yolo_config.image_width); + return config_info; +} + +hailo_bbox_float32_t YOLOv5PostProcessOp::decode(float32_t tx, float32_t ty, float32_t tw, float32_t th, + int wa, int ha, uint32_t col, uint32_t row, uint32_t w_stride, uint32_t h_stride) const +{ + auto w = pow(2.0f * tw, 2.0f) * static_cast(wa) / m_yolo_config.image_width; + auto h = pow(2.0f * th, 2.0f) * static_cast(ha) / m_yolo_config.image_height; + auto x_center = (tx * 2.0f - 0.5f + static_cast(col)) / static_cast(w_stride); + auto y_center = (ty * 2.0f - 0.5f + static_cast(row)) / static_cast(h_stride); + auto x_min = (x_center - (w / 2.0f)); + auto y_min = (y_center - (h / 2.0f)); + return hailo_bbox_float32_t{y_min, x_min, (y_min+h), (x_min+w), 0}; +} + +hailo_bbox_float32_t YOLOXPostProcessOp::decode(float32_t tx, float32_t ty, float32_t tw, float32_t th, + int wa, int ha, uint32_t col, uint32_t row, uint32_t w_stride, uint32_t h_stride) const +{ + auto w = exp(tw) * static_cast(wa) / m_yolo_config.image_width; + auto h = exp(th) * static_cast(ha) / m_yolo_config.image_height; + auto x_center = (tx + static_cast(col)) / static_cast(w_stride); + auto y_center = (ty + static_cast(row)) / static_cast(h_stride); + auto x_min = (x_center - (w / 2.0f)); + auto y_min = (y_center - (h / 2.0f)); + return hailo_bbox_float32_t{y_min, x_min, (y_min+h), (x_min+w), 0}; +} + +} // namespace net_flow +} // namespace hailort + diff --git a/hailort/libhailort/src/net_flow/ops/yolo_post_process.hpp b/hailort/libhailort/src/net_flow/ops/yolo_post_process.hpp new file mode 100644 index 0000000..0c61ca1 --- /dev/null +++ b/hailort/libhailort/src/net_flow/ops/yolo_post_process.hpp @@ -0,0 +1,198 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file yolo_post_process.hpp + * @brief YOLO post process + * + * https://learnopencv.com/object-detection-using-yolov5-and-opencv-dnn-in-c-and-python : + * The headline '4.3.5 POST-PROCESSING YOLOv5 Prediction Output' contains explanations on the YOLOv5 post-processing. + **/ + +#ifndef _HAILO_YOLO_POST_PROCESS_HPP_ +#define _HAILO_YOLO_POST_PROCESS_HPP_ + +#include "net_flow/ops/nms_post_process.hpp" + +namespace hailort +{ +namespace net_flow +{ + +struct YoloPostProcessConfig +{ + // The image height. + float32_t image_height = 0; + + // The image width. + float32_t image_width = 0; + + // A vector of anchors, each element in the vector represents the anchors for a specific layer + // Each layer anchors vector is structured as {w,h} pairs. + std::map> anchors; +}; + + +class YOLOPostProcessOp : public NmsPostProcessOp +{ +public: + hailo_status execute(const std::map &inputs, std::map &outputs) override; + std::string get_op_description() override; + +protected: + virtual hailo_bbox_float32_t decode(float32_t tx, float32_t ty, float32_t tw, float32_t th, + int wa, int ha, uint32_t col, uint32_t row, uint32_t w_stride, uint32_t h_stride) const = 0; + + YOLOPostProcessOp(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const YoloPostProcessConfig &yolo_post_process_config, + const std::string &name) + : NmsPostProcessOp(inputs_metadata, outputs_metadata, nms_post_process_config, name) + , m_yolo_config(yolo_post_process_config) + {} + + YoloPostProcessConfig m_yolo_config; + +private: + /** + * Extract bboxes with confidence level higher then @a confidence_threshold from @a buffer and add them to @a detections. + * + * @param[in] buffer Buffer containing data after inference + * @param[in] quant_info Quantization info corresponding to the @a buffer layer. + * @param[in] shape Shape corresponding to the @a buffer layer. + * @param[in] layer_anchors The layer anchors corresponding to layer receiving the @a buffer. + * Each anchor is structured as {width, height} pairs. + * @param[inout] detections A vector of ::DetectionBbox objects, to add the detected bboxes to. + * @param[inout] classes_detections_count A vector of uint32_t, to add count of detections count per class to. + * + * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. + */ + template + hailo_status extract_detections(const MemoryView &buffer, hailo_quant_info_t quant_info, + hailo_3d_image_shape_t shape, hailo_3d_image_shape_t padded_shape, + const std::vector &layer_anchors, std::vector &detections, std::vector &classes_detections_count) + { + static const uint32_t X_INDEX = 0; + static const uint32_t Y_INDEX = 1; + static const uint32_t W_INDEX = 2; + static const uint32_t H_INDEX = 3; + static const uint32_t OBJECTNESS_INDEX = 4; + static const uint32_t CLASSES_START_INDEX = 5; + + const uint32_t X_OFFSET = X_INDEX * padded_shape.width; + const uint32_t Y_OFFSET = Y_INDEX * padded_shape.width; + const uint32_t W_OFFSET = W_INDEX * padded_shape.width; + const uint32_t H_OFFSET = H_INDEX * padded_shape.width; + const uint32_t OBJECTNESS_OFFSET = OBJECTNESS_INDEX * padded_shape.width; + + // Each layer anchors vector is structured as {w,h} pairs. + // For example, if we have a vector of size 6 (default YOLOv5 vector) then we have 3 anchors for this layer. + assert(layer_anchors.size() % 2 == 0); + const size_t num_of_anchors = (layer_anchors.size() / 2); + + uint32_t entry_size = (uint32_t)((CLASSES_START_INDEX + m_nms_config.classes) * sizeof(DeviceType)); + auto number_of_entries = padded_shape.height * padded_shape.width * num_of_anchors; + // TODO: this can also be part of the Op configuration + auto buffer_size = number_of_entries * entry_size; + CHECK(buffer_size == buffer.size(), HAILO_INVALID_ARGUMENT, + "Failed to extract_detections, buffer_size should be {}, but is {}", buffer_size, buffer.size()); + + auto row_size = padded_shape.width * padded_shape.features; + DeviceType *data = (DeviceType*)buffer.data(); + for (uint32_t row = 0; row < shape.height; row++) { + for (uint32_t col = 0; col < shape.width; col++) { + for (uint32_t anchor = 0; anchor < num_of_anchors; anchor++) { + auto entry_idx = (row_size * row) + col + ((anchor * entry_size) * padded_shape.width); + + auto objectness = Quantization::dequantize_output(data[entry_idx + OBJECTNESS_OFFSET], quant_info); + if (objectness < m_nms_config.nms_score_th) { + continue; + } + + auto tx = Quantization::dequantize_output(data[entry_idx + X_OFFSET], quant_info); + auto ty = Quantization::dequantize_output(data[entry_idx + Y_OFFSET], quant_info); + auto tw = Quantization::dequantize_output(data[entry_idx + W_OFFSET], quant_info); + auto th = Quantization::dequantize_output(data[entry_idx + H_OFFSET], quant_info); + auto bbox = decode(tx, ty, tw, th, layer_anchors[anchor * 2], layer_anchors[anchor * 2 + 1], col, row, + shape.width, shape.height); + + // Source for the calculations - https://github.com/ultralytics/yolov5/blob/HEAD/models/yolo.py + // Explanations for the calculations - https://github.com/ultralytics/yolov5/issues/471 + if (m_nms_config.cross_classes) { + // Pre-NMS optimization. If NMS checks IOU over different classes, only the maximum class is relevant + auto max_id_score_pair = get_max_class(data, entry_idx, CLASSES_START_INDEX, objectness, quant_info, padded_shape.width); + bbox.score = max_id_score_pair.second; + if (max_id_score_pair.second >= m_nms_config.nms_score_th) { + detections.emplace_back(DetectionBbox(bbox, max_id_score_pair.first)); + classes_detections_count[max_id_score_pair.first]++; + } + } + else { + for (uint32_t class_index = 0; class_index < m_nms_config.classes; class_index++) { + auto class_entry_idx = entry_idx + ((CLASSES_START_INDEX + class_index) * padded_shape.width); + auto class_confidence = Quantization::dequantize_output( + data[class_entry_idx], quant_info); + auto class_score = class_confidence * objectness; + if (class_score >= m_nms_config.nms_score_th) { + bbox.score = class_score; + detections.emplace_back(DetectionBbox(bbox, class_index)); + classes_detections_count[class_index]++; + } + } + } + } + } + } + + return HAILO_SUCCESS; + } +}; + +class YOLOv5PostProcessOp : public YOLOPostProcessOp +{ +public: + static Expected> create(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const YoloPostProcessConfig &yolo_post_process_config); + +protected: + virtual hailo_bbox_float32_t decode(float32_t tx, float32_t ty, float32_t tw, float32_t th, + int wa, int ha, uint32_t col, uint32_t row, uint32_t w_stride, uint32_t h_stride) const override; + +private: + YOLOv5PostProcessOp(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const YoloPostProcessConfig &yolo_post_process_config) + : YOLOPostProcessOp(inputs_metadata, outputs_metadata, nms_post_process_config, yolo_post_process_config, "YOLOv5-Post-Process") + {} +}; + +class YOLOXPostProcessOp : public YOLOPostProcessOp +{ +public: + static Expected> create(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const YoloPostProcessConfig &yolo_post_process_config); + +protected: + virtual hailo_bbox_float32_t decode(float32_t tx, float32_t ty, float32_t tw, float32_t th, + int wa, int ha, uint32_t col, uint32_t row, uint32_t w_stride, uint32_t h_stride) const override; + +private: + YOLOXPostProcessOp(const std::map &inputs_metadata, + const std::map &outputs_metadata, + const NmsPostProcessConfig &nms_post_process_config, + const YoloPostProcessConfig &yolo_post_process_config) + : YOLOPostProcessOp(inputs_metadata, outputs_metadata, nms_post_process_config, yolo_post_process_config, "YOLOX-Post-Process") + {} +}; + +} // namespace net_flow +} // namespace hailort + +#endif // _HAILO_YOLO_POST_PROCESS_HPP_ diff --git a/hailort/libhailort/src/net_flow/ops/yolo_post_processing.hpp b/hailort/libhailort/src/net_flow/ops/yolo_post_processing.hpp deleted file mode 100644 index 9db6e53..0000000 --- a/hailort/libhailort/src/net_flow/ops/yolo_post_processing.hpp +++ /dev/null @@ -1,386 +0,0 @@ -/** - * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) -**/ -/** - * @file yolo_post_processing.hpp - * @brief YOLO post processing - * - * https://learnopencv.com/object-detection-using-yolov5-and-opencv-dnn-in-c-and-python : - * The headline '4.3.5 POST-PROCESSING YOLOv5 Prediction Output' contains explanations on the YOLOv5 post-processing. - **/ - -#ifndef _HAILO_YOLO_POST_PROCESSING_HPP_ -#define _HAILO_YOLO_POST_PROCESSING_HPP_ - -#include "hailo/hailort.hpp" - -namespace hailort -{ -namespace net_flow -{ - -static const float32_t REMOVED_CLASS_SCORE = 0.0f; - -struct DetectionBbox -{ - DetectionBbox(float32_t x_min, float32_t y_min, float32_t width, float32_t height, float32_t score, uint32_t class_id) : - m_class_id(class_id), m_bbox{y_min, x_min, (y_min + height), (x_min + width), score} {} - - uint32_t m_class_id; - hailo_bbox_float32_t m_bbox; -}; - -/** - * Computes the value of the sigmoid function on @a x input: f(x) = 1/(1 + e^-x) -*/ -inline float32_t sigmoid(float32_t x) -{ - return 1.0f / (1.0f + expf(-x)); -} - -// TODO: Maybe change to dequantize entry and add @a should_sigmoid to dequantize_output_buffer in `quantization.hpp`. -// Its an API addition but does not break anything. -template -HostType apply_dequantization_activation(DeviceType value, hailo_quant_info_t quant_info, bool should_sigmoid) -{ - auto dequantized_val = Quantization::dequantize_output(value, quant_info); - - if (should_sigmoid) { - return sigmoid(dequantized_val); - } else { - return dequantized_val; - } -} - -class YOLOv5PostProcessingOp -{ -public: - - /** - * @param[in] anchors A vector of anchors, each element in the vector represents the anchors for a specific layer. - * @param[in] image_height The image height. - * @param[in] image_width The image width. - * @param[in] confidence_threshold User given confidence threshold for a bbox. A bbox will be consider as detection if the - * (objectness * class_score) is higher then the confidence_threshold. - * @param[in] iou_threshold User given IOU threshold (intersection over union). This threshold is for performing - * Non-maximum suppression (Removing overlapping boxes). - * @param[in] num_of_classes The model's number of classes. (This depends on the dataset that the model trained on). - * @param[in] should_dequantize Indicates whether the post-processing function should de-quantize the tensors data. - * @param[in] max_bboxes_per_class Maximum amount of bboxes per nms class. - * @param[in] should_sigmoid Indicates whether sigmoid() function should be performed on the @a tensors' data. - * @param[in] one_class_per_bbox Indicates whether the post-processing function should return only one class per detected bbox. - * If set to flase - Two different classes can have the same bbox. - * - * @return Upon success, returns a vector of detection objects. Otherwise, returns Unexpected of ::hailo_status error. - * TODO: For integrating with SDK Json - consider changing anchors vector to a vector of w,h pairs. - * HRT-8526 - Add post-processing support for quantized data - */ - static Expected create(const std::vector> &anchors, - const std::vector &shapes, const std::vector &formats, - const std::vector &quant_infos, float32_t image_height, float32_t image_width, - float32_t confidence_threshold, float32_t iou_threshold, uint32_t num_of_classes, bool should_dequantize, - uint32_t max_bboxes_per_class, bool should_sigmoid, bool one_class_per_bbox=true) - { - return YOLOv5PostProcessingOp(anchors, shapes, formats, quant_infos, image_height, image_width, confidence_threshold, iou_threshold, - num_of_classes, should_dequantize, max_bboxes_per_class, should_sigmoid, one_class_per_bbox); - } - - /** - * Execute YOLOv5 post-processing on inferred data. - * @a HostType can be uint16 or float32. - * TODO: HRT-8525 - Add support for these types. Currently we support only in: @a HostType = float32_t - * - * @param[in] tensors A vector of the input buffers for the post-processing, - * the buffer's shape and the quantization info. - * NOTE: The Order of the @a tensors vector should be corresponding to the order of @a anchors vector given in the creation of YOLOv5PostProcessingOp. - * - * @return Upon success, returns a buffer containing the detection objects, in ::HAILO_FORMAT_ORDER_HAILO_NMS format. - * Otherwise, returns Unexpected of ::hailo_status error. - */ - template - hailo_status execute(const std::vector &tensors, MemoryView dst_view) - { - CHECK(tensors.size() == m_anchors.size(), HAILO_INVALID_ARGUMENT, - "Anchors vector count must be equal to data vector count. Anchors size is {}, data size is {}", m_anchors.size(), tensors.size()); - - std::vector detections; - std::vector classes_detections_count(m_num_of_classes, 0); - detections.reserve(m_max_bboxes_per_class * m_num_of_classes); - for (size_t i = 0; i < tensors.size(); i++) { - hailo_status status; - if (m_formants[i].type == HAILO_FORMAT_TYPE_UINT8) { - status = extract_detections(tensors[i], m_quant_infos[i], m_shapes[i], - m_anchors[i], detections, classes_detections_count); - } else if (m_formants[i].type == HAILO_FORMAT_TYPE_UINT16) { - status = extract_detections(tensors[i], m_quant_infos[i], m_shapes[i], - m_anchors[i], detections, classes_detections_count); - } else { - CHECK_SUCCESS(HAILO_INVALID_ARGUMENT, "YOLOv5 post-process received invalid input type"); - } - CHECK_SUCCESS(status); - } - - // TODO: Add support for TF_FORMAT_ORDER - return hailo_nms_format(std::move(detections), dst_view, classes_detections_count); - } - -private: - YOLOv5PostProcessingOp(const std::vector> &anchors, const std::vector &shapes, - const std::vector &formats, const std::vector &quant_infos, float32_t image_height, float32_t image_width, - float32_t confidence_threshold, float32_t iou_threshold, uint32_t num_of_classes, bool should_dequantize, uint32_t max_bboxes_per_class, bool should_sigmoid, bool one_class_per_bbox) : - m_anchors(anchors), m_shapes(shapes), m_formants(formats), m_quant_infos(quant_infos), m_image_height(image_height), m_image_width(image_width), - m_confidence_threshold(confidence_threshold), m_iou_threshold(iou_threshold), m_num_of_classes(num_of_classes), - m_should_dequantize(should_dequantize), m_max_bboxes_per_class(max_bboxes_per_class), m_should_sigmoid(should_sigmoid), - m_one_class_per_bbox(one_class_per_bbox) - { - (void)m_should_dequantize; - } - - template - std::pair get_max_class(const uint8_t *data, size_t entry_classes_idx, float32_t objectness, hailo_quant_info_t quant_info) - { - std::pair max_id_score_pair; - for (uint32_t class_index = 0; class_index < m_num_of_classes; class_index++) { - auto class_confidence = apply_dequantization_activation(data[entry_classes_idx + class_index], quant_info, m_should_sigmoid); - auto class_score = class_confidence * objectness; - if (class_score > max_id_score_pair.second) { - max_id_score_pair.first = class_index; - max_id_score_pair.second = class_score; - } - } - return max_id_score_pair; - } - - /** - * Extract bboxes with confidence level higher then @a confidence_threshold from @a buffer and add them to @a detections. - * - * @param[in] buffer Buffer containing data after inference and - * @param[in] quant_info Quantization info corresponding to the @a buffer layer. - * @param[in] shape Shape corresponding to the @a buffer layer. - * @param[in] image_height The image height. - * @param[in] image_width The image width. - * @param[in] layer_anchors The layer anchors corresponding to layer receiving the @a buffer. - * Each anchor is structured as {width, height} pairs. - * @param[in] confidence_threshold User given confidence threshold for a bbox. A bbox will be consider as detection if the - * (objectness * class_score) is higher then the confidence_threshold. - * @param[in] num_of_classes The model's number of classes. - * @param[in] should_sigmoid Indicates whether sigmoid() function should be performed on the @a buffer's data. - * @param[inout] detections A vector of ::DetectionBbox objects, to add the detected bboxes to. - * - * @return Upon success, returns ::HAILO_SUCCESS. Otherwise, returns a ::hailo_status error. - */ - template - hailo_status extract_detections(const MemoryView &buffer, hailo_quant_info_t quant_info, hailo_3d_image_shape_t shape, - const std::vector &layer_anchors, std::vector &detections, std::vector &classes_detections_count) - { - static const uint32_t X_INDEX = 0; - static const uint32_t Y_INDEX = 1; - static const uint32_t W_INDEX = 2; - static const uint32_t H_INDEX = 3; - static const uint32_t OBJECTNESS_INDEX = 4; - static const uint32_t CLASSES_START_INDEX = 5; - - // Each layer anchors vector is structured as {w,h} pairs. - // For example, if we have a vector of size 6 (default YOLOv5 vector) then we have 3 anchors for this layer. - assert(layer_anchors.size() % 2 == 0); - const size_t num_of_anchors = (layer_anchors.size() / 2); - - const uint32_t entry_size = CLASSES_START_INDEX + m_num_of_classes; - auto number_of_entries = shape.height * shape.width * num_of_anchors; - // TODO: this can also be part of the Op configuration - auto buffer_size = number_of_entries * entry_size; - CHECK(buffer_size == buffer.size(), HAILO_INVALID_ARGUMENT, - "Failed to extract_detections, buffer_size should be {}, but is {}", buffer_size, buffer.size()); - - auto *data = buffer.data(); - for (size_t row = 0; row < shape.height; row++) { - for (size_t col = 0; col < shape.width; col++) { - for (size_t anchor = 0; anchor < num_of_anchors; anchor++) { - auto entry_idx = entry_size * (num_of_anchors * (shape.height * row + col) + anchor); - - auto objectness = apply_dequantization_activation(data[entry_idx + OBJECTNESS_INDEX], quant_info, m_should_sigmoid); - if (objectness < m_confidence_threshold) { - continue; - } - - auto tx = apply_dequantization_activation(data[entry_idx + X_INDEX], quant_info, m_should_sigmoid); - auto ty = apply_dequantization_activation(data[entry_idx + Y_INDEX], quant_info, m_should_sigmoid); - auto tw = apply_dequantization_activation(data[entry_idx + W_INDEX], quant_info, m_should_sigmoid); - auto th = apply_dequantization_activation(data[entry_idx + H_INDEX], quant_info, m_should_sigmoid); - - // Source for the calculations - https://github.com/ultralytics/yolov5/blob/HEAD/models/yolo.py - // Explanations for the calculations - https://github.com/ultralytics/yolov5/issues/471 - auto w = pow(2.0f * tw, 2.0f) * static_cast(layer_anchors[anchor * 2]) / m_image_width; - auto h = pow(2.0f * th, 2.0f) * static_cast(layer_anchors[anchor * 2 + 1]) / m_image_height; - auto x_center = (tx * 2.0f - 0.5f + static_cast(col)) / static_cast(shape.width); - auto y_center = (ty * 2.0f - 0.5f + static_cast(row)) / static_cast(shape.height); - auto x_min = (x_center - (w / 2.0f)); - auto y_min = (y_center - (h / 2.0f)); - - if (m_one_class_per_bbox) { - auto entry_classes_idx = entry_idx + CLASSES_START_INDEX; - auto max_id_score_pair = get_max_class(data, entry_classes_idx , objectness, quant_info); - if (max_id_score_pair.second >= m_confidence_threshold) { - detections.emplace_back(x_min, y_min, w, h, max_id_score_pair.second, max_id_score_pair.first); - classes_detections_count[max_id_score_pair.first]++; - } - } - else { - for (uint32_t class_index = 0; class_index < m_num_of_classes; class_index++) { - auto class_confidence = apply_dequantization_activation( - data[entry_idx + CLASSES_START_INDEX + class_index], quant_info, m_should_sigmoid); - auto class_score = class_confidence * objectness; - if (class_score >= m_confidence_threshold) { - detections.emplace_back(x_min, y_min, w, h, class_score, class_index); - classes_detections_count[class_index]++; - } - } - } - } - } - } - - return HAILO_SUCCESS; - } - - /** - * Computes the IOU ratio of @a box_1 and @a box_2 - */ - float compute_iou(const DetectionBbox &box_1, const DetectionBbox &box_2) - { - const float overlap_area_width = std::min(box_1.m_bbox.x_max, box_2.m_bbox.x_max) - std::max(box_1.m_bbox.x_min, box_2.m_bbox.x_min); - const float overlap_area_height = std::min(box_1.m_bbox.y_max, box_2.m_bbox.y_max) - std::max(box_1.m_bbox.y_min, box_2.m_bbox.y_min); - if (overlap_area_width <= 0.0f || overlap_area_height <= 0.0f) { - return 0.0f; - } - const float intersection = overlap_area_width * overlap_area_height; - const float box_1_area = (box_1.m_bbox.y_max - box_1.m_bbox.y_min) * (box_1.m_bbox.x_max - box_1.m_bbox.x_min); - const float box_2_area = (box_2.m_bbox.y_max - box_2.m_bbox.y_min) * (box_2.m_bbox.x_max - box_2.m_bbox.x_min); - const float union_area = (box_1_area + box_2_area - intersection); - - return (intersection / union_area); - } - - /** - * Removes overlapping boxes in @a detections by setting the class confidence to zero. - * - * @param[in] detections A vector of @a DetectionBbox containing the detections boxes after ::extract_detections() function. - * - */ - void remove_overlapping_boxes(std::vector &detections, std::vector &classes_detections_count) - { - std::sort(detections.begin(), detections.end(), - [](DetectionBbox a, DetectionBbox b) - { return a.m_bbox.score > b.m_bbox.score; }); - - for (size_t i = 0; i < detections.size(); i++) { - if (detections[i].m_bbox.score == REMOVED_CLASS_SCORE) { - // Detection overlapped with a higher score detection - continue; - } - - for (size_t j = i + 1; j < detections.size(); j++) { - if (detections[j].m_bbox.score == REMOVED_CLASS_SCORE) { - // Detection overlapped with a higher score detection - continue; - } - - if ((detections[i].m_class_id == detections[j].m_class_id) && - (compute_iou(detections[i], detections[j]) >= m_iou_threshold)) { - // Remove detections[j] if the iou is higher then the threshold - detections[j].m_bbox.score = REMOVED_CLASS_SCORE; - assert(classes_detections_count[detections[j].m_class_id] > 0); - classes_detections_count[detections[j].m_class_id]--; - } - } - } - } - - /* - * For each class the layout is - * \code - * struct (packed) { - * uint16_t/float32_t bbox_count; - * hailo_bbox_t/hailo_bbox_float32_t bbox[bbox_count]; - * }; - * \endcode - */ - void fill_nms_format_buffer(MemoryView &buffer, const std::vector &detections, - std::vector &classes_detections_count) - { - // Calculate the number of detections before each class, to help us later calculate the buffer_offset for it's detections. - std::vector num_of_detections_before; - num_of_detections_before.reserve(m_num_of_classes); - uint32_t ignored_detections_count = 0; - for (size_t class_idx = 0; class_idx < m_num_of_classes; class_idx++) { - if (classes_detections_count[class_idx] > m_max_bboxes_per_class) { - ignored_detections_count += (classes_detections_count[class_idx] - m_max_bboxes_per_class); - classes_detections_count[class_idx] = m_max_bboxes_per_class; - } - - if (0 == class_idx) { - num_of_detections_before[class_idx] = 0; - } - else { - num_of_detections_before[class_idx] = num_of_detections_before[class_idx - 1] + classes_detections_count[class_idx - 1]; - } - - // Fill `bbox_count` value for class_idx in the result buffer - float32_t bbox_count_casted = static_cast(classes_detections_count[class_idx]); - auto buffer_offset = (class_idx * sizeof(bbox_count_casted)) + (num_of_detections_before[class_idx] * sizeof(hailo_bbox_float32_t)); - memcpy((buffer.data() + buffer_offset), &bbox_count_casted, sizeof(bbox_count_casted)); - } - - for (auto &detection : detections) { - if (REMOVED_CLASS_SCORE == detection.m_bbox.score) { - // Detection overlapped with a higher score detection and removed in remove_overlapping_boxes() - continue; - } - if (0 == classes_detections_count[detection.m_class_id]) { - // This class' detections count is higher then m_max_bboxes_per_class. - // This detection is ignored due to having lower score (detections vector is sorted by score). - continue; - } - - auto buffer_offset = ((detection.m_class_id + 1) * sizeof(float32_t)) - + (num_of_detections_before[detection.m_class_id] * sizeof(hailo_bbox_float32_t)); - - assert((buffer_offset + sizeof(hailo_bbox_float32_t)) <= buffer.size()); - memcpy((hailo_bbox_float32_t*)(buffer.data() + buffer_offset), &detection.m_bbox, sizeof(hailo_bbox_float32_t)); - num_of_detections_before[detection.m_class_id]++; - classes_detections_count[detection.m_class_id]--; - } - - if (0 != ignored_detections_count) { - LOGGER__INFO("{} Detections were ignored, due to `max_bboxes_per_class` defined as {}.", - ignored_detections_count, m_max_bboxes_per_class); - } - } - - hailo_status hailo_nms_format(std::vector &&detections, MemoryView dst_view, std::vector &classes_detections_count) - { - remove_overlapping_boxes(detections, classes_detections_count); - fill_nms_format_buffer(dst_view, detections, classes_detections_count); - return HAILO_SUCCESS; - } - - std::vector> m_anchors; - std::vector m_shapes; - std::vector m_formants; - std::vector m_quant_infos; - float32_t m_image_height; - float32_t m_image_width; - float32_t m_confidence_threshold; - float32_t m_iou_threshold; - uint32_t m_num_of_classes; - bool m_should_dequantize; - uint32_t m_max_bboxes_per_class; - bool m_should_sigmoid; - bool m_one_class_per_bbox; -}; - -} /* namespace net_flow */ -} /* namespace hailort */ - -#endif /* _HAILO_YOLO_POST_PROCESSING_HPP_ */ diff --git a/hailort/libhailort/src/inference_pipeline.cpp b/hailort/libhailort/src/net_flow/pipeline/inference_pipeline.cpp similarity index 95% rename from hailort/libhailort/src/inference_pipeline.cpp rename to hailort/libhailort/src/net_flow/pipeline/inference_pipeline.cpp index 5910104..f619982 100644 --- a/hailort/libhailort/src/inference_pipeline.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/inference_pipeline.cpp @@ -8,22 +8,26 @@ **/ #include "hailo/inference_pipeline.hpp" +#include "hailo/hailort_defaults.hpp" + #include "common/async_thread.hpp" -#include "vstream_internal.hpp" -#include "hailort_defaults.hpp" -#include "context_switch/network_group_internal.hpp" -#include "context_switch/multi_context/resource_manager.hpp" + +#include "net_flow/pipeline/vstream_internal.hpp" +#include "network_group/network_group_internal.hpp" +#include "core_op/resource_manager/resource_manager.hpp" #include + namespace hailort { InferVStreams::InferVStreams(std::vector &&inputs, std::vector &&outputs, bool is_multi_context, - uint16_t batch_size) : + bool is_scheduled, uint16_t batch_size) : m_inputs(std::move(inputs)), m_outputs(std::move(outputs)), m_is_multi_context(is_multi_context), + m_is_scheduled(is_scheduled), m_batch_size(batch_size) { for (auto &input : m_inputs) { @@ -183,7 +187,8 @@ Expected InferVStreams::create(ConfiguredNetworkGroup &net_group, auto output_vstreams = VStreamsBuilder::create_output_vstreams(net_group, output_params); CHECK_EXPECTED(output_vstreams); - return InferVStreams(input_vstreams.release(), output_vstreams.release(), is_multi_context, batch_size); + return InferVStreams(input_vstreams.release(), output_vstreams.release(), is_multi_context, net_group.is_scheduled(), + batch_size); } hailo_status InferVStreams::infer(const std::map& input_data, @@ -284,8 +289,11 @@ hailo_status InferVStreams::verify_memory_view_size(const std::map lock(m_mutex); - m_was_stream_aborted = true; - } - m_cv.notify_all(); return PipelineElement::execute_abort(); } diff --git a/hailort/libhailort/src/pipeline.hpp b/hailort/libhailort/src/net_flow/pipeline/pipeline.hpp similarity index 94% rename from hailort/libhailort/src/pipeline.hpp rename to hailort/libhailort/src/net_flow/pipeline/pipeline.hpp index e66b192..702f8a7 100644 --- a/hailort/libhailort/src/pipeline.hpp +++ b/hailort/libhailort/src/net_flow/pipeline/pipeline.hpp @@ -12,13 +12,15 @@ #include "hailo/buffer.hpp" #include "hailo/runtime_statistics.hpp" -#include "thread_safe_queue.hpp" + +#include "utils/thread_safe_queue.hpp" #include #include #include #include + namespace hailort { @@ -176,55 +178,6 @@ class PipelineElement; using PushCompleteCallback = std::function; using PullCompleteCallback = std::function; -struct NetFlowPad { - std::string name; - hailo_format_t format; - hailo_quant_info_t quant_info; - uint32_t number_of_classes = 0; // temporarly here, should be only if the previous op is NMS -}; - -struct NetFlowElement { - std::vector input_pads; - std::vector output_pads; - - enum class Type - { - None = 0, - CoreOp = 1, - YoloNmsOp = 2 - }; - - Type type; - std::string name; - std::set input_streams; - - virtual ~NetFlowElement() = default; -}; - -struct YoloBboxDecoder { - std::vector h; - std::vector w; - uint32_t stride = 0; - // uint32_t pad_index; - std::string stream_name; -}; - -struct NetFlowNmsElement : NetFlowElement { - float32_t nms_score_th = 0; - float32_t nms_iou_th = 0; - uint32_t max_proposals_per_class = 0; - uint32_t classes = 0; - bool background_removal = false; - uint32_t background_removal_index = 0; -}; - -struct NetFlowYoloNmsElement final : NetFlowNmsElement { - std::vector bbox_decoders; - float32_t image_height = 0; - float32_t image_width = 0; - uint32_t input_division_factor = 0; -}; - class PipelinePad final : public PipelineObject { public: @@ -434,6 +387,7 @@ protected: virtual std::vector get_queue_size_accumulators() override; virtual hailo_status run_in_thread() = 0; + virtual std::string thread_name() = 0; SpscQueue m_queue; EventPtr m_shutdown_event; @@ -468,6 +422,7 @@ public: protected: virtual hailo_status execute_deactivate() override; virtual hailo_status run_in_thread() override; + virtual std::string thread_name() override { return "PUSH_QUEUE"; }; virtual hailo_status execute_abort() override; }; @@ -507,6 +462,7 @@ public: protected: virtual hailo_status execute_deactivate() override; virtual hailo_status run_in_thread() override; + virtual std::string thread_name() override { return "PULL_QUEUE"; }; }; class UserBufferQueueElement : public PullQueueElement diff --git a/hailort/libhailort/src/vstream.cpp b/hailort/libhailort/src/net_flow/pipeline/vstream.cpp similarity index 89% rename from hailort/libhailort/src/vstream.cpp rename to hailort/libhailort/src/net_flow/pipeline/vstream.cpp index 9007e1a..adfc6fa 100644 --- a/hailort/libhailort/src/vstream.cpp +++ b/hailort/libhailort/src/net_flow/pipeline/vstream.cpp @@ -8,16 +8,20 @@ **/ #include "hailo/vstream.hpp" -#include "hailort_defaults.hpp" -#include "vstream_internal.hpp" +#include "hailo/hailort_defaults.hpp" + #include "common/runtime_statistics_internal.hpp" +#include "net_flow/pipeline/vstream_internal.hpp" + #ifdef HAILO_SUPPORT_MULTI_PROCESS #include "rpc/rpc_definitions.hpp" +#include "service/rpc_client_utils.hpp" #endif // HAILO_SUPPORT_MULTI_PROCESS #include + namespace hailort { @@ -206,75 +210,42 @@ static hailo_nms_info_t fuse_nms_info(const std::vector &nms_i return fused_info; } -Expected nms_element_to_op(const NetFlowYoloNmsElement &element, - const std::vector &shapes, const std::vector &formats, - const std::vector &quant_infos) -{ - std::vector> anchors; - // Each layer anchors vector is structured as {w,h} pairs. - auto bbox_decoders = element.bbox_decoders; - std::sort(bbox_decoders.begin(), bbox_decoders.end(), [](auto &bbox_decoder_0, auto &bbox_decoder_1) { - return bbox_decoder_0.stream_name < bbox_decoder_1.stream_name; - }); - for (auto &bbox_decoder : bbox_decoders) { - std::vector layer_anchors; - layer_anchors.reserve(bbox_decoder.h.size() + bbox_decoder.w.size()); - assert(bbox_decoder.h.size() == bbox_decoder.w.size()); - for (size_t i = 0; i < bbox_decoder.h.size(); ++i) { - layer_anchors.push_back(bbox_decoder.w[i]); - layer_anchors.push_back(bbox_decoder.h[i]); - } - anchors.push_back(layer_anchors); - } - - // TODO: Get it from NetFlowYoloNmsElement when adding support for these params. - static const bool should_dequantize = true; - static const bool should_sigmoid = false; - return net_flow::YOLOv5PostProcessingOp::create(anchors, shapes, formats,quant_infos, - element.image_height, element.image_width, element.nms_score_th, - element.nms_iou_th, element.classes, should_dequantize, element.max_proposals_per_class, should_sigmoid); -} - -Expected> NmsPostProcessMuxElement::create(const NetFlowYoloNmsElement &element, - const std::vector &shapes, const std::vector &formats, - const std::vector &quant_infos, hailo_format_t output_format, +Expected> NmsPostProcessMuxElement::create(std::shared_ptr nms_op, hailo_nms_info_t nms_info, const std::string &name, std::chrono::milliseconds timeout, size_t buffer_pool_size, hailo_pipeline_elem_stats_flags_t elem_flags, hailo_vstream_stats_flags_t vstream_flags, EventPtr shutdown_event, std::shared_ptr> pipeline_status) { - auto buffer_pool = BufferPool::create(HailoRTCommon::get_nms_host_frame_size(nms_info, output_format), + assert(nms_op->outputs_metadata().size() == 1); + auto buffer_pool = BufferPool::create(HailoRTCommon::get_nms_host_frame_size(nms_info, nms_op->outputs_metadata().begin()->second.format), buffer_pool_size, shutdown_event, elem_flags, vstream_flags); CHECK_EXPECTED(buffer_pool, "Failed creating BufferPool"); auto duration_collector = DurationCollector::create(elem_flags); CHECK_EXPECTED(duration_collector); - auto expected_nms_op = nms_element_to_op(element, shapes, formats, quant_infos); - auto nms_elem_ptr = make_shared_nothrow(expected_nms_op.release(), buffer_pool.release(), + auto nms_elem_ptr = make_shared_nothrow(nms_op, buffer_pool.release(), name, timeout, duration_collector.release(), std::move(pipeline_status)); CHECK_AS_EXPECTED(nullptr != nms_elem_ptr, HAILO_OUT_OF_HOST_MEMORY); LOGGER__INFO("Created {}", nms_elem_ptr->name()); - return nms_elem_ptr; } -Expected> NmsPostProcessMuxElement::create(const NetFlowYoloNmsElement &op, - const std::vector &shapes, const std::vector &formats, - const std::vector &quant_infos, hailo_nms_info_t nms_info, const std::string &name, - const hailo_vstream_params_t &vstream_params, EventPtr shutdown_event, std::shared_ptr> pipeline_status) +Expected> NmsPostProcessMuxElement::create(std::shared_ptr nms_op, + hailo_nms_info_t nms_info, const std::string &name, const hailo_vstream_params_t &vstream_params, + EventPtr shutdown_event, std::shared_ptr> pipeline_status) { - return NmsPostProcessMuxElement::create(op, shapes, formats, quant_infos, vstream_params.user_buffer_format, nms_info, name, std::chrono::milliseconds(vstream_params.timeout_ms), + return NmsPostProcessMuxElement::create(nms_op, nms_info, name, std::chrono::milliseconds(vstream_params.timeout_ms), vstream_params.queue_size, vstream_params.pipeline_elements_stats_flags, vstream_params.vstream_stats_flags, shutdown_event, pipeline_status); } -NmsPostProcessMuxElement::NmsPostProcessMuxElement(const net_flow::YOLOv5PostProcessingOp &op, BufferPoolPtr &&pool, +NmsPostProcessMuxElement::NmsPostProcessMuxElement(std::shared_ptr nms_op, BufferPoolPtr &&pool, const std::string &name, std::chrono::milliseconds timeout, DurationCollector &&duration_collector, std::shared_ptr> &&pipeline_status) : - BaseMuxElement(3, name, timeout, std::move(duration_collector), std::move(pipeline_status)), - m_nms_op(op), + BaseMuxElement(nms_op->inputs_metadata().size(), name, timeout, std::move(duration_collector), std::move(pipeline_status)), + m_nms_op(nms_op), m_pool(std::move(pool)) {} @@ -286,22 +257,22 @@ std::vector NmsPostProcessMuxElement::get_queue_size_accumulator return {m_pool->get_queue_size_accumulator()}; } -Expected NmsPostProcessMuxElement::action(std::vector &&inputs, PipelineBuffer &&optional) +Expected NmsPostProcessMuxElement::action(std::vector &&input_buffers, PipelineBuffer &&optional) { - std::vector input_views; - - input_views.reserve(inputs.size()); - for (auto &input_buf : inputs) { - input_views.push_back(input_buf.as_view()); + std::map inputs; + std::map outputs; + for (size_t i = 0; i < input_buffers.size(); ++i) { + inputs.insert({m_sinks_names[i], input_buffers[i].as_view()}); } - auto acquired_buffer = m_pool->get_available_buffer(std::move(optional), m_timeout); if (HAILO_SHUTDOWN_EVENT_SIGNALED == acquired_buffer.status()) { return make_unexpected(acquired_buffer.status()); } CHECK_EXPECTED(acquired_buffer); + outputs.insert({"", acquired_buffer.value().as_view()}); // TODO: fill with correct name m_duration_collector.start_measurement(); - auto post_process_result = m_nms_op.execute(input_views, acquired_buffer.value().as_view()); + + auto post_process_result = m_nms_op->execute(inputs, outputs); m_duration_collector.complete_measurement(); CHECK_SUCCESS_AS_EXPECTED(post_process_result); return acquired_buffer; @@ -458,7 +429,7 @@ Expected> TransformDemuxElement::action(PipelineBuff BaseVStream::BaseVStream(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, - EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, EventPtr &&network_group_activated_event, + EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, EventPtr &&core_op_activated_event, hailo_status &output_status) : m_vstream_info(vstream_info), m_vstream_params(vstream_params), @@ -469,7 +440,7 @@ BaseVStream::BaseVStream(const hailo_vstream_info_t &vstream_info, const hailo_v m_is_aborted(false), m_pipeline_status(std::move(pipeline_status)), m_shutdown_event(shutdown_event), - m_network_group_activated_event(std::move(network_group_activated_event)), + m_core_op_activated_event(std::move(core_op_activated_event)), m_fps_accumulators(get_pipeline_accumulators_by_type(m_pipeline, AccumulatorType::FPS)), m_latency_accumulators(get_pipeline_accumulators_by_type(m_pipeline, AccumulatorType::LATENCY)), m_queue_size_accumulators(get_pipeline_queue_size_accumulators(m_pipeline)), @@ -488,7 +459,7 @@ BaseVStream::BaseVStream(BaseVStream &&other) noexcept : m_is_aborted(std::exchange(other.m_is_aborted, false)), m_pipeline_status(std::move(other.m_pipeline_status)), m_shutdown_event(std::move(other.m_shutdown_event)), - m_network_group_activated_event(std::move(other.m_network_group_activated_event)), + m_core_op_activated_event(std::move(other.m_core_op_activated_event)), m_fps_accumulators(std::move(other.m_fps_accumulators)), m_latency_accumulators(std::move(other.m_latency_accumulators)), m_queue_size_accumulators(std::move(other.m_queue_size_accumulators)), @@ -509,7 +480,7 @@ BaseVStream& BaseVStream::operator=(BaseVStream &&other) noexcept m_is_aborted = std::exchange(other.m_is_aborted, false); m_pipeline_status = std::move(other.m_pipeline_status); m_shutdown_event = std::move(other.m_shutdown_event); - m_network_group_activated_event = std::move(other.m_network_group_activated_event); + m_core_op_activated_event = std::move(other.m_core_op_activated_event); m_fps_accumulators = std::move(other.m_fps_accumulators); m_latency_accumulators = std::move(other.m_latency_accumulators); m_queue_size_accumulators = std::move(other.m_queue_size_accumulators); @@ -567,7 +538,7 @@ hailo_status BaseVStream::stop_vstream() hailo_status BaseVStream::stop_and_clear() { - auto status = m_network_group_activated_event->wait(std::chrono::milliseconds(0)); + auto status = m_core_op_activated_event->wait(std::chrono::milliseconds(0)); CHECK(HAILO_TIMEOUT == status, HAILO_INVALID_OPERATION, "Trying to clear {} vstream before its network group is deactivated", name()); @@ -643,11 +614,11 @@ const std::vector> &BaseVStream::get_pipeline() Expected InputVStream::create(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::shared_ptr pipeline_exit, std::vector> &&pipeline, - std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr network_group_activated_event, + std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator) { auto vstream_internal = InputVStreamInternal::create(vstream_info, vstream_params, pipeline_entry, pipeline_exit, - std::move(pipeline), std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator); + std::move(pipeline), std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator); CHECK_EXPECTED(vstream_internal); InputVStream vstream(vstream_internal.release()); @@ -772,16 +743,31 @@ std::string InputVStream::get_pipeline_description() const return m_vstream->get_pipeline_description(); } +hailo_status InputVStream::before_fork() +{ + return m_vstream->before_fork(); +} + +hailo_status InputVStream::after_fork_in_parent() +{ + return m_vstream->after_fork_in_parent(); +} + +hailo_status InputVStream::after_fork_in_child() +{ + return m_vstream->after_fork_in_child(); +} + InputVStream::InputVStream(std::shared_ptr vstream) : m_vstream(std::move(vstream)) {} Expected OutputVStream::create( const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, - EventPtr network_group_activated_event, AccumulatorPtr pipeline_latency_accumulator) + EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator) { auto vstream_internal = OutputVStreamInternal::create(vstream_info, vstream_params, pipeline_entry, - std::move(pipeline), std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator); + std::move(pipeline), std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator); CHECK_EXPECTED(vstream_internal); OutputVStream vstream(vstream_internal.release()); @@ -901,6 +887,21 @@ std::string OutputVStream::get_pipeline_description() const return m_vstream->get_pipeline_description(); } +hailo_status OutputVStream::before_fork() +{ + return m_vstream->before_fork(); +} + +hailo_status OutputVStream::after_fork_in_parent() +{ + return m_vstream->after_fork_in_parent(); +} + +hailo_status OutputVStream::after_fork_in_child() +{ + return m_vstream->after_fork_in_child(); +} + OutputVStream::OutputVStream(std::shared_ptr vstream) : m_vstream(std::move(vstream)) {} std::map get_pipeline_accumulators_by_type( @@ -950,11 +951,11 @@ std::map> get_pipeline_queue_size_accum Expected> InputVStreamInternal::create(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::shared_ptr pipeline_exit, std::vector> &&pipeline, - std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr network_group_activated_event, + std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator) { auto vstream = InputVStreamImpl::create(vstream_info, vstream_params, pipeline_entry, pipeline_exit, - std::move(pipeline), std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator); + std::move(pipeline), std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator); CHECK_EXPECTED(vstream); auto vstream_ptr = std::shared_ptr(vstream.release()); return vstream_ptr; @@ -963,15 +964,15 @@ Expected> InputVStreamInternal::create(con InputVStreamInternal::InputVStreamInternal(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, - EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, EventPtr &&network_group_activated_event, + EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, EventPtr &&core_op_activated_event, hailo_status &output_status) : BaseVStream(vstream_info, vstream_params, pipeline_entry, std::move(pipeline), std::move(pipeline_status), - shutdown_event, pipeline_latency_accumulator, std::move(network_group_activated_event), output_status){} + shutdown_event, pipeline_latency_accumulator, std::move(core_op_activated_event), output_status){} Expected> InputVStreamImpl::create(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::shared_ptr pipeline_exit, std::vector> &&pipeline, - std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr network_group_activated_event, + std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator) { hailo_status status = HAILO_UNINITIALIZED; @@ -985,7 +986,7 @@ Expected> InputVStreamImpl::create(const hailo } auto vstream_ptr = std::shared_ptr(new InputVStreamImpl(vstream_info, vstream_params, std::move(pipeline_entry), std::move(pipeline), - std::move(pipeline_status), shutdown_event, pipeline_latency_accumulator, std::move(network_group_activated_event), status)); + std::move(pipeline_status), shutdown_event, pipeline_latency_accumulator, std::move(core_op_activated_event), status)); CHECK_SUCCESS_AS_EXPECTED(status, "Failed to create virtual stream"); return vstream_ptr; @@ -994,9 +995,9 @@ Expected> InputVStreamImpl::create(const hailo InputVStreamImpl::InputVStreamImpl(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, - EventPtr network_group_activated_event, hailo_status &output_status) : + EventPtr core_op_activated_event, hailo_status &output_status) : InputVStreamInternal(vstream_info, vstream_params, pipeline_entry, std::move(pipeline), std::move(pipeline_status), - shutdown_event, pipeline_latency_accumulator, std::move(network_group_activated_event), output_status) + shutdown_event, pipeline_latency_accumulator, std::move(core_op_activated_event), output_status) { if (HAILO_SUCCESS != output_status) { return; @@ -1016,9 +1017,9 @@ InputVStreamImpl::~InputVStreamImpl() hailo_status InputVStreamImpl::write(const MemoryView &buffer) { - if (nullptr != m_network_group_activated_event) { + if (nullptr != m_core_op_activated_event) { CHECK(m_is_activated, HAILO_VSTREAM_PIPELINE_NOT_ACTIVATED, "Failed to write buffer! Virtual stream {} is not activated!", name()); - auto status = m_network_group_activated_event->wait(std::chrono::milliseconds(0)); + auto status = m_core_op_activated_event->wait(std::chrono::milliseconds(0)); CHECK(HAILO_TIMEOUT != status, HAILO_NETWORK_GROUP_NOT_ACTIVATED, "Trying to write to vstream {} before its network group is activated", name()); } @@ -1047,22 +1048,20 @@ hailo_status InputVStreamImpl::flush() } #ifdef HAILO_SUPPORT_MULTI_PROCESS -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wreturn-type" Expected> InputVStreamClient::create(uint32_t input_vstream_handle) { grpc::ChannelArguments ch_args; ch_args.SetMaxReceiveMessageSize(-1); - auto channel = grpc::CreateCustomChannel(HAILO_DEFAULT_UDS_ADDR, grpc::InsecureChannelCredentials(), ch_args); + auto channel = grpc::CreateCustomChannel(HAILORT_SERVICE_DEFAULT_ADDR, grpc::InsecureChannelCredentials(), ch_args); CHECK_AS_EXPECTED(channel != nullptr, HAILO_INTERNAL_FAILURE); - auto client = std::unique_ptr(new HailoRtRpcClient(channel)); + auto client = make_unique_nothrow(channel); CHECK_AS_EXPECTED(client != nullptr, HAILO_OUT_OF_HOST_MEMORY); - auto user_buffer_format = client->InputVStream_get_user_buffer_format(input_vstream_handle); + auto user_buffer_format = client->InputVStream_get_user_buffer_format(input_vstream_handle); CHECK_EXPECTED(user_buffer_format); - auto vstream_info = client->InputVStream_get_info(input_vstream_handle); + auto vstream_info = client->InputVStream_get_info(input_vstream_handle); CHECK_EXPECTED(vstream_info); return std::shared_ptr(new InputVStreamClient(std::move(client), std::move(input_vstream_handle), @@ -1093,10 +1092,9 @@ hailo_status InputVStreamClient::flush() hailo_status InputVStreamClient::abort() { - auto channel = grpc::CreateChannel(HAILO_DEFAULT_UDS_ADDR, grpc::InsecureChannelCredentials()); - CHECK(channel != nullptr, HAILO_INTERNAL_FAILURE); - auto abort_client = std::unique_ptr(new HailoRtRpcClient(channel)); - CHECK(abort_client != nullptr, HAILO_OUT_OF_HOST_MEMORY); + auto expected_client = HailoRtRpcClientUtils::create_client(); + CHECK_EXPECTED_AS_STATUS(expected_client); + auto abort_client = expected_client.release(); return abort_client->InputVStream_abort(m_handle); } @@ -1172,7 +1170,34 @@ const std::vector> &InputVStreamClient::get_pip return m_pipeline; } -#pragma GCC diagnostic pop +hailo_status InputVStreamClient::create_client() +{ + auto expected_client = HailoRtRpcClientUtils::create_client(); + CHECK_EXPECTED_AS_STATUS(expected_client); + m_client = expected_client.release(); + return HAILO_SUCCESS; +} + +hailo_status InputVStreamClient::before_fork() +{ + m_client.reset(); + return HAILO_SUCCESS; +} + +hailo_status InputVStreamClient::after_fork_in_parent() +{ + return create_client(); +} + +hailo_status InputVStreamClient::after_fork_in_child() +{ + auto status = create_client(); + CHECK_SUCCESS(status); + auto expected_dup_handle = m_client->InputVStream_dup_handle(OsUtils::get_curr_pid(), m_handle); + CHECK_EXPECTED_AS_STATUS(expected_dup_handle); + m_handle = expected_dup_handle.value(); + return HAILO_SUCCESS; +} #endif // HAILO_SUPPORT_MULTI_PROCESS std::string InputVStreamInternal::get_pipeline_description() const @@ -1190,10 +1215,10 @@ Expected> OutputVStreamInternal::create( const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, - EventPtr network_group_activated_event, AccumulatorPtr pipeline_latency_accumulator) + EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator) { auto vstream = OutputVStreamImpl::create(vstream_info, vstream_params, pipeline_entry, - std::move(pipeline), std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator); + std::move(pipeline), std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator); CHECK_EXPECTED(vstream); auto vstream_ptr = std::shared_ptr(vstream.release()); return vstream_ptr; @@ -1204,14 +1229,14 @@ OutputVStreamInternal::OutputVStreamInternal(const hailo_vstream_info_t &vstream std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, - EventPtr network_group_activated_event, hailo_status &output_status) : + EventPtr core_op_activated_event, hailo_status &output_status) : BaseVStream(vstream_info, vstream_params, pipeline_entry, std::move(pipeline), std::move(pipeline_status), - shutdown_event, pipeline_latency_accumulator, std::move(network_group_activated_event), output_status){} + shutdown_event, pipeline_latency_accumulator, std::move(core_op_activated_event), output_status){} Expected> OutputVStreamImpl::create(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, - EventPtr network_group_activated_event, AccumulatorPtr pipeline_latency_accumulator) + EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator) { hailo_status status = HAILO_UNINITIALIZED; @@ -1227,7 +1252,7 @@ Expected> OutputVStreamImpl::create(const hai } auto vstream_ptr = std::shared_ptr(new OutputVStreamImpl(vstream_info, vstream_params, std::move(pipeline_entry), std::move(pipeline), - std::move(pipeline_status), shutdown_event, pipeline_latency_accumulator, std::move(network_group_activated_event), status)); + std::move(pipeline_status), shutdown_event, pipeline_latency_accumulator, std::move(core_op_activated_event), status)); CHECK_SUCCESS_AS_EXPECTED(status, "Failed to create virtual stream"); return vstream_ptr; @@ -1248,9 +1273,9 @@ OutputVStreamImpl::OutputVStreamImpl(const hailo_vstream_info_t &vstream_info, c std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, - EventPtr network_group_activated_event, hailo_status &output_status) : + EventPtr core_op_activated_event, hailo_status &output_status) : OutputVStreamInternal(vstream_info, vstream_params, pipeline_entry, std::move(pipeline), std::move(pipeline_status), - shutdown_event, pipeline_latency_accumulator, std::move(network_group_activated_event), output_status) + shutdown_event, pipeline_latency_accumulator, std::move(core_op_activated_event), output_status) { if (HAILO_SUCCESS != output_status) { return; @@ -1284,9 +1309,9 @@ OutputVStreamImpl::~OutputVStreamImpl() hailo_status OutputVStreamImpl::read(MemoryView buffer) { - if (nullptr != m_network_group_activated_event) { + if (nullptr != m_core_op_activated_event) { CHECK(m_is_activated, HAILO_VSTREAM_PIPELINE_NOT_ACTIVATED, "read() failed! Virtual stream {} is not activated!", name()); - auto status = m_network_group_activated_event->wait(std::chrono::milliseconds(0)); + auto status = m_core_op_activated_event->wait(std::chrono::milliseconds(0)); if (HAILO_TIMEOUT == status) { LOGGER__INFO("Trying to read from vstream {} before its network_group is activated", name()); return HAILO_NETWORK_GROUP_NOT_ACTIVATED; @@ -1310,22 +1335,20 @@ hailo_status OutputVStreamImpl::read(MemoryView buffer) } #ifdef HAILO_SUPPORT_MULTI_PROCESS -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wreturn-type" Expected> OutputVStreamClient::create(uint32_t outputs_vstream_handle) { grpc::ChannelArguments ch_args; ch_args.SetMaxReceiveMessageSize(-1); - auto channel = grpc::CreateCustomChannel(HAILO_DEFAULT_UDS_ADDR, grpc::InsecureChannelCredentials(), ch_args); + auto channel = grpc::CreateCustomChannel(HAILORT_SERVICE_DEFAULT_ADDR, grpc::InsecureChannelCredentials(), ch_args); CHECK_AS_EXPECTED(channel != nullptr, HAILO_INTERNAL_FAILURE); - auto client = std::unique_ptr(new HailoRtRpcClient(channel)); + auto client = make_unique_nothrow(channel); CHECK_AS_EXPECTED(client != nullptr, HAILO_OUT_OF_HOST_MEMORY); - auto user_buffer_format = client->OutputVStream_get_user_buffer_format(outputs_vstream_handle); + auto user_buffer_format = client->OutputVStream_get_user_buffer_format(outputs_vstream_handle); CHECK_EXPECTED(user_buffer_format); - auto info = client->OutputVStream_get_info(outputs_vstream_handle); + auto info = client->OutputVStream_get_info(outputs_vstream_handle); CHECK_EXPECTED(info); return std::shared_ptr(new OutputVStreamClient(std::move(client), std::move(outputs_vstream_handle), @@ -1351,10 +1374,9 @@ hailo_status OutputVStreamClient::read(MemoryView buffer) hailo_status OutputVStreamClient::abort() { - auto channel = grpc::CreateChannel(HAILO_DEFAULT_UDS_ADDR, grpc::InsecureChannelCredentials()); - CHECK(channel != nullptr, HAILO_INTERNAL_FAILURE); - auto abort_client = std::unique_ptr(new HailoRtRpcClient(channel)); - CHECK(abort_client != nullptr, HAILO_OUT_OF_HOST_MEMORY); + auto expected_client = HailoRtRpcClientUtils::create_client(); + CHECK_EXPECTED_AS_STATUS(expected_client); + auto abort_client = expected_client.release(); return abort_client->OutputVStream_abort(m_handle); } @@ -1430,7 +1452,34 @@ const std::vector> &OutputVStreamClient::get_pi return m_pipeline; } -#pragma GCC diagnostic pop +hailo_status OutputVStreamClient::create_client() +{ + auto expected_client = HailoRtRpcClientUtils::create_client(); + CHECK_EXPECTED_AS_STATUS(expected_client); + m_client = expected_client.release(); + return HAILO_SUCCESS; +} + +hailo_status OutputVStreamClient::before_fork() +{ + m_client.reset(); + return HAILO_SUCCESS; +} + +hailo_status OutputVStreamClient::after_fork_in_parent() +{ + return create_client(); +} + +hailo_status OutputVStreamClient::after_fork_in_child() +{ + auto status = create_client(); + CHECK_SUCCESS(status); + auto expected_dup_handle = m_client->OutputVStream_dup_handle(OsUtils::get_curr_pid(), m_handle); + CHECK_EXPECTED_AS_STATUS(expected_dup_handle); + m_handle = expected_dup_handle.value(); + return HAILO_SUCCESS; +} #endif // HAILO_SUPPORT_MULTI_PROCESS Expected> HwReadElement::create(std::shared_ptr stream, const std::string &name, std::chrono::milliseconds timeout, @@ -1469,7 +1518,7 @@ HwReadElement::HwReadElement(std::shared_ptr stream, BufferPoolPtr m_transform_pool(transform_pool), m_timeout(timeout), m_shutdown_event(shutdown_event), - m_activation_wait_or_shutdown(stream->get_network_group_activated_event(), shutdown_event), + m_activation_wait_or_shutdown(stream->get_core_op_activated_event(), shutdown_event), m_transform_context(std::move(transform_context)) {} @@ -1855,9 +1904,9 @@ Expected> VStreamsBuilderUtils::create_inputs(std::sha std::vector> elements; std::vector vstreams; - EventPtr network_group_activated_event = nullptr; + EventPtr core_op_activated_event = nullptr; if (!input_stream->is_scheduled()) { - network_group_activated_event = input_stream->get_network_group_activated_event(); + core_op_activated_event = input_stream->get_core_op_activated_event(); } auto shutdown_event = Event::create_shared(Event::State::not_signalled); @@ -1900,13 +1949,13 @@ Expected> VStreamsBuilderUtils::create_inputs(std::sha input_stream->set_timeout(user_timeout); auto vstream = InputVStream::create(vstream_info, vstream_params, pre_infer_elem.release(), hw_write_elem.release(), std::move(elements), - std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED(vstream); vstreams.emplace_back(vstream.release()); } else { input_stream->set_timeout(user_timeout); auto vstream = InputVStream::create(vstream_info, vstream_params, hw_write_elem.value(), hw_write_elem.value(), std::move(elements), - std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED(vstream); vstreams.emplace_back(vstream.release()); } @@ -1924,9 +1973,9 @@ Expected> VStreamsBuilderUtils::create_outputs(std::s std::vector> elements; std::vector vstreams; - EventPtr network_group_activated_event = nullptr; + EventPtr core_op_activated_event = nullptr; if (!output_stream->is_scheduled()) { - network_group_activated_event = output_stream->get_network_group_activated_event(); + core_op_activated_event = output_stream->get_core_op_activated_event(); } auto shutdown_event = Event::create_shared(Event::State::not_signalled); @@ -2003,13 +2052,13 @@ Expected> VStreamsBuilderUtils::create_outputs(std::s output_stream->set_timeout(std::chrono::milliseconds(HAILO_INFINITE)); hw_read_queue_elem->get()->set_timeout(std::chrono::milliseconds(HAILO_INFINITE)); auto vstream = OutputVStream::create(vstream_info->second, vstream_params, post_infer_queue_elem.release(), std::move(elements), - std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED(vstream); vstreams.emplace_back(vstream.release()); } else { output_stream->set_timeout(std::chrono::milliseconds(vstream_params.timeout_ms)); auto vstream = OutputVStream::create(vstream_info->second, vstream_params, hw_read_elem.release(), std::move(elements), - std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED(vstream); vstreams.emplace_back(vstream.release()); } @@ -2068,18 +2117,8 @@ Expected> VStreamsBuilderUtils::create_output_nms(Out Expected> VStreamsBuilderUtils::create_output_post_process_nms(OutputStreamPtrVector &output_streams, hailo_vstream_params_t vstreams_params, const std::map &output_vstream_infos, - const NetFlowYoloNmsElement &nms_op) + const NetFlowElement &nms_op) { - static const auto EXPECTED_OUTPUTS = 3; - CHECK_AS_EXPECTED(output_streams.size() == EXPECTED_OUTPUTS, - HAILO_INVALID_ARGUMENT, "Core expected to have exactly {} outputs when using NMS post-processing", EXPECTED_OUTPUTS); - - std::sort(output_streams.begin(), output_streams.end(), [](auto &stream_0, auto &stream_1) { - std::string name0(stream_0->get_info().name); - std::string name1(stream_1->get_info().name); - return name0 < name1; - }); - auto shutdown_event = Event::create_shared(Event::State::not_signalled); CHECK_AS_EXPECTED(nullptr != shutdown_event, HAILO_OUT_OF_HOST_MEMORY); @@ -2132,9 +2171,9 @@ hailo_status VStreamsBuilderUtils::add_demux(std::shared_ptr outpu base_elements.push_back(demux_elem.value()); CHECK_SUCCESS(PipelinePad::link_pads(hw_read_elem, demux_elem.value())); - EventPtr network_group_activated_event = nullptr; + EventPtr core_op_activated_event = nullptr; if (!output_stream->is_scheduled()) { - network_group_activated_event = output_stream->get_network_group_activated_event(); + core_op_activated_event = output_stream->get_core_op_activated_event(); } uint32_t i = 0; @@ -2186,7 +2225,7 @@ hailo_status VStreamsBuilderUtils::add_demux(std::shared_ptr outpu CHECK_SUCCESS(PipelinePad::link_pads(post_infer_elem.value(), post_infer_queue_elem.value())); auto vstream = OutputVStream::create(vstream_info->second, vstream_params, post_infer_queue_elem.release(), std::move(current_vstream_elements), - std::move(pipeline_status_copy), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status_copy), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED_AS_STATUS(vstream); vstreams.emplace_back(vstream.release()); } else { @@ -2199,7 +2238,7 @@ hailo_status VStreamsBuilderUtils::add_demux(std::shared_ptr outpu CHECK_SUCCESS(PipelinePad::link_pads(demux_queue_elem.value(), user_copy_elem.value())); auto vstream = OutputVStream::create(vstream_info->second, vstream_params, user_copy_elem.release(), std::move(current_vstream_elements), - std::move(pipeline_status_copy), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status_copy), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED_AS_STATUS(vstream); vstreams.emplace_back(vstream.release()); } @@ -2265,9 +2304,9 @@ hailo_status VStreamsBuilderUtils::add_nms_fuse(OutputStreamPtrVector &output_st auto should_transform = OutputTransformContext::is_transformation_required({}, src_stream_format, {}, vstreams_params.user_buffer_format, vstream_info->second.quant_info); - EventPtr network_group_activated_event = nullptr; + EventPtr core_op_activated_event = nullptr; if (!output_streams[0]->is_scheduled()) { - network_group_activated_event = output_streams[0]->get_network_group_activated_event(); + core_op_activated_event = output_streams[0]->get_core_op_activated_event(); } if (should_transform) { @@ -2295,12 +2334,12 @@ hailo_status VStreamsBuilderUtils::add_nms_fuse(OutputStreamPtrVector &output_st CHECK_SUCCESS(PipelinePad::link_pads(post_infer_elem.value(), post_infer_queue_elem.value())); auto vstream = OutputVStream::create(vstream_info->second, vstreams_params, post_infer_queue_elem.release(), std::move(elements), - std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED_AS_STATUS(vstream); vstreams.emplace_back(vstream.release()); } else { auto vstream = OutputVStream::create(vstream_info->second, vstreams_params, nms_elem.release(), std::move(elements), - std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED_AS_STATUS(vstream); vstreams.emplace_back(vstream.release()); } @@ -2312,7 +2351,7 @@ hailo_status VStreamsBuilderUtils::add_nms_post_process(OutputStreamPtrVector &o std::vector> &elements, std::vector &vstreams, EventPtr shutdown_event, std::shared_ptr> pipeline_status, const std::map &output_vstream_infos, - const NetFlowYoloNmsElement &nms_op) + const NetFlowElement &nms_op) { auto first_stream_info = output_streams[0]->get_info(); if (vstreams_params.user_buffer_format.type == HAILO_FORMAT_TYPE_AUTO) { @@ -2327,42 +2366,40 @@ hailo_status VStreamsBuilderUtils::add_nms_post_process(OutputStreamPtrVector &o CHECK(vstreams_params.user_buffer_format.order == HAILO_FORMAT_ORDER_HAILO_NMS, HAILO_INVALID_ARGUMENT, "NMS output format order must be HAILO_FORMAT_ORDER_HAILO_NMS"); - hailo_nms_info_t nms_info = { - nms_op.classes, - nms_op.max_proposals_per_class, - sizeof(hailo_bbox_float32_t), - nms_op.input_division_factor, - false, - hailo_nms_defuse_info_t() - }; - - std::vector shapes; - shapes.reserve(output_streams.size()); - std::vector formats; - formats.reserve(output_streams.size()); - std::vector quant_infos; - quant_infos.reserve(output_streams.size()); + std::map inputs_metadata; + std::map outputs_metadata; for (uint32_t i = 0; i < output_streams.size(); ++i) { const auto &curr_stream_info = output_streams[i]->get_info(); - shapes.push_back(curr_stream_info.shape); - formats.push_back(curr_stream_info.format); - quant_infos.push_back(curr_stream_info.quant_info); + net_flow::BufferMetaData input_metadata = { + curr_stream_info.shape, + curr_stream_info.hw_shape, + curr_stream_info.format, + curr_stream_info.quant_info + }; + inputs_metadata.insert({curr_stream_info.name, input_metadata}); } - const auto &output_pads = nms_op.output_pads; + const auto &output_pads = nms_op.op->outputs_metadata(); assert(output_pads.size() == 1); - auto vstream_info = output_vstream_infos.find(output_pads[0].name); + auto vstream_info = output_vstream_infos.find(output_pads.begin()->first); CHECK(vstream_info != output_vstream_infos.end(), HAILO_NOT_FOUND, "Failed to find vstream info of {}", nms_op.name); + net_flow::BufferMetaData output_metadata = { + vstream_info->second.shape, + vstream_info->second.shape, + vstream_info->second.format, + vstream_info->second.quant_info + }; + outputs_metadata.insert({vstream_info->first, output_metadata}); - auto nms_elem = NmsPostProcessMuxElement::create(nms_op, shapes, formats, quant_infos, nms_info, + auto nms_elem = NmsPostProcessMuxElement::create(nms_op.op, nms_op.nms_info, PipelineObject::create_element_name("NmsPostProcessMuxElement", nms_op.name, 0), vstreams_params, shutdown_event, pipeline_status); CHECK_EXPECTED_AS_STATUS(nms_elem); hailo_format_t nms_src_format; nms_src_format.flags = HAILO_FORMAT_FLAGS_QUANTIZED; - nms_src_format.order = HAILO_FORMAT_ORDER_NHWC; + nms_src_format.order = HAILO_FORMAT_ORDER_NHCW; nms_src_format.type = first_stream_info.format.type; for (uint32_t i = 0; i < output_streams.size(); ++i) { @@ -2370,21 +2407,14 @@ hailo_status VStreamsBuilderUtils::add_nms_post_process(OutputStreamPtrVector &o output_streams[i]->set_timeout(HAILO_INFINITE_TIMEOUT); auto should_transform = OutputTransformContext::is_transformation_required(curr_stream_info.hw_shape, curr_stream_info.format, - curr_stream_info.shape, nms_src_format, vstream_info->second.quant_info); + curr_stream_info.hw_shape, nms_src_format, vstream_info->second.quant_info); - std::unique_ptr transform_context = nullptr; - - if (should_transform) { - auto expected_transform_context = OutputTransformContext::create(curr_stream_info.hw_shape, curr_stream_info.format, - curr_stream_info.shape, nms_src_format, vstream_info->second.quant_info, nms_info); - CHECK_EXPECTED_AS_STATUS(expected_transform_context); - transform_context = expected_transform_context.release(); - } + CHECK(!should_transform, HAILO_INVALID_ARGUMENT, "Unexpected transformation required for {}", curr_stream_info.name); auto hw_read_elem = HwReadElement::create(output_streams[i], PipelineObject::create_element_name("HwReadElement", curr_stream_info.name, curr_stream_info.index), HAILO_INFINITE_TIMEOUT, vstreams_params.queue_size, vstreams_params.pipeline_elements_stats_flags, - vstreams_params.vstream_stats_flags, shutdown_event, pipeline_status, std::move(transform_context)); + vstreams_params.vstream_stats_flags, shutdown_event, pipeline_status); CHECK_EXPECTED_AS_STATUS(hw_read_elem); elements.push_back(hw_read_elem.value()); @@ -2396,19 +2426,20 @@ hailo_status VStreamsBuilderUtils::add_nms_post_process(OutputStreamPtrVector &o elements.push_back(nms_source_queue_elem.value()); CHECK_SUCCESS(PipelinePad::link_pads(hw_read_elem.value(), nms_source_queue_elem.value())); CHECK_SUCCESS(PipelinePad::link_pads(nms_source_queue_elem.value(), nms_elem.value(), 0, i)); + nms_elem.value()->add_sink_name(curr_stream_info.name); } elements.push_back(nms_elem.value()); auto pipeline_latency_accumulator = create_pipeline_latency_accumulator(vstreams_params); CHECK_EXPECTED_AS_STATUS(pipeline_latency_accumulator); - EventPtr network_group_activated_event = nullptr; + EventPtr core_op_activated_event = nullptr; if (!output_streams[0]->is_scheduled()) { - network_group_activated_event = output_streams[0]->get_network_group_activated_event(); + core_op_activated_event = output_streams[0]->get_core_op_activated_event(); } auto vstream = OutputVStream::create(vstream_info->second, vstreams_params, nms_elem.release(), std::move(elements), - std::move(pipeline_status), shutdown_event, network_group_activated_event, pipeline_latency_accumulator.release()); + std::move(pipeline_status), shutdown_event, core_op_activated_event, pipeline_latency_accumulator.release()); CHECK_EXPECTED_AS_STATUS(vstream); vstreams.emplace_back(vstream.release()); diff --git a/hailort/libhailort/src/vstream_internal.hpp b/hailort/libhailort/src/net_flow/pipeline/vstream_internal.hpp similarity index 91% rename from hailort/libhailort/src/vstream_internal.hpp rename to hailort/libhailort/src/net_flow/pipeline/vstream_internal.hpp index a4ff6e2..ef1126f 100644 --- a/hailort/libhailort/src/vstream_internal.hpp +++ b/hailort/libhailort/src/net_flow/pipeline/vstream_internal.hpp @@ -26,17 +26,19 @@ #ifndef _HAILO_VSTREAM_INTERNAL_HPP_ #define _HAILO_VSTREAM_INTERNAL_HPP_ -#include "pipeline.hpp" -#include "hef_internal.hpp" -#include "net_flow/ops/yolo_post_processing.hpp" #include "hailo/transform.hpp" #include "hailo/stream.hpp" -#include "context_switch/network_group_internal.hpp" + +#include "hef/hef_internal.hpp" +#include "net_flow/pipeline/pipeline.hpp" +#include "net_flow/ops/yolo_post_process.hpp" +#include "network_group/network_group_internal.hpp" #ifdef HAILO_SUPPORT_MULTI_PROCESS -#include "hailort_rpc_client.hpp" +#include "service/hailort_rpc_client.hpp" #endif // HAILO_SUPPORT_MULTI_PROCESS + namespace hailort { @@ -65,11 +67,15 @@ public: virtual hailo_status stop_vstream(); virtual hailo_status stop_and_clear(); + virtual hailo_status before_fork() { return HAILO_SUCCESS; }; + virtual hailo_status after_fork_in_parent() { return HAILO_SUCCESS; }; + virtual hailo_status after_fork_in_child() { return HAILO_SUCCESS; }; + protected: BaseVStream(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, - EventPtr &&network_group_activated_event, hailo_status &output_status); + EventPtr &&core_op_activated_event, hailo_status &output_status); BaseVStream() = default; virtual std::string get_pipeline_description() const = 0; @@ -83,7 +89,7 @@ protected: volatile bool m_is_aborted; std::shared_ptr> m_pipeline_status; EventPtr m_shutdown_event; - EventPtr m_network_group_activated_event; + EventPtr m_core_op_activated_event; std::map m_fps_accumulators; std::map m_latency_accumulators; std::map> m_queue_size_accumulators; @@ -97,7 +103,7 @@ public: static Expected> create(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::shared_ptr pipeline_exit, std::vector> &&pipeline, - std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr network_group_activated_event, + std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator); InputVStreamInternal(InputVStreamInternal &&other) noexcept = default; InputVStreamInternal &operator=(InputVStreamInternal &&other) noexcept = default; @@ -112,7 +118,7 @@ protected: InputVStreamInternal(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, - EventPtr &&network_group_activated_event, hailo_status &output_status); + EventPtr &&core_op_activated_event, hailo_status &output_status); InputVStreamInternal() = default; }; @@ -124,7 +130,7 @@ public: const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, - EventPtr network_group_activated_event, AccumulatorPtr pipeline_latency_accumulator); + EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator); OutputVStreamInternal(OutputVStreamInternal &&other) noexcept = default; OutputVStreamInternal &operator=(OutputVStreamInternal &&other) noexcept = default; virtual ~OutputVStreamInternal() = default; @@ -137,7 +143,7 @@ protected: OutputVStreamInternal(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, - EventPtr network_group_activated_event, hailo_status &output_status); + EventPtr core_op_activated_event, hailo_status &output_status); OutputVStreamInternal() = default; }; @@ -147,7 +153,7 @@ public: static Expected> create(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::shared_ptr pipeline_exit, std::vector> &&pipeline, - std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr network_group_activated_event, + std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator); InputVStreamImpl(InputVStreamImpl &&) noexcept = default; InputVStreamImpl(const InputVStreamImpl &) = delete; @@ -161,7 +167,7 @@ private: InputVStreamImpl(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, - EventPtr network_group_activated_event, hailo_status &output_status); + EventPtr core_op_activated_event, hailo_status &output_status); }; class OutputVStreamImpl : public OutputVStreamInternal @@ -171,7 +177,7 @@ public: const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, - EventPtr network_group_activated_event, AccumulatorPtr pipeline_latency_accumulator); + EventPtr core_op_activated_event, AccumulatorPtr pipeline_latency_accumulator); OutputVStreamImpl(OutputVStreamImpl &&) noexcept = default; OutputVStreamImpl(const OutputVStreamImpl &) = delete; OutputVStreamImpl &operator=(OutputVStreamImpl &&) noexcept = default; @@ -194,7 +200,7 @@ private: OutputVStreamImpl(const hailo_vstream_info_t &vstream_info, const hailo_vstream_params_t &vstream_params, std::shared_ptr pipeline_entry, std::vector> &&pipeline, std::shared_ptr> &&pipeline_status, EventPtr shutdown_event, AccumulatorPtr pipeline_latency_accumulator, - EventPtr network_group_activated_event, hailo_status &output_status); + EventPtr core_op_activated_event, hailo_status &output_status); std::function m_cant_read_callback; std::function m_can_read_callback; @@ -226,10 +232,14 @@ public: virtual const std::map> &get_queue_size_accumulators() const override; virtual AccumulatorPtr get_pipeline_latency_accumulator() const override; virtual const std::vector> &get_pipeline() const override; + virtual hailo_status before_fork() override; + virtual hailo_status after_fork_in_parent() override; + virtual hailo_status after_fork_in_child() override; private: InputVStreamClient(std::unique_ptr client, uint32_t input_vstream_handle, hailo_format_t &&user_buffer_format, hailo_vstream_info_t &&info); + hailo_status create_client(); std::unique_ptr m_client; uint32_t m_handle; @@ -261,11 +271,16 @@ public: virtual const std::map> &get_queue_size_accumulators() const override; virtual AccumulatorPtr get_pipeline_latency_accumulator() const override; virtual const std::vector> &get_pipeline() const override; + virtual hailo_status before_fork() override; + virtual hailo_status after_fork_in_parent() override; + virtual hailo_status after_fork_in_child() override; private: OutputVStreamClient(std::unique_ptr client, uint32_t outputs_vstream_handle, hailo_format_t &&user_buffer_format, hailo_vstream_info_t &&info); + hailo_status create_client(); + std::unique_ptr m_client; uint32_t m_handle; hailo_format_t m_user_buffer_format; @@ -329,28 +344,30 @@ private: class NmsPostProcessMuxElement : public BaseMuxElement { public: - static Expected> create(const NetFlowYoloNmsElement &nms_op, - const std::vector &shapes, const std::vector &formats, - const std::vector &quant_infos, hailo_format_t format, hailo_nms_info_t nms_info, - const std::string &name, std::chrono::milliseconds timeout, size_t buffer_pool_size, + static Expected> create(std::shared_ptr nms_op, + hailo_nms_info_t nms_info, const std::string &name, std::chrono::milliseconds timeout, size_t buffer_pool_size, hailo_pipeline_elem_stats_flags_t elem_flags, hailo_vstream_stats_flags_t vstream_flags, EventPtr shutdown_event, std::shared_ptr> pipeline_status); - static Expected> create(const NetFlowYoloNmsElement &nms_op, - const std::vector &shapes, const std::vector &formats, - const std::vector &quant_infos, hailo_nms_info_t nms_info, const std::string &name, - const hailo_vstream_params_t &vstream_params, EventPtr shutdown_event, std::shared_ptr> pipeline_status); - NmsPostProcessMuxElement(const net_flow::YOLOv5PostProcessingOp &nms_op, BufferPoolPtr &&pool, const std::string &name, + static Expected> create(std::shared_ptr nms_op, + hailo_nms_info_t nms_info, const std::string &name, const hailo_vstream_params_t &vstream_params, + EventPtr shutdown_event, std::shared_ptr> pipeline_status); + NmsPostProcessMuxElement(std::shared_ptr nms_op, BufferPoolPtr &&pool, const std::string &name, std::chrono::milliseconds timeout, DurationCollector &&duration_collector, std::shared_ptr> &&pipeline_status); virtual std::vector get_queue_size_accumulators() override; + void add_sink_name(const std::string &name) // TODO: remove this (HRT-8875) + { + m_sinks_names.push_back(name); + } protected: virtual Expected action(std::vector &&inputs, PipelineBuffer &&optional) override; private: - net_flow::YOLOv5PostProcessingOp m_nms_op; + std::shared_ptr m_nms_op; BufferPoolPtr m_pool; + std::vector m_sinks_names; // TODO: remove this (HRT-8875) }; class NmsMuxElement : public BaseMuxElement @@ -484,7 +501,7 @@ public: static Expected> create_output_post_process_nms(OutputStreamPtrVector &output_streams, hailo_vstream_params_t vstreams_params, const std::map &output_vstream_infos, - const NetFlowYoloNmsElement &nms_op); + const NetFlowElement &nms_op); static hailo_status add_demux(std::shared_ptr output_stream, NameToVStreamParamsMap &vstreams_params_map, std::vector> &&elements, std::vector &vstreams, std::shared_ptr hw_read_elem, EventPtr shutdown_event, std::shared_ptr> pipeline_status, @@ -497,7 +514,7 @@ public: std::vector> &elements, std::vector &vstreams, EventPtr shutdown_event, std::shared_ptr> pipeline_status, const std::map &output_vstream_infos, - const NetFlowYoloNmsElement &nms_op); + const NetFlowElement &nms_op); static Expected create_pipeline_latency_accumulator(const hailo_vstream_params_t &vstreams_params); }; diff --git a/hailort/libhailort/src/network_group/CMakeLists.txt b/hailort/libhailort/src/network_group/CMakeLists.txt new file mode 100644 index 0000000..beec975 --- /dev/null +++ b/hailort/libhailort/src/network_group/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/network_group.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/network_group/network_group.cpp b/hailort/libhailort/src/network_group/network_group.cpp new file mode 100644 index 0000000..bc09df0 --- /dev/null +++ b/hailort/libhailort/src/network_group/network_group.cpp @@ -0,0 +1,408 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file network_group.cpp + * @brief: Configured Network Group and Activated Network Group + **/ + +#include "hailo/transform.hpp" +#include "hailo/vstream.hpp" +#include "hailo/hailort_defaults.hpp" + +#include "common/utils.hpp" +#include "common/runtime_statistics_internal.hpp" + +#include "network_group/network_group_internal.hpp" +#include "hef/hef_internal.hpp" +#include "eth/eth_stream.hpp" +#include "vdma/vdma_stream.hpp" +#include "mipi/mipi_stream.hpp" +#include "device_common/control.hpp" +#include "net_flow/pipeline/vstream_internal.hpp" +#include "core_op/resource_manager/resource_manager.hpp" + + +namespace hailort +{ + +Expected> ConfiguredNetworkGroup::activate() +{ + const auto network_group_params = HailoRTDefaults::get_active_network_group_params(); + return activate(network_group_params); +} + +Expected> ConfiguredNetworkGroupBase::activate( + const hailo_activate_network_group_params_t &network_group_params) +{ + return get_core_op()->activate(network_group_params); +} + +/* Network group base functions */ +Expected ConfiguredNetworkGroupBase::get_latency_measurement(const std::string &network_name) +{ + return get_core_op()->get_latency_measurement(network_name); +} + +Expected ConfiguredNetworkGroupBase::get_output_streams_from_vstream_names( + const std::map &outputs_params) +{ + return get_core_op()->get_output_streams_from_vstream_names(outputs_params); +} + +Expected ConfiguredNetworkGroupBase::get_output_streams_by_vstream_name(const std::string &name) +{ + return get_core_op()->get_output_streams_by_vstream_name(name); +} + +Expected ConfiguredNetworkGroupBase::get_layer_info(const std::string &stream_name) +{ + return get_core_op()->get_layer_info(stream_name); +} + +ConfiguredNetworkGroupBase::ConfiguredNetworkGroupBase( + const ConfigureNetworkParams &config_params, std::vector> &&core_ops, + std::vector> &&net_flow_ops) : + m_config_params(config_params), + m_core_ops(std::move(core_ops)), + m_net_flow_ops(std::move(net_flow_ops)) +{} + +// static func +uint16_t ConfiguredNetworkGroupBase::get_smallest_configured_batch_size(const ConfigureNetworkParams &config_params) +{ + // There are two possible situations: + // 1) All networks in the network group have the same configured (and hence smallest) batch_size => + // We return that batch size. + // 2) Not all of the networks have the same configured (and hence smallest) batch_size. Currently, when + // using dynamic_batch_sizes, all networks will use the same dynamic_batch_size (until HRT-6535 is done). + // Hence, we must not set a dynamic_batch_size to a value greater than the smallest configured network + // batch_size (e.g. all the resources allocated are for at most the configured network batch_size). + + /* We iterate over all network's batch_sizes to get the non-default min. + Ignoring HAILO_DEFAULT_BATCH_SIZE as it is not a real batch-value, + but indicating the scheduler should optimize batches by himself */ + uint16_t min_batch_size = UINT16_MAX; + for (const auto &network_params_pair : config_params.network_params_by_name) { + if ((HAILO_DEFAULT_BATCH_SIZE != network_params_pair.second.batch_size) && + (network_params_pair.second.batch_size < min_batch_size)) { + min_batch_size = network_params_pair.second.batch_size; + } + } + return (UINT16_MAX == min_batch_size) ? DEFAULT_ACTUAL_BATCH_SIZE : min_batch_size; +} + +Expected> ConfiguredNetworkGroupBase::activate_with_batch(uint16_t dynamic_batch_size, + bool resume_pending_stream_transfers) +{ + return get_core_op()->activate_with_batch(dynamic_batch_size, resume_pending_stream_transfers); +} + +const std::string &ConfiguredNetworkGroupBase::get_network_group_name() const +{ + return get_core_op_metadata()->core_op_name(); +} + +const std::string &ConfiguredNetworkGroupBase::name() const +{ + return get_core_op_metadata()->core_op_name(); +} + +hailo_status ConfiguredNetworkGroupBase::activate_low_level_streams(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) +{ + return get_core_op()->activate_low_level_streams(dynamic_batch_size, resume_pending_stream_transfers); +} + +hailo_status ConfiguredNetworkGroupBase::deactivate_low_level_streams() +{ + return get_core_op()->deactivate_low_level_streams(); +} + +std::shared_ptr ConfiguredNetworkGroupBase::get_core_op() const +{ + assert(m_core_ops.size() == 1); + return m_core_ops[0]; +} + +const std::shared_ptr ConfiguredNetworkGroupBase::get_core_op_metadata() const +{ + assert(m_core_ops.size() == 1); + return m_core_ops[0]->metadata(); +} + +Expected ConfiguredNetworkGroupBase::get_stream_batch_size(const std::string &stream_name) +{ + return get_core_op()->get_stream_batch_size(stream_name); +} + +bool ConfiguredNetworkGroupBase::is_multi_context() const +{ + return get_core_op()->is_multi_context(); +} + +const ConfigureNetworkParams ConfiguredNetworkGroupBase::get_config_params() const +{ + return get_core_op()->get_config_params(); +} + +Expected> ConfiguredNetworkGroupBase::get_vstream_names_from_stream_name(const std::string &stream_name) +{ + return get_core_op()->get_vstream_names_from_stream_name(stream_name); +} + +const SupportedFeatures &ConfiguredNetworkGroupBase::get_supported_features() +{ + return get_core_op()->get_supported_features(); +} + +hailo_status ConfiguredNetworkGroupBase::create_input_stream_from_config_params(Device &device, + const hailo_stream_parameters_t &stream_params, const std::string &stream_name) +{ + return get_core_op()->create_input_stream_from_config_params(device, stream_params, stream_name); +} + +hailo_status ConfiguredNetworkGroupBase::create_vdma_input_stream(Device &device, const std::string &stream_name, + const LayerInfo &layer_info, const hailo_stream_parameters_t &stream_params) +{ + return get_core_op()->create_vdma_input_stream(device, stream_name, layer_info, stream_params); +} + +hailo_status ConfiguredNetworkGroupBase::create_output_stream_from_config_params(Device &device, + const hailo_stream_parameters_t &stream_params, const std::string &stream_name) +{ + return get_core_op()->create_output_stream_from_config_params(device, stream_params, stream_name); +} + +hailo_status ConfiguredNetworkGroupBase::create_vdma_output_stream(Device &device, const std::string &stream_name, + const LayerInfo &layer_info, const hailo_stream_parameters_t &stream_params) +{ + return get_core_op()->create_vdma_output_stream(device, stream_name, layer_info, stream_params); +} + +hailo_status ConfiguredNetworkGroupBase::create_streams_from_config_params(Device &device) +{ + return get_core_op()->create_streams_from_config_params(device); +} + +Expected ConfiguredNetworkGroupBase::get_input_streams_by_network(const std::string &network_name) +{ + return get_core_op()->get_input_streams_by_network(network_name); +} + +Expected ConfiguredNetworkGroupBase::get_output_streams_by_network(const std::string &network_name) +{ + return get_core_op()->get_output_streams_by_network(network_name); +} + +InputStreamRefVector ConfiguredNetworkGroupBase::get_input_streams() +{ + return get_core_op()->get_input_streams(); +} + +OutputStreamRefVector ConfiguredNetworkGroupBase::get_output_streams() +{ + return get_core_op()->get_output_streams(); +} + +ExpectedRef ConfiguredNetworkGroupBase::get_input_stream_by_name(const std::string& name) +{ + return get_core_op()->get_input_stream_by_name(name); +} + +ExpectedRef ConfiguredNetworkGroupBase::get_output_stream_by_name(const std::string& name) +{ + return get_core_op()->get_output_stream_by_name(name); +} + +std::vector> ConfiguredNetworkGroupBase::get_input_streams_by_interface( + hailo_stream_interface_t stream_interface) +{ + return get_core_op()->get_input_streams_by_interface(stream_interface); +} + +std::vector> ConfiguredNetworkGroupBase::get_output_streams_by_interface( + hailo_stream_interface_t stream_interface) +{ + return get_core_op()->get_output_streams_by_interface(stream_interface); +} + +hailo_status ConfiguredNetworkGroupBase::wait_for_activation(const std::chrono::milliseconds &timeout) +{ + return get_core_op()->wait_for_activation(timeout); +} + +Expected>> ConfiguredNetworkGroupBase::get_output_vstream_groups() +{ + return get_core_op()->get_output_vstream_groups(); +} + +Expected>> ConfiguredNetworkGroupBase::make_output_vstream_params_groups( + bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size) +{ + return get_core_op()->make_output_vstream_params_groups(quantized, format_type, timeout_ms, queue_size); +} + +Expected> ConfiguredNetworkGroupBase::make_input_vstream_params( + bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size, + const std::string &network_name) +{ + return get_core_op()->make_input_vstream_params(quantized, format_type, timeout_ms, queue_size, network_name); +} + +Expected> ConfiguredNetworkGroupBase::make_output_vstream_params( + bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size, + const std::string &network_name) +{ + return get_core_op()->make_output_vstream_params(quantized, format_type, timeout_ms, queue_size, network_name); +} + +Expected> ConfiguredNetworkGroupBase::get_network_infos() const +{ + return get_core_op()->get_network_infos(); +} + +Expected> ConfiguredNetworkGroupBase::get_all_stream_infos( + const std::string &network_name) const +{ + return get_core_op()->get_all_stream_infos(network_name); +} + +Expected> ConfiguredNetworkGroupBase::get_input_vstream_infos( + const std::string &network_name) const +{ + return get_core_op()->get_input_vstream_infos(network_name); +} + +Expected> ConfiguredNetworkGroupBase::get_output_vstream_infos( + const std::string &network_name) const +{ + return get_core_op()->get_output_vstream_infos(network_name); +} + +Expected> ConfiguredNetworkGroupBase::get_all_vstream_infos( + const std::string &network_name) const +{ + return get_core_op()->get_all_vstream_infos(network_name); +} + +AccumulatorPtr ConfiguredNetworkGroupBase::get_activation_time_accumulator() const +{ + return get_core_op()->get_activation_time_accumulator(); +} + +AccumulatorPtr ConfiguredNetworkGroupBase::get_deactivation_time_accumulator() const +{ + return get_core_op()->get_deactivation_time_accumulator(); +} + +static hailo_vstream_params_t expand_vstream_params_autos(const hailo_stream_info_t &stream_info, + const hailo_vstream_params_t &vstream_params) +{ + auto local_vstream_params = vstream_params; + local_vstream_params.user_buffer_format = HailoRTDefaults::expand_auto_format(vstream_params.user_buffer_format, + stream_info.format); + return local_vstream_params; +} + +static std::map vstream_infos_vector_to_map(std::vector &&vstream_info_vector) +{ + std::map vstream_infos_map; + for (const auto &vstream_info : vstream_info_vector) { + vstream_infos_map.emplace(std::string(vstream_info.name), vstream_info); + } + + return vstream_infos_map; +} + +Expected> ConfiguredNetworkGroupBase::create_input_vstreams(const std::map &inputs_params) +{ + auto input_vstream_infos = get_input_vstream_infos(); + CHECK_EXPECTED(input_vstream_infos); + auto input_vstream_infos_map = vstream_infos_vector_to_map(input_vstream_infos.release()); + + std::vector vstreams; + vstreams.reserve(inputs_params.size()); + for (const auto &name_params_pair : inputs_params) { + auto input_stream_expected = get_shared_input_stream_by_name(name_params_pair.first); + CHECK_EXPECTED(input_stream_expected); + auto input_stream = input_stream_expected.release(); + + const auto vstream_info = input_vstream_infos_map.find(name_params_pair.first); + CHECK_AS_EXPECTED(vstream_info != input_vstream_infos_map.end(), HAILO_NOT_FOUND, + "Failed to find vstream info of {}", name_params_pair.first); + + const auto vstream_params = expand_vstream_params_autos(input_stream->get_info(), name_params_pair.second); + auto inputs = VStreamsBuilderUtils::create_inputs(input_stream, vstream_info->second, vstream_params); + CHECK_EXPECTED(inputs); + + vstreams.insert(vstreams.end(), std::make_move_iterator(inputs->begin()), std::make_move_iterator(inputs->end())); + } + return vstreams; +} + +Expected> ConfiguredNetworkGroupBase::create_output_vstreams(const std::map &outputs_params) +{ + std::vector vstreams; + vstreams.reserve(outputs_params.size()); + auto output_streams = get_output_streams_from_vstream_names(outputs_params); + CHECK_EXPECTED(output_streams); + + auto output_vstream_infos = get_output_vstream_infos(); + CHECK_EXPECTED(output_vstream_infos); + auto output_vstream_infos_map = vstream_infos_vector_to_map(output_vstream_infos.release()); + + // We iterate through all output streams, and if they are nms, we collect them together by their original stream name. + // We need this step because all nms output streams of the same original stream need to be fused together + + std::unordered_map> post_process_nms_ops; + std::set post_process_stream_inputs; + for (auto &op : m_net_flow_ops) { + post_process_nms_ops.insert({op->name, op}); + post_process_stream_inputs.insert(op->input_streams.begin(), op->input_streams.end()); + } + std::map> nms_op_output_streams; + std::map> nms_output_streams; + for (auto &stream_params_pair : output_streams.value()) { + if ((HAILO_FORMAT_ORDER_HAILO_NMS == stream_params_pair.first->get_info().format.order && stream_params_pair.first->get_info().nms_info.is_defused) && + (outputs_params.end() != outputs_params.find(stream_params_pair.first->get_info().nms_info.defuse_info.original_name))) { + auto original_name = stream_params_pair.first->get_info().nms_info.defuse_info.original_name; + nms_output_streams.emplace(original_name, std::pair( + OutputStreamPtrVector(), outputs_params.at(original_name))); + nms_output_streams[original_name].first.push_back(stream_params_pair.first); + } else if (post_process_stream_inputs.count(stream_params_pair.first->get_info().name)) { + for (auto &op : m_net_flow_ops) { + if (op->input_streams.count(stream_params_pair.first->get_info().name)) { + assert(op->op->outputs_metadata().size() == 1); + nms_op_output_streams.emplace(op->name, std::pair( + OutputStreamPtrVector(), outputs_params.at(op->op->outputs_metadata().begin()->first))); + nms_op_output_streams[op->name].first.push_back(stream_params_pair.first); + } + } + } else { + auto outputs = VStreamsBuilderUtils::create_outputs(stream_params_pair.first, stream_params_pair.second, output_vstream_infos_map); + CHECK_EXPECTED(outputs); + vstreams.insert(vstreams.end(), std::make_move_iterator(outputs->begin()), std::make_move_iterator(outputs->end())); + } + } + for (auto &nms_output_stream_pair : nms_output_streams) { + auto outputs = VStreamsBuilderUtils::create_output_nms(nms_output_stream_pair.second.first, nms_output_stream_pair.second.second, + output_vstream_infos_map); + CHECK_EXPECTED(outputs); + vstreams.insert(vstreams.end(), std::make_move_iterator(outputs->begin()), std::make_move_iterator(outputs->end())); + } + for (auto &nms_output_stream_pair : nms_op_output_streams) { + auto op = post_process_nms_ops.at(nms_output_stream_pair.first); + auto outputs = VStreamsBuilderUtils::create_output_post_process_nms(nms_output_stream_pair.second.first, + nms_output_stream_pair.second.second, output_vstream_infos_map, + *op); + CHECK_EXPECTED(outputs); + vstreams.insert(vstreams.end(), std::make_move_iterator(outputs->begin()), std::make_move_iterator(outputs->end())); + } + + get_core_op()->set_vstreams_multiplexer_callbacks(vstreams); + + return vstreams; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/context_switch/network_group_internal.hpp b/hailort/libhailort/src/network_group/network_group_internal.hpp similarity index 67% rename from hailort/libhailort/src/context_switch/network_group_internal.hpp rename to hailort/libhailort/src/network_group/network_group_internal.hpp index cd2233f..11c5513 100644 --- a/hailort/libhailort/src/context_switch/network_group_internal.hpp +++ b/hailort/libhailort/src/network_group/network_group_internal.hpp @@ -4,29 +4,28 @@ **/ /** * @file network_group_internal.hpp - * @brief Class declaration for ConfiguredNetworkGroupBase and ActivatedNetworkGroupBase that implement the basic ConfiguredNetworkGroup + * @brief TODO: HRT-9547 - Change doc after moving NGs to global context + add explanation on lagacy names + * Class declaration for ConfiguredNetworkGroupBase and ActivatedCoreOp that implement the basic ConfiguredNetworkGroup * and ActivatedNetworkGroup interfaces. All internal classes that are relevant should inherit from the - * ConfiguredNetworkGroupBase and ActivatedNetworkGroupBase classes. + * ConfiguredNetworkGroupBase and ActivatedCoreOp classes. * Hence, the hierarchy is as follows: * -------------------------------------------------------------------------------------------------------------- * | ConfiguredNetworkGroup | (External "interface") - * | ________________________________|________________________________ | - * | / \ | - * | ConfiguredNetworkGroupBase ConfiguredNetworkGroupClient | (Base classes) - * | / | \ | - * | VdmaConfigNetworkGroup | HcpConfigNetworkGroup | (Actual implementations) - * | VDeviceNetworkGroup | - * | | | - * | vector of VdmaConfigNetworkGroup | + * | ________________________________|___________________________ | + * | / \ | + * | ConfiguredNetworkGroupBase ConfiguredNetworkGroupClient | (Base classes) + * | | | + * | | | + * | vector of CoreOps | (Actual implementations) * -------------------------------------------------------------------------------------------------------------| * | ActivatedNetworkGroup | (External "interface") * | | | - * | ActivatedNetworkGroupBase | (Base classes) + * | ActivatedCoreOp | (Base classes) * | __________________|_____________________________________________________ | * | / | \ | - * | VdmaConfigActivatedNetworkGroup VDeviceActivatedNetworkGroup HcpConfigActivatedNetworkGroup | (Actual implementations) + * | VdmaConfigActivatedCoreOp VDeviceActivatedCoreOp HcpConfigActivatedCoreOp | (Actual implementations) * | | | - * | vector of VdmaConfigActivatedNetworkGroup | + * | vector of VdmaConfigActivatedCoreOp | * -------------------------------------------------------------------------------------------------------------- **/ @@ -35,55 +34,38 @@ #include "hailo/hailort.h" #include "hailo/network_group.hpp" -#include "hef_internal.hpp" + #include "common/latency_meter.hpp" + +#include "hef/hef_internal.hpp" +#include "vdma/channel/boundary_channel.hpp" +#include "core_op/active_core_op_holder.hpp" +#include "core_op/core_op.hpp" + #include "control_protocol.h" -#include "vdma_channel.hpp" -#include "context_switch/active_network_group_holder.hpp" #ifdef HAILO_SUPPORT_MULTI_PROCESS -#include "hailort_rpc_client.hpp" +#include "service/hailort_rpc_client.hpp" #endif // HAILO_SUPPORT_MULTI_PROCESS + namespace hailort { -/** Represents a vector of InputStream ptrs */ -using InputStreamPtrVector = std::vector>; - -/** Represents a vector of OutputStream ptrs */ -using OutputStreamPtrVector = std::vector>; - -class ActivatedNetworkGroupBase : public ActivatedNetworkGroup +class ConfiguredNetworkGroupBase : public ConfiguredNetworkGroup { public: - virtual ~ActivatedNetworkGroupBase() = default; - ActivatedNetworkGroupBase(const ActivatedNetworkGroupBase &other) = delete; - ActivatedNetworkGroupBase &operator=(const ActivatedNetworkGroupBase &other) = delete; - ActivatedNetworkGroupBase &operator=(ActivatedNetworkGroupBase &&other) = delete; - ActivatedNetworkGroupBase(ActivatedNetworkGroupBase &&other) = default; - - virtual uint32_t get_invalid_frames_count() override; - -protected: - hailo_activate_network_group_params_t m_network_group_params; - - ActivatedNetworkGroupBase(const hailo_activate_network_group_params_t &network_group_params, - std::map> &input_streams, - std::map> &output_streams, - EventPtr &&network_group_activated_event, hailo_status &status); - - EventPtr m_network_group_activated_event; - std::map> &m_input_streams; - std::map> &m_output_streams; + static Expected> create(const ConfigureNetworkParams &config_params, + std::vector> &&core_ops, std::vector> &&net_flow_ops) + { + auto net_group_ptr = std::shared_ptr(new (std::nothrow) + ConfiguredNetworkGroupBase(config_params, std::move(core_ops), std::move(net_flow_ops))); + // auto net_group_ptr = make_shared_nothrow(config_params, std::move(core_ops), std::move(net_flow_ops)); + CHECK_NOT_NULL_AS_EXPECTED(net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); -private: - hailo_status validate_network_group_params(const hailo_activate_network_group_params_t &network_group_params); -}; + return net_group_ptr; + } -class ConfiguredNetworkGroupBase : public ConfiguredNetworkGroup -{ -public: virtual ~ConfiguredNetworkGroupBase() = default; ConfiguredNetworkGroupBase(const ConfiguredNetworkGroupBase &other) = delete; ConfiguredNetworkGroupBase &operator=(const ConfiguredNetworkGroupBase &other) = delete; @@ -91,8 +73,10 @@ public: ConfiguredNetworkGroupBase(ConfiguredNetworkGroupBase &&other) = default; Expected> activate_with_batch( - uint16_t dynamic_batch_size = CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE); - virtual Expected> activate(const hailo_activate_network_group_params_t &network_group_params) override; + uint16_t dynamic_batch_size = CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE, + bool resume_pending_stream_transfers = false); + virtual Expected> activate( + const hailo_activate_network_group_params_t &network_group_params) override; virtual hailo_status wait_for_activation(const std::chrono::milliseconds &timeout) override; virtual const std::string &get_network_group_name() const override; @@ -122,9 +106,6 @@ public: virtual Expected>> get_output_vstream_groups() override; - virtual hailo_status activate_impl(uint16_t dynamic_batch_size) = 0; - virtual hailo_status deactivate_impl() = 0; - virtual Expected> get_network_infos() const override; virtual Expected> get_all_stream_infos(const std::string &network_name="") const override; virtual Expected> get_input_vstream_infos(const std::string &network_name="") const override; @@ -137,34 +118,99 @@ public: virtual bool is_multi_context() const override; virtual const ConfigureNetworkParams get_config_params() const override; - static Expected create_hw_latency_meter(Device &device, - const std::vector &layers); + // TODO: HRT-9551 - Change to get_core_op_by_name() when multiple core_ops supported + std::shared_ptr get_core_op() const; + // TODO: HRT-9546 Remove + const std::shared_ptr get_core_op_metadata() const; + + Expected> get_vstream_names_from_stream_name(const std::string &stream_name); + const SupportedFeatures &get_supported_features(); + + Expected get_stream_batch_size(const std::string &stream_name); + + virtual Expected> create_input_vstreams(const std::map &inputs_params) override; + virtual Expected> create_output_vstreams(const std::map &outputs_params) override; - Expected> get_vstream_names_from_stream_name(const std::string &stream_name) + Expected> get_shared_input_stream_by_name(const std::string &stream_name) { - return m_network_group_metadata.get_vstream_names_from_stream_name(stream_name); + return get_core_op()->get_shared_input_stream_by_name(stream_name); + } + + Expected> get_shared_output_stream_by_name(const std::string &stream_name) + { + return get_core_op()->get_shared_output_stream_by_name(stream_name); } - const SupportedFeatures &get_supported_features() + EventPtr get_core_op_activated_event() { - return m_network_group_metadata.supported_features(); + return get_core_op()->m_core_op_activated_event; } - Expected get_stream_batch_size(const std::string &stream_name); + hailo_status activate_impl(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers = false) + { + return get_core_op()->activate_impl(dynamic_batch_size, resume_pending_stream_transfers); + } - virtual Expected> create_input_vstreams(const std::map &inputs_params); - virtual Expected> create_output_vstreams(const std::map &outputs_params); + hailo_status deactivate_impl(bool keep_nn_config_during_reset) + { + return get_core_op()->deactivate_impl(keep_nn_config_during_reset); + } - std::map> m_input_streams; - std::map> m_output_streams; + Expected> create_activated_network_group( + const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) + { + return get_core_op()->create_activated_network_group(network_group_params, dynamic_batch_size, resume_pending_stream_transfers); + } -protected: - ConfiguredNetworkGroupBase(const ConfigureNetworkParams &config_params, - const NetworkGroupMetadata &network_group_metadata, std::vector> &&net_flow_ops, hailo_status &status); + Expected> get_latency_meters() + { + return get_core_op()->get_latency_meters(); + } + + Expected get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) + { + return get_core_op()->get_boundary_vdma_channel_by_stream_name(stream_name); + } + + virtual bool is_scheduled() const override + { + return get_core_op()->is_scheduled(); + } + + virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) override + { + return get_core_op()->set_scheduler_timeout(timeout, network_name); + } + + virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) override + { + return get_core_op()->set_scheduler_threshold(threshold, network_name); + } + + virtual Expected get_default_streams_interface() override + { + return get_core_op()->get_default_streams_interface(); + } + + virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name) override + { + return get_core_op()->set_scheduler_priority(priority, network_name); + } - virtual Expected> create_activated_network_group( - const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size) = 0; + std::vector> &get_core_ops() + { + return m_core_ops; + } + +private: + ConfiguredNetworkGroupBase(const ConfigureNetworkParams &config_params, + std::vector> &&core_ops, std::vector> &&net_flow_ops); + static uint16_t get_smallest_configured_batch_size(const ConfigureNetworkParams &config_params); + hailo_status create_vdma_input_stream(Device &device, const std::string &stream_name, + const LayerInfo &layer_info, const hailo_stream_parameters_t &stream_params); + hailo_status create_vdma_output_stream(Device &device, const std::string &stream_name, + const LayerInfo &layer_info, const hailo_stream_parameters_t &stream_params); hailo_status create_output_stream_from_config_params(Device &device, const hailo_stream_parameters_t &stream_params, const std::string &stream_name); hailo_status create_input_stream_from_config_params(Device &device, @@ -172,32 +218,20 @@ protected: hailo_status add_mux_streams_by_edges_names(OutputStreamWithParamsVector &result, const std::unordered_map &outputs_edges_params); Expected get_output_streams_by_vstream_name(const std::string &name); - - hailo_status activate_low_level_streams(uint16_t dynamic_batch_size); - hailo_status deactivate_low_level_streams(); - Expected get_layer_info(const std::string &stream_name); - virtual Expected> get_latency_meters() = 0; - virtual Expected> get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) = 0; + hailo_status activate_low_level_streams(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers); + hailo_status deactivate_low_level_streams(); const ConfigureNetworkParams m_config_params; - const uint16_t m_min_configured_batch_size; // TODO: remove after HRT-6535 - EventPtr m_network_group_activated_event; - const NetworkGroupMetadata m_network_group_metadata; - AccumulatorPtr m_activation_time_accumulator; - AccumulatorPtr m_deactivation_time_accumulator; - -private: - friend class VDeviceNetworkGroup; - - static uint16_t get_smallest_configured_batch_size(const ConfigureNetworkParams &config_params); - + std::vector> m_core_ops; std::vector> m_net_flow_ops; -}; -using ActiveNetGroupHolder = ActiveNetworkGroupHolder; + friend class VDeviceCoreOp; + friend class VDeviceActivatedCoreOp; +}; +// Move client ng to different header #ifdef HAILO_SUPPORT_MULTI_PROCESS class ConfiguredNetworkGroupClient : public ConfiguredNetworkGroup { @@ -244,8 +278,10 @@ public: virtual Expected> get_output_vstream_infos(const std::string &network_name="") const override; virtual Expected> get_all_vstream_infos(const std::string &network_name="") const override; + virtual bool is_scheduled() const override; virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) override; virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) override; + virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name) override; virtual AccumulatorPtr get_activation_time_accumulator() const override; virtual AccumulatorPtr get_deactivation_time_accumulator() const override; @@ -256,7 +292,13 @@ public: virtual Expected> create_input_vstreams(const std::map &inputs_params); virtual Expected> create_output_vstreams(const std::map &outputs_params); + virtual hailo_status before_fork() override; + virtual hailo_status after_fork_in_parent() override; + virtual hailo_status after_fork_in_child() override; + private: + hailo_status create_client(); + std::unique_ptr m_client; uint32_t m_handle; std::string m_network_group_name; diff --git a/hailort/libhailort/src/network_group_scheduler.cpp b/hailort/libhailort/src/network_group_scheduler.cpp deleted file mode 100644 index b4a0527..0000000 --- a/hailort/libhailort/src/network_group_scheduler.cpp +++ /dev/null @@ -1,924 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file network_group_scheduler.cpp - * @brief: Network scheduler - **/ - -#include "network_group_scheduler.hpp" -#include "context_switch/network_group_internal.hpp" -#include "context_switch/vdevice_network_group.hpp" -#include "hef_internal.hpp" -#include "vdevice_stream_multiplexer_wrapper.hpp" -#include "tracer_macros.hpp" -#include "scheduler_oracle.hpp" - -#include - -namespace hailort -{ - -#define SINGLE_CONTEXT_BATCH_SIZE (1) - -// TODO: use device handles instead device count -NetworkGroupScheduler::NetworkGroupScheduler(hailo_scheduling_algorithm_t algorithm, uint32_t device_count) : - m_changing_current_batch_size(), - m_should_ng_stop(), - m_algorithm(algorithm), - m_before_read_write_mutex(), - m_write_read_cv(), - m_should_monitor(false) -#if defined(__GNUC__) - , m_mon_tmp_output() -#endif -{ - for (uint32_t i = 0; i < device_count; i++) { - m_devices.push_back(make_shared_nothrow(i)); - } - - // TODO: HRT-7391 - Change scheduler monitor to work only when MON command is active - m_should_monitor = SchedulerMon::should_monitor(); - if (m_should_monitor) { - auto status = start_mon(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to initiate hailo monitor of networks, with status {}", status); - } - } -} - -NetworkGroupScheduler::~NetworkGroupScheduler() -{ - for (auto device_info : m_devices) { - if (INVALID_NETWORK_GROUP_HANDLE != device_info->current_network_group_handle) { - auto current_ng = m_cngs[device_info->current_network_group_handle]->get_network_group(); - auto current_network_group_bundle = std::dynamic_pointer_cast(current_ng); - assert(nullptr != current_network_group_bundle); - auto vdma_network_group = current_network_group_bundle->get_network_group_by_device_index(device_info->device_id); - if (!vdma_network_group) { - LOGGER__ERROR("Error retrieving network group in scheduler destructor"); - } else { - if (HAILO_SUCCESS != VdmaConfigManager::switch_network_group(vdma_network_group.value(), nullptr, 0)) { - LOGGER__ERROR("Error deactivating network group when destroying scheduler"); - } - } - } - } - - if (m_should_monitor) { - m_should_monitor = false; - m_mon_shutdown_event->signal(); - if (m_mon_thread.joinable()) { - m_mon_thread.join(); - } - } -} - -Expected NetworkGroupScheduler::create_round_robin(uint32_t device_count) -{ - auto ptr = make_shared_nothrow(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN, device_count); - CHECK_AS_EXPECTED(nullptr != ptr, HAILO_OUT_OF_HOST_MEMORY); - - return ptr; -} - -std::string get_curr_pid_as_str() -{ -#ifdef _WIN32 - auto pid = GetCurrentProcessId(); -#else - auto pid = getpid(); -#endif - return std::to_string(pid); -} - -hailo_status NetworkGroupScheduler::start_mon() -{ -#if defined(__GNUC__) - m_last_measured_timestamp = std::chrono::steady_clock::now(); - m_mon_shutdown_event = Event::create_shared(Event::State::not_signalled); - CHECK(nullptr != m_mon_shutdown_event, HAILO_OUT_OF_HOST_MEMORY); - - auto tmp_file = open_temp_mon_file(); - CHECK_EXPECTED_AS_STATUS(tmp_file); - m_mon_tmp_output = tmp_file.release(); - - m_mon_thread = std::thread([this] () - { - while (m_should_monitor) { - auto status = m_mon_shutdown_event->wait(DEFAULT_SCHEDULER_MON_INTERVAL); - if (HAILO_TIMEOUT == status) { - dump_state(); - } else if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Scheduler monitor failed with status {}", status); - return; - } - } - return; - }); - - return HAILO_SUCCESS; -#else - return HAILO_NOT_IMPLEMENTED; -#endif -} - -#if defined(__GNUC__) -Expected> NetworkGroupScheduler::open_temp_mon_file() -{ - std::string file_name = get_curr_pid_as_str(); - auto tmp_file = TempFile::create(file_name, SCHEDULER_MON_TMP_DIR); - CHECK_EXPECTED(tmp_file); - - auto tmp_file_ptr = make_shared_nothrow(tmp_file.release()); - CHECK_AS_EXPECTED(nullptr != tmp_file_ptr, HAILO_OUT_OF_HOST_MEMORY); - - return tmp_file_ptr; -} - -void NetworkGroupScheduler::dump_state() -{ - auto file = LockedFile::create(m_mon_tmp_output->name(), "w"); - if (HAILO_SUCCESS != file.status()) { - LOGGER__ERROR("Failed to open and lock file {}, with status: {}", m_mon_tmp_output->name(), file.status()); - return; - } - - ProtoMon mon; - mon.set_pid(get_curr_pid_as_str()); - log_monitor_networks_infos(mon); - log_monitor_frames_infos(mon); - - // Clear accumulators - for (auto &handle_duration_pair : m_active_duration) { - handle_duration_pair.second = 0; - } - for (auto &handle_fps_pair : m_fps_accumulator) { - handle_fps_pair.second = 0; - } - - if (!mon.SerializeToFileDescriptor(file->get_fd())) { - LOGGER__ERROR("Failed to SerializeToFileDescriptor(), with errno: {}", errno); - } -} -#endif - -std::string NetworkGroupScheduler::get_network_group_name(const scheduler_ng_handle_t &network_group_handle) -{ - return m_cngs[network_group_handle]->get_network_group_name(); -} - -// TODO: HRT-7392 - Reduce core percentage when scheduler is idle -void NetworkGroupScheduler::log_monitor_networks_infos(ProtoMon &mon) -{ - auto curr_time = std::chrono::steady_clock::now(); - const auto measurement_duration = std::chrono::duration_cast>(curr_time - m_last_measured_timestamp).count(); - - for (uint32_t network_group_handle = 0; network_group_handle < m_last_measured_activation_timestamp.size(); network_group_handle++) { - assert(contains(m_active_duration, network_group_handle)); - auto curr_ng_active_time = m_active_duration[network_group_handle]; - - for (auto device_info : m_devices) { - if (network_group_handle == device_info->current_network_group_handle) { - // Network is currently active - auto time_diff = std::chrono::duration_cast>( - curr_time - m_last_measured_activation_timestamp[device_info->current_network_group_handle]).count(); - curr_ng_active_time += time_diff; - m_last_measured_activation_timestamp[device_info->current_network_group_handle] = curr_time; - } - } - - auto active_time = ((curr_ng_active_time * 100) / measurement_duration); - auto outputs_count = static_cast(m_cngs[network_group_handle]->get_outputs_names().size()); - auto fps = static_cast((m_fps_accumulator[network_group_handle] / outputs_count) / measurement_duration); - - auto net_info = mon.add_networks_infos(); - net_info->set_network_name(get_network_group_name(network_group_handle)); - net_info->set_active_time(active_time); - net_info->set_fps(fps); - } - - m_last_measured_timestamp = curr_time; -} - -void NetworkGroupScheduler::log_monitor_frames_infos(ProtoMon &mon) -{ - for (uint32_t network_group_handle = 0; network_group_handle < m_cngs.size(); network_group_handle++) { - auto net_frames_info = mon.add_net_frames_infos(); - net_frames_info->set_network_name(get_network_group_name(network_group_handle)); - - for (auto &stream_name : m_cngs[network_group_handle]->get_inputs_names()) { - auto stream_frames_info = net_frames_info->add_streams_frames_infos(); - stream_frames_info->set_stream_name(stream_name); - stream_frames_info->set_stream_direction(PROTO__STREAM_DIRECTION__HOST_TO_DEVICE); - auto status = set_h2d_frames_counters(network_group_handle, stream_name, *stream_frames_info); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to set stream's {} frames count, status = {}", stream_name, status); - continue; - } - } - - for (auto &stream_name : m_cngs[network_group_handle]->get_outputs_names()) { - auto stream_frames_info = net_frames_info->add_streams_frames_infos(); - stream_frames_info->set_stream_name(stream_name); - stream_frames_info->set_stream_direction(PROTO__STREAM_DIRECTION__DEVICE_TO_HOST); - auto status = set_d2h_frames_counters(network_group_handle, stream_name, *stream_frames_info); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to set stream's {} frames count, status = {}", stream_name, status); - continue; - } - } - } -} - -hailo_status NetworkGroupScheduler::set_h2d_frames_counters(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - ProtoMonStreamFramesInfo &stream_frames_info) -{ - assert(m_cngs.size() > network_group_handle); - auto current_cng = m_cngs[network_group_handle]->get_network_group(); - - auto input_stream = current_cng->get_input_stream_by_name(stream_name); - CHECK_EXPECTED_AS_STATUS(input_stream); - - InputStreamBase &vdevice_input = static_cast(input_stream->get()); - auto buffer_frames_size = vdevice_input.get_buffer_frames_size(); - if (HAILO_SUCCESS == buffer_frames_size.status()) { - stream_frames_info.set_buffer_frames_size(static_cast(buffer_frames_size.value())); - } else { - stream_frames_info.set_buffer_frames_size(SCHEDULER_MON_NAN_VAL); - } - - auto pending_frames_count = vdevice_input.get_pending_frames_count(); - if (HAILO_SUCCESS == pending_frames_count.status()) { - stream_frames_info.set_pending_frames_count(static_cast(pending_frames_count.value())); - } else { - stream_frames_info.set_pending_frames_count(SCHEDULER_MON_NAN_VAL); - } - - return HAILO_SUCCESS; -} - -hailo_status NetworkGroupScheduler::set_d2h_frames_counters(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - ProtoMonStreamFramesInfo &stream_frames_info) -{ - assert(m_cngs.size() > network_group_handle); - auto current_cng = m_cngs[network_group_handle]->get_network_group(); - - auto output_stream = current_cng->get_output_stream_by_name(stream_name); - CHECK_EXPECTED_AS_STATUS(output_stream); - - OutputStreamBase &vdevice_output = static_cast(output_stream->get()); - auto buffer_frames_size = vdevice_output.get_buffer_frames_size(); - if (HAILO_SUCCESS == buffer_frames_size.status()) { - stream_frames_info.set_buffer_frames_size(static_cast(buffer_frames_size.value())); - } else { - stream_frames_info.set_buffer_frames_size(SCHEDULER_MON_NAN_VAL); - } - - auto pending_frames_count = vdevice_output.get_pending_frames_count(); - if (HAILO_SUCCESS == pending_frames_count.status()) { - stream_frames_info.set_pending_frames_count(static_cast(pending_frames_count.value())); - } else { - stream_frames_info.set_pending_frames_count(SCHEDULER_MON_NAN_VAL); - } - - return HAILO_SUCCESS; -} - -Expected NetworkGroupScheduler::add_network_group(std::shared_ptr added_cng) -{ - scheduler_ng_handle_t network_group_handle = INVALID_NETWORK_GROUP_HANDLE; - { - std::unique_lock lock(m_before_read_write_mutex); - - network_group_handle = static_cast(m_cngs.size()); - TRACE(AddNetworkGroupTrace, "", added_cng->name(), DEFAULT_SCHEDULER_TIMEOUT.count(), DEFAULT_SCHEDULER_MIN_THRESHOLD, network_group_handle); - - auto stream_infos = added_cng->get_all_stream_infos(); - CHECK_EXPECTED(stream_infos); - - auto scheduled_ng = ScheduledNetworkGroup::create(added_cng, stream_infos.value()); - CHECK_EXPECTED(scheduled_ng); - - m_cngs.emplace_back(scheduled_ng.release()); - - m_changing_current_batch_size[network_group_handle] = false; - - for (const auto &stream_info : stream_infos.value()) { - m_should_ng_stop[network_group_handle][stream_info.name] = false; - } - - for (auto& device_info : m_devices) { - for (const auto &stream_info : stream_infos.value()) { - if (HAILO_H2D_STREAM == stream_info.direction) { - device_info->current_cycle_requested_transferred_frames_h2d[network_group_handle][stream_info.name] = 0; - } else { - device_info->current_cycle_finished_transferred_frames_d2h[network_group_handle][stream_info.name] = 0; - device_info->current_cycle_finished_read_frames_d2h[network_group_handle][stream_info.name] = 0; - } - } - } - - // Monitor members - m_last_measured_activation_timestamp[network_group_handle] = {}; - m_active_duration[network_group_handle] = 0; - m_fps_accumulator[network_group_handle] = 0; - } - m_write_read_cv.notify_all(); - return network_group_handle; -} - -bool NetworkGroupScheduler::is_network_group_active(const scheduler_ng_handle_t &network_group_handle) -{ - for (auto device_info : m_devices) { - if (network_group_handle == device_info->current_network_group_handle) { - return true; - } - } - - return false; -} - -bool NetworkGroupScheduler::is_switching_current_network_group(const scheduler_ng_handle_t &network_group_handle) -{ - for (auto device_info : m_devices) { - if (network_group_handle == device_info->current_network_group_handle && device_info->is_switching_network_group) { - return true; - } - } - - return false; -} - -bool NetworkGroupScheduler::is_multi_device() -{ - return m_devices.size() > 1; -} - -hailo_status NetworkGroupScheduler::wait_for_write(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - const std::chrono::milliseconds &timeout, const std::function &should_cancel) -{ - { - std::unique_lock lock(m_before_read_write_mutex); - - hailo_status status = HAILO_SUCCESS; - auto wait_res = m_write_read_cv.wait_for(lock, timeout, [this, network_group_handle, stream_name, &should_cancel, &status] { - - if (should_cancel()) { - status = HAILO_STREAM_ABORTED_BY_USER; - return true; // return true so that the wait will finish - } - - if (should_ng_stop(network_group_handle)) { - status = HAILO_STREAM_ABORTED_BY_USER; - return true; // return true so that the wait will finish - } - - auto should_wait = should_wait_for_write(network_group_handle, stream_name); - if (HAILO_SUCCESS != should_wait.status()) { - status = should_wait.status(); - return true; // return true so that the wait will finish - } - return !should_wait.value(); - }); - CHECK(wait_res, HAILO_TIMEOUT, "{} (H2D) failed with status={}, timeout={}ms", stream_name, HAILO_TIMEOUT, timeout.count()); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - return status; - } - CHECK_SUCCESS(status); - - m_cngs[network_group_handle]->mark_frame_sent(); - m_cngs[network_group_handle]->requested_write_frames().increase(stream_name); - } - m_write_read_cv.notify_all(); - - return HAILO_SUCCESS; -} - -Expected NetworkGroupScheduler::should_wait_for_write(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name) -{ - auto scheduled_ng = m_cngs[network_group_handle]; - - if (should_ng_stop(network_group_handle)) { - return make_unexpected(HAILO_STREAM_ABORTED_BY_USER); - } - - auto pre_transfer_h2d_frames = scheduled_ng->requested_write_frames(stream_name) + scheduled_ng->finished_write_frames(stream_name); - bool has_written_max_batch_size = ((scheduled_ng->use_dynamic_batch_flow() || is_multi_device()) && - ((scheduled_ng->get_max_batch_size() * m_devices.size()) == pre_transfer_h2d_frames)); - - bool should_stop_writing_because_switching = ((!(scheduled_ng->use_dynamic_batch_flow() || is_multi_device())) && - (is_switching_current_network_group(network_group_handle) || m_changing_current_batch_size[network_group_handle]) && - is_network_group_active(network_group_handle) && scheduled_ng->has_input_written_most_frames(stream_name)); - - auto total_written_frames = scheduled_ng->total_written_frames_count()[stream_name]; - auto min_finished_read = scheduled_ng->finished_read_frames_min_value(); - auto ongoing_frames = (min_finished_read < total_written_frames) ? (total_written_frames - min_finished_read) : 0; - bool has_enough_space_for_writes = scheduled_ng->has_enough_space_in_read_buffers(ongoing_frames); - - if (has_written_max_batch_size || should_stop_writing_because_switching || (!has_enough_space_for_writes)) { - return true; - } - - return false; -} - -hailo_status NetworkGroupScheduler::signal_write_finish(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name) -{ - { - std::unique_lock lock(m_before_read_write_mutex); - auto scheduled_ng = m_cngs[network_group_handle]; - - if (should_ng_stop(network_group_handle)) { - return HAILO_STREAM_ABORTED_BY_USER; - } - - scheduled_ng->finished_write_frames().increase(stream_name); - scheduled_ng->requested_write_frames().decrease(stream_name); - - auto device_id = NetworkGroupSchedulerOracle::get_avail_device(*this, network_group_handle); - if (INVALID_DEVICE_ID != device_id) { - auto status = switch_network_group(network_group_handle, device_id); - CHECK_SUCCESS(status); - } - - for (auto &device_info : m_devices) { - if (device_info->current_network_group_handle == network_group_handle && !(scheduled_ng->use_dynamic_batch_flow() || is_multi_device())) { - auto status = send_all_pending_buffers(network_group_handle, device_info->device_id); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("send_all_pending_buffers has failed with status=HAILO_STREAM_ABORTED_BY_USER"); - return status; - } - CHECK_SUCCESS(status); - } - } - } - m_write_read_cv.notify_all(); - - return HAILO_SUCCESS; -} - -hailo_status NetworkGroupScheduler::switch_network_group(const scheduler_ng_handle_t &network_group_handle, uint32_t device_id, bool /*keep_nn_config*/) -{ - auto scheduled_ng = m_cngs[network_group_handle]; - auto curr_device_info = m_devices[device_id]; - - // initialize current cycle maps - for (const auto &name : scheduled_ng->get_inputs_names()) { - curr_device_info->current_cycle_requested_transferred_frames_h2d[network_group_handle][name] = 0; - } - - for (const auto &name : scheduled_ng->get_outputs_names()) { - curr_device_info->current_cycle_finished_transferred_frames_d2h[network_group_handle][name] = 0; - curr_device_info->current_cycle_finished_read_frames_d2h[network_group_handle][name] = 0; - } - - uint16_t batch_size = CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE; - uint16_t burst_size = CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE; - if (scheduled_ng->use_dynamic_batch_flow()) { - burst_size = std::min(static_cast(scheduled_ng->finished_write_frames_min_value()), scheduled_ng->get_max_batch_size()); - batch_size = burst_size; - } else { - burst_size = is_multi_device() ? static_cast(scheduled_ng->finished_write_frames_min_value()) : SINGLE_CONTEXT_BATCH_SIZE; - batch_size = SINGLE_CONTEXT_BATCH_SIZE; - } - - bool has_same_batch_size_as_previous = (curr_device_info->current_batch_size == batch_size); - curr_device_info->current_batch_size = batch_size; - curr_device_info->current_burst_size = burst_size; - - m_last_measured_activation_timestamp[network_group_handle] = std::chrono::steady_clock::now(); - - if (curr_device_info->current_network_group_handle != network_group_handle) { - curr_device_info->is_switching_network_group = false; - } - - if ((network_group_handle != curr_device_info->current_network_group_handle) || (!has_same_batch_size_as_previous)) { - assert(m_cngs.size() > network_group_handle); - auto next_active_cng = scheduled_ng->get_network_group(); - auto next_active_cng_wrapper = std::dynamic_pointer_cast(next_active_cng); - assert(nullptr != next_active_cng_wrapper); - auto next_active_cng_expected = next_active_cng_wrapper->get_network_group_by_device_index(curr_device_info->device_id); - CHECK_EXPECTED_AS_STATUS(next_active_cng_expected); - - std::shared_ptr current_active_vdma_cng = nullptr; - if (curr_device_info->current_network_group_handle != INVALID_NETWORK_GROUP_HANDLE) { - reset_current_ng_timestamps(curr_device_info->device_id); - auto current_active_cng = m_cngs[curr_device_info->current_network_group_handle]->get_network_group(); - auto current_active_cng_bundle = std::dynamic_pointer_cast(current_active_cng); - assert(nullptr != current_active_cng_bundle); - auto current_active_cng_expected = current_active_cng_bundle->get_network_group_by_device_index(curr_device_info->device_id); - CHECK_EXPECTED_AS_STATUS(current_active_cng_expected); - current_active_vdma_cng = current_active_cng_expected.release(); - } - - TRACE(SwitchNetworkGroupTrace, "", network_group_handle); - auto status = VdmaConfigManager::switch_network_group(current_active_vdma_cng, next_active_cng_expected.value(), batch_size); - CHECK_SUCCESS(status, "Failed switching network group"); - - // Register to get interrupts - has to be after network group is activated - for (auto &output_stream : next_active_cng_expected.value()->get_output_streams()) { - OutputStreamBase &vdevice_output = static_cast(output_stream.get()); - status = vdevice_output.register_for_d2h_interrupts( - [this, name = output_stream.get().name(), format = vdevice_output.get_layer_info().format.order, scheduled_ng, network_group_handle, device_id] - (uint32_t frames) { - { - std::unique_lock lock(m_before_read_write_mutex); - if (hailo_format_order_t::HAILO_FORMAT_ORDER_HAILO_NMS != format) { - TRACE(OutputVdmaEnqueueTrace, "", network_group_handle, name, frames); - // TODO: Remove d2h_finished_transferred_frames and use current_cycle_finished_transferred_frames_d2h instead - scheduled_ng->d2h_finished_transferred_frames(name) += frames; - m_devices[device_id]->current_cycle_finished_transferred_frames_d2h[network_group_handle][name] += frames; - } - if (!(is_multi_device() || scheduled_ng->use_dynamic_batch_flow()) || has_ng_drained_everything(network_group_handle, device_id)) { - choose_next_network_group(device_id); - } - } - m_write_read_cv.notify_all(); - }); - CHECK_SUCCESS(status); - } - } - - scheduled_ng->set_last_run_timestamp(std::chrono::steady_clock::now()); // Mark timestamp on activation - curr_device_info->current_network_group_handle = network_group_handle; - - // Finished switching batch size - m_changing_current_batch_size[network_group_handle] = false; - - auto status = send_all_pending_buffers(network_group_handle, device_id); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("send_all_pending_buffers has failed with status=HAILO_STREAM_ABORTED_BY_USER"); - return status; - } - CHECK_SUCCESS(status); - - return HAILO_SUCCESS; -} - -hailo_status NetworkGroupScheduler::send_all_pending_buffers(const scheduler_ng_handle_t &network_group_handle, uint32_t device_id) -{ - auto current_device_info = m_devices[device_id]; - if ((INVALID_NETWORK_GROUP_HANDLE == current_device_info->current_network_group_handle) || (current_device_info->current_network_group_handle != network_group_handle)) { - return HAILO_SUCCESS; - } - - auto scheduled_ng = m_cngs[network_group_handle]; - - while(true) { - auto finished_send = false; - for (const auto &name : scheduled_ng->get_inputs_names()) { - if ((scheduled_ng->finished_write_frames(name) == 0) || (((scheduled_ng->use_dynamic_batch_flow()) || (is_multi_device())) && - ((current_device_info->current_cycle_requested_transferred_frames_h2d[network_group_handle][name] == current_device_info->current_burst_size)))) { - finished_send = true; - break; - } - } - if (finished_send) { - break; - } - - for (const auto &name : scheduled_ng->get_inputs_names()) { - auto status = send_pending_buffer(network_group_handle, name, device_id); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("send_pending_buffer has failed with status=HAILO_STREAM_ABORTED_BY_USER"); - return status; - } - CHECK_SUCCESS(status); - } - scheduled_ng->push_device_index(device_id); - } - - return HAILO_SUCCESS; -} - -hailo_status NetworkGroupScheduler::send_pending_buffer(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - uint32_t device_id) -{ - assert(m_cngs.size() > network_group_handle); - auto scheduled_ng = m_cngs[network_group_handle]; - - auto current_cng = scheduled_ng->get_network_group(); - auto input_stream = current_cng->get_input_stream_by_name(stream_name); - CHECK_EXPECTED_AS_STATUS(input_stream); - - VDeviceInputStreamMultiplexerWrapper &vdevice_input = static_cast(input_stream->get()); - TRACE(InputVdmaEnqueueTrace, "", network_group_handle, stream_name); - auto status = vdevice_input.send_pending_buffer(device_id); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("send_pending_buffer has failed with status=HAILO_STREAM_ABORTED_BY_USER"); - return status; - } - CHECK_SUCCESS(status); - - scheduled_ng->h2d_requested_transferred_frames().increase(stream_name); - m_devices[device_id]->current_cycle_requested_transferred_frames_h2d[network_group_handle][stream_name]++; - scheduled_ng->finished_write_frames().decrease(stream_name); - - scheduled_ng->h2d_finished_transferred_frames().increase(stream_name); - scheduled_ng->h2d_requested_transferred_frames().decrease(stream_name); - - if (should_ng_stop(network_group_handle)) { - return HAILO_STREAM_ABORTED_BY_USER; - } - - return HAILO_SUCCESS; -} - -void NetworkGroupScheduler::reset_current_ng_timestamps(uint32_t device_id) -{ - auto curr_device_info = m_devices[device_id]; - if (INVALID_NETWORK_GROUP_HANDLE == curr_device_info->current_network_group_handle) { - return; - } - - m_cngs[curr_device_info->current_network_group_handle]->set_last_run_timestamp(std::chrono::steady_clock::now()); // Mark timestamp on de-activation - - const auto active_duration_sec = std::chrono::duration_cast>( - std::chrono::steady_clock::now() - m_last_measured_activation_timestamp[curr_device_info->current_network_group_handle]).count(); - - assert(contains(m_active_duration, curr_device_info->current_network_group_handle)); - m_active_duration[curr_device_info->current_network_group_handle] += active_duration_sec; -} - -NetworkGroupScheduler::ReadyInfo NetworkGroupScheduler::is_network_group_ready(const scheduler_ng_handle_t &network_group_handle, bool check_threshold, uint32_t device_id) -{ - ReadyInfo result; - result.is_ready = false; - - if (should_ng_stop(network_group_handle)) { - // Do not switch to an aborted network group - return result; - } - - auto scheduled_ng = m_cngs[network_group_handle]; - // Check if there arent any write requests - bool has_pending_writes = scheduled_ng->finished_write_frames_min_value() > 0; - - // Check if there arent any read requests - bool has_pending_user_reads = false; - for (const auto &name : scheduled_ng->get_outputs_names()) { - if (scheduled_ng->requested_read_frames(name) > 0) { - has_pending_user_reads = true; - break; - } - } - - std::vector over_threshold; - over_threshold.reserve(scheduled_ng->get_inputs_names().size()); - std::vector over_timeout; - over_timeout.reserve(scheduled_ng->get_inputs_names().size()); - - if (check_threshold) { - for (const auto &name : scheduled_ng->get_inputs_names()) { - auto threshold_exp = scheduled_ng->get_threshold(name); - if (!threshold_exp) { - LOGGER__ERROR("Failed to get threshold for stream {}", name); - return result; - } - auto threshold = (DEFAULT_SCHEDULER_MIN_THRESHOLD == threshold_exp.value()) ? 1 : threshold_exp.value(); - auto timeout_exp = scheduled_ng->get_timeout(); - if (!timeout_exp) { - LOGGER__ERROR("Failed to get timeout for stream {}", name); - return result; - } - auto timeout = timeout_exp.release(); - - // Check if there arent enough write requests to reach threshold and timeout didnt passed - auto write_requests = scheduled_ng->requested_write_frames(name) + scheduled_ng->finished_write_frames(name); - auto stream_over_threshold = write_requests >= threshold; - auto stream_over_timeout = timeout <= (std::chrono::steady_clock::now() - scheduled_ng->get_last_run_timestamp()); - over_threshold.push_back(stream_over_threshold); - over_timeout.push_back(stream_over_timeout); - if (stream_over_threshold || stream_over_timeout) { - continue; - } else { - result.is_ready = false; - return result; - } - } - } - - auto has_pending_vdma_frames = get_max_value_of_unordered_map(m_devices[device_id]->current_cycle_requested_transferred_frames_h2d[network_group_handle]) != - get_min_value_of_unordered_map(m_devices[device_id]->current_cycle_finished_read_frames_d2h[network_group_handle]); - - result.threshold = std::all_of(over_threshold.begin(), over_threshold.end(), [](auto over) { return over; }); - result.timeout = std::all_of(over_timeout.begin(), over_timeout.end(), [](auto over) { return over; }); - result.is_ready = has_pending_writes && has_pending_user_reads && (!has_pending_vdma_frames); - - return result; -} - -Expected NetworkGroupScheduler::wait_for_read(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - const std::chrono::milliseconds &timeout) -{ - uint32_t device_id = INVALID_DEVICE_ID; - { - std::unique_lock lock(m_before_read_write_mutex); - - auto scheduled_ng = m_cngs[network_group_handle]; - - scheduled_ng->requested_read_frames().increase(stream_name); - - hailo_status status = HAILO_SUCCESS; - auto wait_res = m_write_read_cv.wait_for(lock, timeout, [this, network_group_handle, scheduled_ng, stream_name, &status] { - - if (should_ng_stop(network_group_handle)) { - status = HAILO_STREAM_ABORTED_BY_USER; - return true; // return true so that the wait will finish - } - - auto device_id = NetworkGroupSchedulerOracle::get_avail_device(*this, network_group_handle); - if (INVALID_DEVICE_ID != device_id) { - status = switch_network_group(network_group_handle, device_id); - if (HAILO_SUCCESS != status) { - return true; // return true so that the wait will finish - } - } - - return scheduled_ng->can_stream_read(stream_name); - }); - CHECK_AS_EXPECTED(wait_res, HAILO_TIMEOUT, "{} (D2H) failed with status={}, timeout={}ms", stream_name, HAILO_TIMEOUT, timeout.count()); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - return make_unexpected(status); - } - CHECK_SUCCESS_AS_EXPECTED(status); - - scheduled_ng->ongoing_read_frames().increase(stream_name); - scheduled_ng->requested_read_frames().decrease(stream_name); - device_id = scheduled_ng->pop_device_index(stream_name); - } - m_write_read_cv.notify_all(); - - return device_id; -} - - -hailo_status NetworkGroupScheduler::signal_read_finish(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, uint32_t device_id) -{ - { - std::unique_lock lock(m_before_read_write_mutex); - - auto scheduled_ng = m_cngs[network_group_handle]; - - scheduled_ng->finished_read_frames().increase(stream_name); - m_devices[device_id]->current_cycle_finished_read_frames_d2h[network_group_handle][stream_name]++; - scheduled_ng->d2h_finished_transferred_frames().decrease(stream_name); - scheduled_ng->ongoing_read_frames().decrease(stream_name); - m_fps_accumulator[network_group_handle]++; - - decrease_ng_counters(network_group_handle); - } - m_write_read_cv.notify_all(); - - return HAILO_SUCCESS; -} - - -bool NetworkGroupScheduler::has_ng_finished(const scheduler_ng_handle_t &network_group_handle, uint32_t device_id) -{ - if (INVALID_NETWORK_GROUP_HANDLE == network_group_handle) { - return true; // If no network group is running, consider it as finished - } - - auto scheduled_ng = m_cngs[network_group_handle]; - - if (scheduled_ng->use_dynamic_batch_flow() || is_multi_device()) { - for (const auto &name : scheduled_ng->get_outputs_names()) { - if (m_devices[device_id]->current_cycle_finished_read_frames_d2h[network_group_handle][name] < m_devices[device_id]->current_batch_size) { - return false; - } - } - - return true; - } - - uint32_t written_frames = get_max_value_of_unordered_map(scheduled_ng->total_written_frames_count()); - for (const auto &name : scheduled_ng->get_outputs_names()) { - if (scheduled_ng->finished_read_frames(name) < written_frames) { - return false; - } - } - return true; -} - -void NetworkGroupScheduler::decrease_ng_counters(const scheduler_ng_handle_t &network_group_handle) -{ - return m_cngs[network_group_handle]->decrease_current_ng_counters(); -} - -bool NetworkGroupScheduler::has_ng_drained_everything(const scheduler_ng_handle_t &network_group_handle, uint32_t device_id) -{ - if (INVALID_NETWORK_GROUP_HANDLE == network_group_handle) { - // If no network group is running, consider it as drained - return true; - } - - if (ng_all_streams_aborted(network_group_handle)) { - // We treat NG as drained only if all streams are aborted - to make sure there aren't any ongoing transfers - return true; - } - - if ((!m_cngs[network_group_handle]->is_nms()) && (is_multi_device() || m_cngs[network_group_handle]->use_dynamic_batch_flow())) { - auto current_device_info = m_devices[device_id]; - auto max_transferred_h2d = get_max_value_of_unordered_map(current_device_info->current_cycle_requested_transferred_frames_h2d[network_group_handle]); - auto min_transferred_d2h = get_min_value_of_unordered_map(current_device_info->current_cycle_finished_transferred_frames_d2h[network_group_handle]); - - return (max_transferred_h2d == min_transferred_d2h); - } - - return m_cngs[network_group_handle]->has_ng_drained_everything(!(m_cngs[network_group_handle]->use_dynamic_batch_flow() || is_multi_device())); -} - -hailo_status NetworkGroupScheduler::enable_stream(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name) -{ - { - std::unique_lock lock(m_before_read_write_mutex); - - if (!m_should_ng_stop[network_group_handle][stream_name]) { - return HAILO_SUCCESS; - } - - m_should_ng_stop[network_group_handle][stream_name] = false; - } - m_write_read_cv.notify_all(); - return HAILO_SUCCESS; -} - -hailo_status NetworkGroupScheduler::disable_stream(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name) -{ - { - std::unique_lock lock(m_before_read_write_mutex); - - if (m_should_ng_stop[network_group_handle][stream_name]) { - return HAILO_SUCCESS; - } - - m_should_ng_stop[network_group_handle][stream_name] = true; - } - m_write_read_cv.notify_all(); - return HAILO_SUCCESS; -} - -hailo_status NetworkGroupScheduler::set_timeout(const scheduler_ng_handle_t &network_group_handle, const std::chrono::milliseconds &timeout, const std::string &/*network_name*/) -{ - // TODO: call in loop for set_timeout with the relevant stream-names (of the given network) - return m_cngs[network_group_handle]->set_timeout(timeout); -} - -hailo_status NetworkGroupScheduler::set_threshold(const scheduler_ng_handle_t &network_group_handle, uint32_t threshold, const std::string &/*network_name*/) -{ - // TODO: call in loop for set_timeout with the relevant stream-names (of the given network) - return m_cngs[network_group_handle]->set_threshold(threshold); -} - -void NetworkGroupScheduler::choose_next_network_group(size_t device_id) -{ - if (!m_devices[device_id]->is_switching_network_group) { - NetworkGroupSchedulerOracle::choose_next_model(*this, m_devices[device_id]->device_id); - } -} - -bool NetworkGroupScheduler::should_ng_stop(const scheduler_ng_handle_t &network_group_handle) -{ - for (const auto &name_flag_pair : m_should_ng_stop[network_group_handle]) { - if (name_flag_pair.second) { - return true; - } - } - - return false; -} - -bool NetworkGroupScheduler::ng_all_streams_aborted(const scheduler_ng_handle_t &network_group_handle) -{ - for (const auto &name_flag_pair : m_should_ng_stop[network_group_handle]) { - if (!name_flag_pair.second) { - return false; - } - } - return true; -} - -void NetworkGroupScheduler::notify_all() -{ - { - // Acquire mutex to make sure the notify_all will wake the blocking threads on the cv - std::unique_lock lock(m_before_read_write_mutex); - } - m_write_read_cv.notify_all(); -} - -void NetworkGroupScheduler::mark_failed_write(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name) -{ - { - std::unique_lock lock(m_before_read_write_mutex); - assert(m_cngs.size() > network_group_handle); - m_cngs[network_group_handle]->requested_write_frames().decrease(stream_name); - } - m_write_read_cv.notify_all(); -} - - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/network_group_scheduler.hpp b/hailort/libhailort/src/network_group_scheduler.hpp deleted file mode 100644 index 9b6b2b7..0000000 --- a/hailort/libhailort/src/network_group_scheduler.hpp +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file network_group_scheduler.hpp - * @brief Class declaration for NetworkGroupScheduler that schedules network groups to be active depending on the scheduling algorithm. - **/ - -#ifndef _HAILO_NETWORK_GROUP_SCHEDULER_HPP_ -#define _HAILO_NETWORK_GROUP_SCHEDULER_HPP_ - -#include "hailo/hailort.h" -#include "hailo/expected.hpp" -#include "hailo/network_group.hpp" -#include "common/utils.hpp" -#include "common/filesystem.hpp" -#include "scheduler_mon.hpp" -#include "scheduled_network_group.hpp" - -#include - -#define DEFAULT_SCHEDULER_TIMEOUT (std::chrono::milliseconds(0)) -#define DEFAULT_SCHEDULER_MIN_THRESHOLD (0) - -namespace hailort -{ - -#define INVALID_NETWORK_GROUP_HANDLE (UINT32_MAX) -#define INVALID_DEVICE_ID (UINT32_MAX) - -using scheduler_ng_handle_t = uint32_t; - -class NetworkGroupScheduler; -using NetworkGroupSchedulerPtr = std::shared_ptr; - -// We use mostly weak pointer for the scheduler to prevent circular dependency of the pointers -using NetworkGroupSchedulerWeakPtr = std::weak_ptr; - -using stream_name_t = std::string; - -struct ActiveDeviceInfo { - ActiveDeviceInfo(uint32_t device_id) : current_network_group_handle(INVALID_NETWORK_GROUP_HANDLE), - next_network_group_handle(INVALID_NETWORK_GROUP_HANDLE), is_switching_network_group(false), current_batch_size(0), current_burst_size(0), - current_cycle_requested_transferred_frames_h2d(), current_cycle_finished_transferred_frames_d2h(), current_cycle_finished_read_frames_d2h(), - device_id(device_id) - {} - scheduler_ng_handle_t current_network_group_handle; - scheduler_ng_handle_t next_network_group_handle; - std::atomic_bool is_switching_network_group; - std::atomic_uint32_t current_batch_size; - std::atomic_uint32_t current_burst_size; - std::unordered_map> current_cycle_requested_transferred_frames_h2d; - std::unordered_map> current_cycle_finished_transferred_frames_d2h; - std::unordered_map> current_cycle_finished_read_frames_d2h; - uint32_t device_id; -}; - -class NetworkGroupScheduler -{ -public: - static Expected create_round_robin(uint32_t device_count); - NetworkGroupScheduler(hailo_scheduling_algorithm_t algorithm, uint32_t device_count); - - virtual ~NetworkGroupScheduler(); - NetworkGroupScheduler(const NetworkGroupScheduler &other) = delete; - NetworkGroupScheduler &operator=(const NetworkGroupScheduler &other) = delete; - NetworkGroupScheduler &operator=(NetworkGroupScheduler &&other) = delete; - NetworkGroupScheduler(NetworkGroupScheduler &&other) noexcept = delete; - - hailo_scheduling_algorithm_t algorithm() - { - return m_algorithm; - } - - Expected add_network_group(std::shared_ptr added_cng); - - hailo_status wait_for_write(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - const std::chrono::milliseconds &timeout, const std::function &should_cancel); - hailo_status signal_write_finish(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name); - Expected wait_for_read(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - const std::chrono::milliseconds &timeout); - hailo_status signal_read_finish(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, uint32_t device_id); - - hailo_status enable_stream(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name); - hailo_status disable_stream(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name); - - hailo_status set_timeout(const scheduler_ng_handle_t &network_group_handle, const std::chrono::milliseconds &timeout, const std::string &network_name); - hailo_status set_threshold(const scheduler_ng_handle_t &network_group_handle, uint32_t threshold, const std::string &network_name); - - void notify_all(); - void mark_failed_write(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name); - -protected: - struct ReadyInfo { - bool threshold = false; - bool timeout = false; - bool is_ready = false; - }; - - void choose_next_network_group(size_t device_id); - ReadyInfo is_network_group_ready(const scheduler_ng_handle_t &network_group_handle, bool check_threshold, uint32_t device_id); - - std::vector> m_devices; - std::unordered_map m_changing_current_batch_size; - std::unordered_map> m_should_ng_stop; - - std::vector> m_cngs; - -private: - hailo_status switch_network_group(const scheduler_ng_handle_t &network_group_handle, uint32_t device_id, - bool keep_nn_config = false); - void reset_current_ng_timestamps(uint32_t device_id); - - Expected should_wait_for_write(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name); - hailo_status send_all_pending_buffers(const scheduler_ng_handle_t &network_group_handle, uint32_t device_id); - hailo_status send_pending_buffer(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, uint32_t device_id); - - void decrease_ng_counters(const scheduler_ng_handle_t &network_group_handle); - bool has_ng_drained_everything(const scheduler_ng_handle_t &network_group_handle, uint32_t device_id); - bool has_ng_finished(const scheduler_ng_handle_t &network_group_handle, uint32_t device_id); - bool should_ng_stop(const scheduler_ng_handle_t &network_group_handle); - bool ng_all_streams_aborted(const scheduler_ng_handle_t &network_group_handle); - - std::string get_network_group_name(const scheduler_ng_handle_t &network_group_handle); - bool is_network_group_active(const scheduler_ng_handle_t &network_group_handle); - bool is_switching_current_network_group(const scheduler_ng_handle_t &network_group_handle); - bool is_multi_device(); - - hailo_status start_mon(); - void log_monitor_networks_infos(ProtoMon &mon); - void log_monitor_frames_infos(ProtoMon &mon); - hailo_status set_h2d_frames_counters(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - ProtoMonStreamFramesInfo &stream_frames_info); - hailo_status set_d2h_frames_counters(const scheduler_ng_handle_t &network_group_handle, const std::string &stream_name, - ProtoMonStreamFramesInfo &stream_frames_info); -#if defined(__GNUC__) - Expected> open_temp_mon_file(); - void dump_state(); -#endif - - hailo_scheduling_algorithm_t m_algorithm; - std::mutex m_before_read_write_mutex; - std::condition_variable m_write_read_cv; - scheduler_ng_handle_t m_last_choosen_network_group; - - // Params for the scheduler MON - std::atomic_bool m_should_monitor; - std::thread m_mon_thread; - EventPtr m_mon_shutdown_event; -#if defined(__GNUC__) - std::shared_ptr m_mon_tmp_output; -#endif - std::chrono::time_point m_last_measured_timestamp; - std::unordered_map> m_last_measured_activation_timestamp; - // TODO: Consider adding Accumulator classes for more info (min, max, mean, etc..) - std::unordered_map m_active_duration; - std::unordered_map m_fps_accumulator; - - friend class NetworkGroupSchedulerOracle; -}; - -} /* namespace hailort */ - -#endif /* _HAILO_NETWORK_GROUP_SCHEDULER_HPP_ */ diff --git a/hailort/libhailort/src/os/hailort_driver.hpp b/hailort/libhailort/src/os/hailort_driver.hpp index c55fc67..ff2b3ab 100755 --- a/hailort/libhailort/src/os/hailort_driver.hpp +++ b/hailort/libhailort/src/os/hailort_driver.hpp @@ -13,20 +13,23 @@ #include "hailo/hailort.h" #include "hailo/expected.hpp" -#include "os/file_descriptor.hpp" -#include "vdma/channel_id.hpp" + #include "common/utils.hpp" -#include "hailo_ioctl_common.h" + +#include "os/file_descriptor.hpp" +#include "vdma/channel/channel_id.hpp" #include #include #include #include +#include #ifdef __QNX__ #include #endif // __QNX__ + namespace hailort { @@ -46,6 +49,8 @@ static_assert((0 == ((PENDING_BUFFERS_SIZE - 1) & PENDING_BUFFERS_SIZE)), "PENDI #define PCIE_EXPECTED_MD5_LENGTH (16) constexpr size_t VDMA_CHANNELS_PER_ENGINE = 32; +constexpr size_t MAX_VDMA_ENGINES_COUNT = 3; +constexpr size_t MAX_VDMA_CHANNELS_COUNT = MAX_VDMA_ENGINES_COUNT * VDMA_CHANNELS_PER_ENGINE; constexpr uint8_t MIN_H2D_CHANNEL_INDEX = 0; constexpr uint8_t MAX_H2D_CHANNEL_INDEX = 15; constexpr uint8_t MIN_D2H_CHANNEL_INDEX = MAX_H2D_CHANNEL_INDEX + 1; @@ -63,6 +68,22 @@ struct ChannelInterruptTimestampList { size_t count; }; +struct ChannelIrqData { + vdma::ChannelId channel_id; + bool is_active; + uint16_t desc_num_processed; + uint8_t host_error; + uint8_t device_error; +}; + +struct IrqData { + uint8_t channels_count; + std::array channels_irq_data; +}; + +// Bitmap per engine +using ChannelsBitmap = std::array; + #if defined(__linux__) || defined(_MSC_VER) using vdma_mapped_buffer_driver_identifier = uintptr_t; #elif defined(__QNX__) @@ -114,7 +135,6 @@ public: }; using VdmaBufferHandle = size_t; - using VdmaChannelHandle = uint64_t; static Expected create(const std::string &dev_path); @@ -133,15 +153,12 @@ public: hailo_status write_vdma_channel_register(vdma::ChannelId channel_id, DmaDirection data_direction, size_t offset, size_t reg_size, uint32_t data); - hailo_status vdma_buffer_sync(VdmaBufferHandle buffer, DmaDirection sync_direction, void *address, size_t buffer_size); + hailo_status vdma_buffer_sync(VdmaBufferHandle buffer, DmaDirection sync_direction, size_t offset, size_t count); - Expected vdma_channel_enable(vdma::ChannelId channel_id, DmaDirection data_direction, - bool enable_timestamps_measure); - hailo_status vdma_channel_disable(vdma::ChannelId channel_index, VdmaChannelHandle channel_handle); - Expected wait_channel_interrupts(vdma::ChannelId channel_id, - VdmaChannelHandle channel_handle, const std::chrono::milliseconds &timeout); - hailo_status vdma_channel_abort(vdma::ChannelId channel_id, VdmaChannelHandle channel_handle); - hailo_status vdma_channel_clear_abort(vdma::ChannelId channel_id, VdmaChannelHandle channel_handle); + hailo_status vdma_interrupts_enable(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure); + hailo_status vdma_interrupts_disable(const ChannelsBitmap &channel_id); + Expected vdma_interrupts_wait(const ChannelsBitmap &channels_bitmap); + Expected vdma_interrupts_read_timestamps(vdma::ChannelId channel_id); Expected> read_notification(); hailo_status disable_notifications(); @@ -172,7 +189,7 @@ public: * of user allocated buffer */ Expected vdma_buffer_map(void *user_address, size_t required_size, DmaDirection data_direction, - vdma_mapped_buffer_driver_identifier &driver_buff_handle); + const vdma_mapped_buffer_driver_identifier &driver_buff_handle); /** * Unmaps user buffer mapped using HailoRTDriver::map_buffer. @@ -197,7 +214,7 @@ public: * Configure vdma channel descriptors to point to the given user address. */ hailo_status descriptors_list_bind_vdma_buffer(uintptr_t desc_handle, VdmaBufferHandle buffer_handle, - uint16_t desc_page_size, uint8_t channel_index, size_t offset); + uint16_t desc_page_size, uint8_t channel_index, uint32_t starting_desc); Expected vdma_low_memory_buffer_alloc(size_t size); hailo_status vdma_low_memory_buffer_free(uintptr_t buffer_handle); @@ -272,9 +289,9 @@ public: HailoRTDriver(HailoRTDriver &&other) noexcept = default; HailoRTDriver &operator=(HailoRTDriver &&other) = default; - static const VdmaChannelHandle INVALID_VDMA_CHANNEL_HANDLE; - static const uintptr_t INVALID_DRIVER_BUFFER_HANDLE_VALUE; - static const uint8_t INVALID_VDMA_CHANNEL_INDEX; + static const uintptr_t INVALID_DRIVER_BUFFER_HANDLE_VALUE; + static const size_t INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE; + static const uint8_t INVALID_VDMA_CHANNEL_INDEX; private: hailo_status read_memory_ioctl(MemoryType memory_type, uint64_t address, void *buf, size_t size); @@ -283,6 +300,17 @@ private: HailoRTDriver(const std::string &dev_path, FileDescriptor &&fd, hailo_status &status); bool is_valid_channel_id(const vdma::ChannelId &channel_id); + bool is_valid_channels_bitmap(const ChannelsBitmap &bitmap) + { + for (size_t engine_index = m_dma_engines_count; engine_index < MAX_VDMA_ENGINES_COUNT; engine_index++) { + if (bitmap[engine_index]) { + LOGGER__ERROR("Engine {} does not exist on device (engines count {})", engine_index, + m_dma_engines_count); + return false; + } + } + return true; + } FileDescriptor m_fd; std::string m_dev_path; diff --git a/hailort/libhailort/src/os/mmap_buffer.hpp b/hailort/libhailort/src/os/mmap_buffer.hpp index 1f6031c..e66cdfd 100644 --- a/hailort/libhailort/src/os/mmap_buffer.hpp +++ b/hailort/libhailort/src/os/mmap_buffer.hpp @@ -6,7 +6,7 @@ * @file mmap_buffer.hpp * @brief RAII wrapper around memory mapping (mmap) * - * + * **/ #ifndef _OS_MMAP_BUFFER_H_ @@ -23,7 +23,6 @@ namespace hailort class MmapBufferImpl final { public: - static Expected create_shared_memory(size_t length); static Expected create_file_map(size_t length, FileDescriptor &file, uintptr_t offset); @@ -37,10 +36,10 @@ public: MmapBufferImpl(const MmapBufferImpl &other) = delete; MmapBufferImpl &operator=(const MmapBufferImpl &other) = delete; - MmapBufferImpl(MmapBufferImpl &&other) noexcept : - m_address(std::exchange(other.m_address, INVALID_ADDR)), + MmapBufferImpl(MmapBufferImpl &&other) noexcept : + m_address(std::exchange(other.m_address, INVALID_ADDR)), m_length(std::move(other.m_length)) {}; - MmapBufferImpl &operator=(MmapBufferImpl &&other) noexcept + MmapBufferImpl &operator=(MmapBufferImpl &&other) noexcept { std::swap(m_address, other.m_address); std::swap(m_length, other.m_length); @@ -48,11 +47,11 @@ public: return *this; }; - void *get() { + void *address() { return m_address; } - explicit operator bool() const + bool is_mapped() const { return (INVALID_ADDR != m_address); } @@ -68,7 +67,7 @@ private: void *m_address; size_t m_length; - bool m_unmappable; + bool m_unmappable; }; template @@ -90,7 +89,6 @@ public: return MmapBuffer(std::move(mmap.release())); } - MmapBuffer() = default; ~MmapBuffer() = default; @@ -101,29 +99,28 @@ public: T* operator->() { - return get(); + return address(); } - T* get() { - return reinterpret_cast(m_mmap.get()); + T* address() { + return reinterpret_cast(m_mmap.address()); } - template std::enable_if_t::value, U&> operator*() { - return get()[0]; + return address()[0]; } template std::enable_if_t::value, U&> operator[](size_t i) { - return get()[i]; + return address()[i]; } - explicit operator bool() const + bool is_mapped() const { - return bool(m_mmap); + return m_mmap.is_mapped(); } // 'munmap' the current mapped buffer (if currently mapped). diff --git a/hailort/libhailort/src/os/posix/hailort_driver.cpp b/hailort/libhailort/src/os/posix/hailort_driver.cpp index 7b4d4a1..47b3a1e 100755 --- a/hailort/libhailort/src/os/posix/hailort_driver.cpp +++ b/hailort/libhailort/src/os/posix/hailort_driver.cpp @@ -22,6 +22,7 @@ namespace hailort { static_assert(VDMA_CHANNELS_PER_ENGINE == MAX_VDMA_CHANNELS_PER_ENGINE, "Driver and libhailort parameters mismatch"); +static_assert(MAX_VDMA_ENGINES == MAX_VDMA_ENGINES_COUNT, "Driver and libhailort parameters mismatch"); static_assert(MIN_D2H_CHANNEL_INDEX == VDMA_DEST_CHANNELS_START, "Driver and libhailort parameters mismatch"); static hailo_dma_data_direction direction_to_dma_data_direction(HailoRTDriver::DmaDirection direction) { @@ -86,11 +87,12 @@ static hailo_transfer_memory_type translate_memory_type(HailoRTDriver::MemoryTyp return HAILO_TRANSFER_MEMORY_MAX_ENUM; } -static Expected create_interrupt_timestamp_list(hailo_vdma_channel_wait_params &inter_data) +static Expected create_interrupt_timestamp_list( + hailo_vdma_interrupts_read_timestamp_params &inter_data) { - CHECK_AS_EXPECTED(inter_data.timestamps_count <= MAX_IRQ_TIMESTAMPS_SIZE, HAILO_PCIE_DRIVER_FAIL, - "Invalid channel interrupt timestamps count returned {}", inter_data.timestamps_count); - ChannelInterruptTimestampList timestamp_list; + CHECK_AS_EXPECTED(inter_data.timestamps_count <= MAX_IRQ_TIMESTAMPS_SIZE, HAILO_DRIVER_FAIL, + "Invalid channel interrupts timestamps count returned {}", inter_data.timestamps_count); + ChannelInterruptTimestampList timestamp_list{}; timestamp_list.count = inter_data.timestamps_count; for (size_t i = 0; i < timestamp_list.count; i++) { @@ -100,8 +102,9 @@ static Expected create_interrupt_timestamp_list(h return timestamp_list; } -const HailoRTDriver::VdmaChannelHandle HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE = INVALID_CHANNEL_HANDLE_VALUE; +// TODO: validate wraparounds for buffer/mapping handles in the driver (HRT-9509) const uintptr_t HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE = INVALID_DRIVER_HANDLE_VALUE; +const size_t HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE = INVALID_DRIVER_HANDLE_VALUE; const uint8_t HailoRTDriver::INVALID_VDMA_CHANNEL_INDEX = INVALID_VDMA_CHANNEL; Expected HailoRTDriver::create(const std::string &dev_path) @@ -132,16 +135,8 @@ hailo_status HailoRTDriver::hailo_ioctl(int fd, int request, void* request_struc #else #error "unsupported platform!" #endif // __linux__ - switch (error_status) { - case ETIMEDOUT: - return HAILO_TIMEOUT; - case ECONNABORTED: - return HAILO_STREAM_ABORTED_BY_USER; - case ECONNRESET: - return HAILO_STREAM_NOT_ACTIVATED; - default: - return HAILO_PCIE_DRIVER_FAIL; - } + + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } @@ -199,7 +194,7 @@ HailoRTDriver::HailoRTDriver(const std::string &dev_path, FileDescriptor &&fd, h break; default: LOGGER__ERROR("Invalid dma type returned from ioctl {}", device_properties.dma_type); - status = HAILO_PCIE_DRIVER_FAIL; + status = HAILO_DRIVER_FAIL; return; } @@ -219,7 +214,7 @@ Expected> HailoRTDriver::read_notification() int err = 0; auto status = hailo_ioctl(this->m_fd, HAILO_READ_NOTIFICATION, ¬ification_buffer, err); if (HAILO_SUCCESS != status) { - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } std::vector notification(notification_buffer.buffer_len); @@ -233,7 +228,7 @@ hailo_status HailoRTDriver::disable_notifications() auto status = hailo_ioctl(this->m_fd, HAILO_DISABLE_NOTIFICATION, 0, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("HAILO_DISABLE_NOTIFICATION failed with errno: {}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; @@ -292,7 +287,7 @@ Expected HailoRTDriver::read_vdma_channel_register(vdma::ChannelId cha auto status = hailo_ioctl(m_fd, HAILO_VDMA_CHANNEL_READ_REGISTER, ¶ms, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("HailoRTDriver::read_vdma_channel_register failed with errno:{}", err); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return std::move(params.data); @@ -316,7 +311,7 @@ hailo_status HailoRTDriver::write_vdma_channel_register(vdma::ChannelId channel_ auto status = hailo_ioctl(m_fd, HAILO_VDMA_CHANNEL_WRITE_REGISTER, ¶ms, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("HailoRTDriver::write_vdma_channel_register failed with errno:{}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; @@ -395,7 +390,7 @@ hailo_status HailoRTDriver::read_memory_ioctl(MemoryType memory_type, uint64_t a auto status = hailo_ioctl(this->m_fd, HAILO_MEMORY_TRANSFER, &transfer, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("HailoRTDriver::read_memory failed with errno:{}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } memcpy(buf, transfer.buffer, transfer.count); @@ -428,130 +423,124 @@ hailo_status HailoRTDriver::write_memory_ioctl(MemoryType memory_type, uint64_t auto status = hailo_ioctl(this->m_fd, HAILO_MEMORY_TRANSFER, &transfer, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("HailoRTDriver::write_memory failed with errno:{}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } -hailo_status HailoRTDriver::vdma_buffer_sync(VdmaBufferHandle handle, DmaDirection sync_direction, void *address, - size_t buffer_size) +hailo_status HailoRTDriver::vdma_buffer_sync(VdmaBufferHandle handle, DmaDirection sync_direction, size_t offset, size_t count) { #if defined(__linux__) CHECK(sync_direction != DmaDirection::BOTH, HAILO_INVALID_ARGUMENT, "Can't sync vdma data both host and device"); hailo_vdma_buffer_sync_params sync_info{ .handle = handle, .sync_type = (sync_direction == DmaDirection::H2D) ? HAILO_SYNC_FOR_DEVICE : HAILO_SYNC_FOR_HOST, - .buffer_address = address, - .buffer_size = buffer_size + .offset = offset, + .count = count }; int err = 0; auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_BUFFER_SYNC, &sync_info, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("HAILO_VDMA_BUFFER_SYNC failed with errno:{}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; // TODO: HRT-6717 - Remove ifdef when Implement sync ioctl (if determined needed in qnx) #elif defined( __QNX__) (void) handle; (void) sync_direction; - (void) address; - (void) buffer_size; + (void) offset; + (void) count; return HAILO_SUCCESS; #else #error "unsupported platform!" #endif // __linux__ } - -Expected HailoRTDriver::vdma_channel_enable(vdma::ChannelId channel_id, - DmaDirection data_direction, bool enable_timestamps_measure) +hailo_status HailoRTDriver::vdma_interrupts_enable(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure) { - CHECK_AS_EXPECTED(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - CHECK_AS_EXPECTED(data_direction != DmaDirection::BOTH, HAILO_INVALID_ARGUMENT, "Invalid direction given"); - hailo_vdma_channel_enable_params params { - .engine_index = channel_id.engine_index, - .channel_index = channel_id.channel_index, - .direction = direction_to_dma_data_direction(data_direction), - .enable_timestamps_measure = enable_timestamps_measure, - .channel_handle = INVALID_CHANNEL_HANDLE_VALUE, - }; + CHECK(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given"); + hailo_vdma_interrupts_enable_params params{}; + std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine); + params.enable_timestamps_measure = enable_timestamps_measure; int err = 0; - auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_CHANNEL_ENABLE, ¶ms, err); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to enable interrupt for channel {} with errno:{}", channel_id, err); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); - } + auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_INTERRUPTS_ENABLE, ¶ms, err); + CHECK_SUCCESS(status, "Failed to enable vdma interrupts with errno:{}", err); - return VdmaChannelHandle(params.channel_handle); + return HAILO_SUCCESS; } -hailo_status HailoRTDriver::vdma_channel_disable(vdma::ChannelId channel_id, VdmaChannelHandle channel_handle) +hailo_status HailoRTDriver::vdma_interrupts_disable(const ChannelsBitmap &channels_bitmap) { - CHECK(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - hailo_vdma_channel_disable_params params { - .engine_index = channel_id.engine_index, - .channel_index = channel_id.channel_index, - .channel_handle = channel_handle - }; + CHECK(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given"); + hailo_vdma_interrupts_disable_params params{}; + std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine); int err = 0; - auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_CHANNEL_DISABLE, ¶ms, err); + auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_INTERRUPTS_DISABLE, ¶ms, err); if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to disable interrupt for channel {} with errno:{}", channel_id, err); - return HAILO_PCIE_DRIVER_FAIL; + LOGGER__ERROR("Failed to disable vdma interrupts with errno:{}", err); + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } -Expected HailoRTDriver::wait_channel_interrupts(vdma::ChannelId channel_id, - VdmaChannelHandle channel_handle, const std::chrono::milliseconds &timeout) +static Expected to_irq_data(const hailo_vdma_interrupts_wait_params& params, + uint8_t engines_count) { - CHECK_AS_EXPECTED(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - CHECK_AS_EXPECTED(timeout.count() >= 0, HAILO_INVALID_ARGUMENT); + static_assert(ARRAY_ENTRIES(IrqData::channels_irq_data) == ARRAY_ENTRIES(params.irq_data), "Mismatch irq data size"); + CHECK_AS_EXPECTED(params.channels_count <= ARRAY_ENTRIES(params.irq_data), HAILO_DRIVER_FAIL, + "Invalid channels count returned from vdma_interrupts_wait"); -#if defined(__linux__) - struct hailo_channel_interrupt_timestamp timestamps[MAX_IRQ_TIMESTAMPS_SIZE]; -#endif + IrqData irq{}; + irq.channels_count = params.channels_count; + for (uint8_t i = 0; i < params.channels_count; i++) { + const auto engine_index = params.irq_data[i].engine_index; + const auto channel_index = params.irq_data[i].channel_index; + CHECK_AS_EXPECTED(engine_index < engines_count, HAILO_DRIVER_FAIL, + "Invalid engine index {} returned from vdma_interrupts_wait, max {}", engine_index, engines_count); + CHECK_AS_EXPECTED(channel_index < MAX_VDMA_CHANNELS_PER_ENGINE, HAILO_DRIVER_FAIL, + "Invalid channel_index index {} returned from vdma_interrupts_wait", channel_index); - hailo_vdma_channel_wait_params data { - .engine_index = channel_id.engine_index, - .channel_index = channel_id.channel_index, - .channel_handle = channel_handle, - .timeout_ms = static_cast(timeout.count()), - .timestamps_count = MAX_IRQ_TIMESTAMPS_SIZE, -// In linux send address to local buffer because there isnt room on stack for array -#if defined(__linux__) - .timestamps = timestamps, -#elif defined(__QNX__) - .timestamps = {} -#else -#error "unsupported platform!" -#endif // __linux__ - }; + irq.channels_irq_data[i].channel_id.engine_index = engine_index; + irq.channels_irq_data[i].channel_id.channel_index = channel_index; + irq.channels_irq_data[i].is_active = params.irq_data[i].is_active; + irq.channels_irq_data[i].desc_num_processed = params.irq_data[i].host_num_processed; + irq.channels_irq_data[i].host_error = params.irq_data[i].host_error; + irq.channels_irq_data[i].device_error = params.irq_data[i].device_error; + } + return irq; +} + +Expected HailoRTDriver::vdma_interrupts_wait(const ChannelsBitmap &channels_bitmap) +{ + CHECK_AS_EXPECTED(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given"); + hailo_vdma_interrupts_wait_params params{}; + std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine); int err = 0; - auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_CHANNEL_WAIT_INT, &data, err); + auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_INTERRUPTS_WAIT, ¶ms, err); if (HAILO_SUCCESS != status) { - if (HAILO_TIMEOUT == status) { - LOGGER__ERROR("Waiting for interrupt for channel {} timed-out (errno=ETIMEDOUT)", channel_id); - return make_unexpected(status); - } - if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("Channel (index={}) was aborted!", channel_id); - return make_unexpected(status); - } - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__INFO("Channel (index={}) was deactivated!", channel_id); - return make_unexpected(status); - } - LOGGER__ERROR("Failed to wait interrupt for channel {} with errno:{}", channel_id, err); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + LOGGER__ERROR("Failed to wait vdma interrupts with errno:{}", err); + return make_unexpected(HAILO_DRIVER_FAIL); } + return to_irq_data(params, static_cast(m_dma_engines_count)); +} + +Expected HailoRTDriver::vdma_interrupts_read_timestamps(vdma::ChannelId channel_id) +{ + hailo_vdma_interrupts_read_timestamp_params data{}; + data.engine_index = channel_id.engine_index; + data.channel_index = channel_id.channel_index; + + int err = 0; + auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS, &data, err); + CHECK_SUCCESS_AS_EXPECTED(status); + return create_interrupt_timestamp_list(data); } @@ -604,17 +593,17 @@ hailo_status HailoRTDriver::read_log(uint8_t *buffer, size_t buffer_size, size_t .read_bytes = 0 }; - CHECK(buffer_size <= sizeof(params.buffer), HAILO_PCIE_DRIVER_FAIL, + CHECK(buffer_size <= sizeof(params.buffer), HAILO_DRIVER_FAIL, "Given buffer size {} is bigger than buffer size used to read logs {}", buffer_size, sizeof(params.buffer)); int err = 0; auto status = hailo_ioctl(this->m_fd, HAILO_READ_LOG, ¶ms, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to read log with errno:{}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } - CHECK(params.read_bytes <= sizeof(params.buffer), HAILO_PCIE_DRIVER_FAIL, + CHECK(params.read_bytes <= sizeof(params.buffer), HAILO_DRIVER_FAIL, "Amount of bytes read from log {} is bigger than size of buffer {}", params.read_bytes, sizeof(params.buffer)); memcpy(buffer, params.buffer, params.read_bytes); @@ -629,14 +618,14 @@ hailo_status HailoRTDriver::reset_nn_core() auto status = hailo_ioctl(this->m_fd, HAILO_RESET_NN_CORE, nullptr, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to reset nn core with errno:{}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } Expected HailoRTDriver::vdma_buffer_map(void *user_address, size_t required_size, - DmaDirection data_direction, vdma_mapped_buffer_driver_identifier &driver_buff_handle) + DmaDirection data_direction, const vdma_mapped_buffer_driver_identifier &driver_buff_handle) { #if defined(__linux__) @@ -665,7 +654,7 @@ Expected HailoRTDriver::vdma_buffer_map(void *u auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_BUFFER_MAP, &map_user_buffer_info, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to map user buffer with errno:{}", err); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return VdmaBufferHandle(map_user_buffer_info.mapped_handle); @@ -681,7 +670,7 @@ hailo_status HailoRTDriver::vdma_buffer_unmap(VdmaBufferHandle handle) auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_BUFFER_UNMAP, &unmap_user_buffer_info, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to unmap user buffer with errno:{}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; @@ -695,7 +684,7 @@ Expected> HailoRTDriver::descriptors_list_create( auto status = hailo_ioctl(this->m_fd, HAILO_DESC_LIST_CREATE, &create_desc_info, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to create descriptors list with errno:{}", err); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return std::make_pair(create_desc_info.desc_handle, create_desc_info.dma_address); @@ -707,84 +696,32 @@ hailo_status HailoRTDriver::descriptors_list_release(uintptr_t desc_handle) auto status = hailo_ioctl(this->m_fd, HAILO_DESC_LIST_RELEASE, &desc_handle, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to release descriptors list with errno: {}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } hailo_status HailoRTDriver::descriptors_list_bind_vdma_buffer(uintptr_t desc_handle, VdmaBufferHandle buffer_handle, - uint16_t desc_page_size, uint8_t channel_index, size_t offset) + uint16_t desc_page_size, uint8_t channel_index, uint32_t starting_desc) { hailo_desc_list_bind_vdma_buffer_params config_info; config_info.buffer_handle = buffer_handle; config_info.desc_handle = desc_handle; config_info.desc_page_size = desc_page_size; config_info.channel_index = channel_index; - config_info.offset = offset; + config_info.starting_desc = starting_desc; int err = 0; auto status = hailo_ioctl(this->m_fd, HAILO_DESC_LIST_BIND_VDMA_BUFFER, &config_info, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to bind vdma buffer to descriptors list with errno: {}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } -hailo_status HailoRTDriver::vdma_channel_abort(vdma::ChannelId channel_id, VdmaChannelHandle channel_handle) -{ - CHECK(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - - hailo_vdma_channel_abort_params params = { - .engine_index = channel_id.engine_index, - .channel_index = channel_id.channel_index, - .channel_handle = channel_handle - }; - - int err = 0; - auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_CHANNEL_ABORT, ¶ms, err); - if (HAILO_SUCCESS != status) { - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__DEBUG("Channel (index={}) was deactivated!", channel_id); - return status; - } - else { - LOGGER__ERROR("Failed to abort vdma channel (index={}) with errno: {}", channel_id, err); - return HAILO_PCIE_DRIVER_FAIL; - } - } - - return HAILO_SUCCESS; -} - -hailo_status HailoRTDriver::vdma_channel_clear_abort(vdma::ChannelId channel_id, VdmaChannelHandle channel_handle) -{ - CHECK(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - - hailo_vdma_channel_clear_abort_params params = { - .engine_index = channel_id.engine_index, - .channel_index = channel_id.channel_index, - .channel_handle = channel_handle - }; - - int err = 0; - auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_CHANNEL_CLEAR_ABORT, ¶ms, err); - if (HAILO_SUCCESS != status) { - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__DEBUG("Channel (index={}) was deactivated!", channel_id); - return status; - } - else { - LOGGER__ERROR("Failed to clear abort vdma channel (index={}) with errno: {}", channel_id, err); - return HAILO_PCIE_DRIVER_FAIL; - } - } - - return HAILO_SUCCESS; -} - Expected HailoRTDriver::vdma_low_memory_buffer_alloc(size_t size) { CHECK_AS_EXPECTED(m_allocate_driver_buffer, HAILO_INVALID_OPERATION, @@ -799,7 +736,7 @@ Expected HailoRTDriver::vdma_low_memory_buffer_alloc(size_t size) auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC, &allocate_params, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to allocate buffer with errno: {}", err); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return std::move(allocate_params.buffer_handle); @@ -814,7 +751,7 @@ hailo_status HailoRTDriver::vdma_low_memory_buffer_free(uintptr_t buffer_handle) auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE, (void*)buffer_handle, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to free allocated buffer with errno: {}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; @@ -828,7 +765,7 @@ Expected> HailoRTDriver::vdma_continuous_buffer_a auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC, ¶ms, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed allocate continuous buffer with errno:{}", err); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return std::make_pair(params.buffer_handle, params.dma_address); @@ -840,7 +777,7 @@ hailo_status HailoRTDriver::vdma_continuous_buffer_free(uintptr_t buffer_handle) auto status = hailo_ioctl(this->m_fd, HAILO_VDMA_CONTINUOUS_BUFFER_FREE, (void*)buffer_handle, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to free continuous buffer with errno: {}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; @@ -855,7 +792,7 @@ hailo_status HailoRTDriver::mark_as_used() auto status = hailo_ioctl(this->m_fd, HAILO_MARK_AS_IN_USE, ¶ms, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to mark device as in use with errno: {}", err); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } if (params.in_use) { return HAILO_DEVICE_IN_USE; diff --git a/hailort/libhailort/src/os/posix/mmap_buffer.cpp b/hailort/libhailort/src/os/posix/mmap_buffer.cpp index e26ae19..0d2ff57 100644 --- a/hailort/libhailort/src/os/posix/mmap_buffer.cpp +++ b/hailort/libhailort/src/os/posix/mmap_buffer.cpp @@ -34,7 +34,7 @@ void * const MmapBufferImpl::INVALID_ADDR = MAP_FAILED; Expected MmapBufferImpl::create_shared_memory(size_t length) { - void *address = mmap(nullptr, length, PROT_WRITE | PROT_READ, + void *address = mmap(nullptr, length, PROT_WRITE | PROT_READ, MAP_ANONYMOUS | MAP_SHARED | MAP_UNINITIALIZED, INVALID_FD, /*offset=*/ 0); @@ -60,7 +60,7 @@ Expected MmapBufferImpl::create_file_map(size_t length, FileDesc auto status = HailoRTDriver::hailo_ioctl(file, HAILO_NON_LINUX_DESC_LIST_MMAP, &map_vdma_list_params, err); if (HAILO_SUCCESS != status) { LOGGER__ERROR("HAILO_NON_LINUX_DESC_LIST_MMAP failed with errno:{}", err); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } void *address = mmap(nullptr, length, PROT_WRITE | PROT_READ | PROT_NOCACHE, MAP_SHARED | MAP_PHYS, NOFD, (off_t)map_vdma_list_params.user_address); @@ -74,14 +74,17 @@ Expected MmapBufferImpl::create_file_map(size_t length, FileDesc hailo_status MmapBufferImpl::unmap() { - if (INVALID_ADDR != m_address) { - if (0 != munmap(m_address, m_length)) { - LOGGER__ERROR("munmap of address {}, length: {} failed with errno {}", (void*)m_address, m_length, errno); - return HAILO_INTERNAL_FAILURE; - } - m_address = INVALID_ADDR; - m_length = 0; + if (!is_mapped()) { + return HAILO_SUCCESS; } + + if (0 != munmap(m_address, m_length)) { + LOGGER__ERROR("munmap of address {}, length: {} failed with errno {}", (void*)m_address, m_length, errno); + return HAILO_INTERNAL_FAILURE; + } + + m_address = INVALID_ADDR; + m_length = 0; return HAILO_SUCCESS; } diff --git a/hailort/libhailort/src/os/posix/qnx/driver_scan.cpp b/hailort/libhailort/src/os/posix/qnx/driver_scan.cpp index bc7f731..5c422eb 100644 --- a/hailort/libhailort/src/os/posix/qnx/driver_scan.cpp +++ b/hailort/libhailort/src/os/posix/qnx/driver_scan.cpp @@ -32,7 +32,7 @@ Expected> list_devices() } else { LOGGER__ERROR("Failed to open hailo pcie class ({}), errno {}", HAILO_PCIE_CLASS_PATH, errno); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } } diff --git a/hailort/libhailort/src/os/posix/qnx/event.cpp b/hailort/libhailort/src/os/posix/qnx/event.cpp index ed6892d..9303b93 100644 --- a/hailort/libhailort/src/os/posix/qnx/event.cpp +++ b/hailort/libhailort/src/os/posix/qnx/event.cpp @@ -9,10 +9,13 @@ * This class implements our Events API over the neosmart pevents events. It also implement the Semaphore behavior and API * Using the pevents events. For more information check out the implementation of pevents https://github.com/neosmart/pevents **/ -#include "hailo/event.hpp" + #include "hailo/hailort.h" +#include "hailo/event.hpp" + #include "common/utils.hpp" -#include "event_internal.hpp" + +#include "utils/event_internal.hpp" #include #include @@ -21,6 +24,7 @@ #include "pevents.h" #undef WFMO + #define INVALID_EVENT_HANDLE (nullptr) #define WAIT_OBJECT_0 (0) diff --git a/hailort/libhailort/src/os/posix/unix/driver_scan.cpp b/hailort/libhailort/src/os/posix/unix/driver_scan.cpp index a91fc08..6ba7dae 100644 --- a/hailort/libhailort/src/os/posix/unix/driver_scan.cpp +++ b/hailort/libhailort/src/os/posix/unix/driver_scan.cpp @@ -30,7 +30,7 @@ Expected> list_devices() } else { LOGGER__ERROR("Failed to open hailo pcie class ({}), errno {}", HAILO_CLASS_PATH, errno); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } } @@ -53,11 +53,11 @@ Expected query_device_info(const std::string &device_ const std::string device_id_path = std::string(HAILO_CLASS_PATH) + "/" + device_name + "/" + HAILO_BOARD_LOCATION_FILENAME; std::ifstream device_id_file(device_id_path); - CHECK_AS_EXPECTED(device_id_file.good(), HAILO_PCIE_DRIVER_FAIL, "Failed open {}", device_id_path); + CHECK_AS_EXPECTED(device_id_file.good(), HAILO_DRIVER_FAIL, "Failed open {}", device_id_path); std::string device_id; std::getline(device_id_file, device_id); - CHECK_AS_EXPECTED(device_id_file.eof(), HAILO_PCIE_DRIVER_FAIL, "Failed read {}", device_id_path); + CHECK_AS_EXPECTED(device_id_file.eof(), HAILO_DRIVER_FAIL, "Failed read {}", device_id_path); HailoRTDriver::DeviceInfo device_info = {}; device_info.dev_path = std::string("/dev/") + device_name; diff --git a/hailort/libhailort/src/os/posix/unix/event.cpp b/hailort/libhailort/src/os/posix/unix/event.cpp index 465b4c8..4c4525e 100644 --- a/hailort/libhailort/src/os/posix/unix/event.cpp +++ b/hailort/libhailort/src/os/posix/unix/event.cpp @@ -8,15 +8,19 @@ * * TODO: doc **/ -#include "hailo/event.hpp" + #include "hailo/hailort.h" +#include "hailo/event.hpp" + #include "common/utils.hpp" -#include "event_internal.hpp" + +#include "utils/event_internal.hpp" #include #include #include + namespace hailort { diff --git a/hailort/libhailort/src/os/windows/driver_scan.cpp b/hailort/libhailort/src/os/windows/driver_scan.cpp index 88d68ec..3870675 100644 --- a/hailort/libhailort/src/os/windows/driver_scan.cpp +++ b/hailort/libhailort/src/os/windows/driver_scan.cpp @@ -179,7 +179,7 @@ static Expected parse_uint32_property(const std::wstring &dev_interfac uint32_t number = 0; if (!prop.Number(number)) { LOGGER__ERROR("Failed parsing prop"); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return number; } diff --git a/hailort/libhailort/src/os/windows/event.cpp b/hailort/libhailort/src/os/windows/event.cpp index dd053ac..ab1a35d 100644 --- a/hailort/libhailort/src/os/windows/event.cpp +++ b/hailort/libhailort/src/os/windows/event.cpp @@ -6,14 +6,18 @@ * @file event.cpp * @brief Event & Semaphore wrapper for Windows **/ -#include "hailo/event.hpp" + #include "hailo/hailort.h" +#include "hailo/event.hpp" + #include "common/utils.hpp" -#include "event_internal.hpp" + +#include "utils/event_internal.hpp" #include #include + namespace hailort { diff --git a/hailort/libhailort/src/os/windows/hailort_driver.cpp b/hailort/libhailort/src/os/windows/hailort_driver.cpp index c167918..7707978 100644 --- a/hailort/libhailort/src/os/windows/hailort_driver.cpp +++ b/hailort/libhailort/src/os/windows/hailort_driver.cpp @@ -22,6 +22,7 @@ namespace hailort { static_assert(VDMA_CHANNELS_PER_ENGINE == MAX_VDMA_CHANNELS_PER_ENGINE, "Driver and libhailort parameters mismatch"); +static_assert(MAX_VDMA_ENGINES == MAX_VDMA_ENGINES_COUNT, "Driver and libhailort parameters mismatch"); static_assert(MIN_D2H_CHANNEL_INDEX == VDMA_DEST_CHANNELS_START, "Driver and libhailort parameters mismatch"); //TODO HRT-7309: merge with posix @@ -274,8 +275,9 @@ static int ioctl(HANDLE h, ULONG val, tCompatibleHailoIoctlData *ioctl_data) return 0; } -const HailoRTDriver::VdmaChannelHandle HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE = INVALID_CHANNEL_HANDLE_VALUE; +// TODO: validate wraparounds for buffer/mapping handles in the driver (HRT-9509) const uintptr_t HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE = INVALID_DRIVER_HANDLE_VALUE; +const size_t HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE = INVALID_DRIVER_HANDLE_VALUE; const uint8_t HailoRTDriver::INVALID_VDMA_CHANNEL_INDEX = INVALID_VDMA_CHANNEL; static hailo_status validate_driver_version(const hailo_driver_info &driver_info) @@ -301,7 +303,7 @@ HailoRTDriver::HailoRTDriver(const std::string &dev_path, FileDescriptor &&fd, h hailo_driver_info& driver_info = data.Buffer.DriverInfo; if (0 > ioctl(m_fd, HAILO_QUERY_DRIVER_INFO, &data)) { LOGGER__ERROR("Failed to query driver info, errno {}", errno); - status = HAILO_PCIE_DRIVER_FAIL; + status = HAILO_DRIVER_FAIL; return; } status = validate_driver_version(driver_info); @@ -313,7 +315,7 @@ HailoRTDriver::HailoRTDriver(const std::string &dev_path, FileDescriptor &&fd, h hailo_device_properties& device_properties = data.Buffer.DeviceProperties; if (0 > ioctl(m_fd, HAILO_QUERY_DEVICE_PROPERTIES, &data)) { LOGGER__ERROR("Failed query pcie device properties, errno {}", errno); - status = HAILO_PCIE_DRIVER_FAIL; + status = HAILO_DRIVER_FAIL; return; } @@ -329,7 +331,7 @@ HailoRTDriver::HailoRTDriver(const std::string &dev_path, FileDescriptor &&fd, h break; default: LOGGER__ERROR("Invalid dma type returned from ioctl {}", device_properties.dma_type); - status = HAILO_PCIE_DRIVER_FAIL; + status = HAILO_DRIVER_FAIL; return; } @@ -375,7 +377,7 @@ Expected> HailoRTDriver::read_notification() auto rc = ioctl(this->m_fd, HAILO_READ_NOTIFICATION, &data); if (0 > rc) { - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } std::vector notification(notification_buffer.buffer_len); @@ -388,7 +390,7 @@ hailo_status HailoRTDriver::disable_notifications() tCompatibleHailoIoctlData data = {}; int res = ioctl(m_fd, HAILO_DISABLE_NOTIFICATION, &data); - CHECK(0 <= res, HAILO_PCIE_DRIVER_FAIL, "HAILO_DISABLE_NOTIFICATION failed with errno: {}", errno); + CHECK(0 <= res, HAILO_DRIVER_FAIL, "HAILO_DISABLE_NOTIFICATION failed with errno: {}", errno); return HAILO_SUCCESS; } @@ -471,7 +473,7 @@ hailo_status HailoRTDriver::read_memory_ioctl(MemoryType memory_type, uint64_t a if (0 > ioctl(m_fd, HAILO_MEMORY_TRANSFER, &data)) { LOGGER__ERROR("HailoRTDriver::read_memory failed with errno:{}", errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } memcpy(buf, transfer.buffer, transfer.count); @@ -510,7 +512,7 @@ hailo_status HailoRTDriver::write_memory_ioctl(MemoryType memory_type, uint64_t if (0 > ioctl(this->m_fd, HAILO_MEMORY_TRANSFER, &data)) { LOGGER__ERROR("HailoRTDriver::write_memory failed with errno: {}", errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; @@ -533,7 +535,7 @@ Expected HailoRTDriver::read_vdma_channel_register(vdma::ChannelId cha if (0 > ioctl(this->m_fd, HAILO_VDMA_CHANNEL_READ_REGISTER, &data)) { LOGGER__ERROR("HailoRTDriver::read_vdma_channel_register failed with errno: {}", errno); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return std::move(params.data); @@ -556,110 +558,127 @@ hailo_status HailoRTDriver::write_vdma_channel_register(vdma::ChannelId channel_ if (0 > ioctl(this->m_fd, HAILO_VDMA_CHANNEL_WRITE_REGISTER, &data)) { LOGGER__ERROR("HailoRTDriver::write_vdma_channel_register failed with errno: {}", errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } -hailo_status HailoRTDriver::vdma_buffer_sync(VdmaBufferHandle handle, DmaDirection sync_direction, void *address, - size_t buffer_size) +hailo_status HailoRTDriver::vdma_buffer_sync(VdmaBufferHandle handle, DmaDirection sync_direction, size_t offset, size_t count) { CHECK(sync_direction != DmaDirection::BOTH, HAILO_INVALID_ARGUMENT, "Can't sync vdma data both host and device"); tCompatibleHailoIoctlData data = {}; hailo_vdma_buffer_sync_params& sync_info = data.Buffer.VdmaBufferSync; sync_info.handle = handle; sync_info.sync_type = (sync_direction == DmaDirection::H2D) ? HAILO_SYNC_FOR_DEVICE : HAILO_SYNC_FOR_HOST; - sync_info.buffer_address = address; - sync_info.buffer_size = buffer_size; + sync_info.offset = offset; + sync_info.count = count; if (0 > ioctl(this->m_fd, HAILO_VDMA_BUFFER_SYNC, &data)) { LOGGER__ERROR("HAILO_VDMA_BUFFER_SYNC failed with errno: {}", errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } -Expected HailoRTDriver::vdma_channel_enable(vdma::ChannelId channel_id, - DmaDirection data_direction, bool enable_timestamps_measure) + +hailo_status HailoRTDriver::vdma_interrupts_enable(const ChannelsBitmap &channels_bitmap, bool enable_timestamps_measure) { - CHECK_AS_EXPECTED(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - CHECK_AS_EXPECTED(data_direction != DmaDirection::BOTH, HAILO_INVALID_ARGUMENT, "Invalid direction given"); + CHECK(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given"); tCompatibleHailoIoctlData data = {}; - hailo_vdma_channel_enable_params& params = data.Buffer.ChannelEnable; - params.engine_index = channel_id.engine_index; - params.channel_index = channel_id.channel_index; - params.direction = direction_to_dma_data_direction(data_direction); + hailo_vdma_interrupts_enable_params& params = data.Buffer.VdmaInterruptsEnable; + std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine); params.enable_timestamps_measure = enable_timestamps_measure; - if (0 > ioctl(this->m_fd, HAILO_VDMA_CHANNEL_ENABLE, &data)) { - LOGGER__ERROR("Failed to enable interrupt for channel {} with errno: {}", channel_id.channel_index, errno); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); - } + CHECK(ioctl(this->m_fd, HAILO_VDMA_INTERRUPTS_ENABLE, &data) >= 0, HAILO_DRIVER_FAIL, + "Failed to enable vdma interrupts with errno:{}", errno); - return std::move(params.channel_handle); + return HAILO_SUCCESS; } -hailo_status HailoRTDriver::vdma_channel_disable(vdma::ChannelId channel_id, VdmaChannelHandle channel_handle) +hailo_status HailoRTDriver::vdma_interrupts_disable(const ChannelsBitmap &channels_bitmap) { - CHECK(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); + CHECK(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given"); tCompatibleHailoIoctlData data = {}; - hailo_vdma_channel_disable_params& params = data.Buffer.ChannelDisable; - params.engine_index = channel_id.engine_index; - params.channel_index = channel_id.channel_index; - params.channel_handle = channel_handle; + hailo_vdma_interrupts_disable_params& params = data.Buffer.VdmaInterruptsDisable; + std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine); + - if (0 > ioctl(this->m_fd, HAILO_VDMA_CHANNEL_DISABLE, &data)) { - LOGGER__ERROR("Failed to disable interrupt for channel {} with errno: {}", channel_id, errno); - return HAILO_PCIE_DRIVER_FAIL; + if (0 > ioctl(this->m_fd, HAILO_VDMA_INTERRUPTS_DISABLE, &data)) { + LOGGER__ERROR("Failed to disable vdma interrupts with errno: {}", errno); + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } -//TODO: unify -static Expected create_interrupt_timestamp_list(hailo_vdma_channel_wait_params &inter_data) +// TODO: HRT-7309 - unite with posix +static Expected create_interrupt_timestamp_list( + hailo_vdma_interrupts_read_timestamp_params &inter_data) { - CHECK_AS_EXPECTED(inter_data.timestamps_count <= MAX_IRQ_TIMESTAMPS_SIZE, HAILO_PCIE_DRIVER_FAIL, - "Invalid channel interrupt timestamps count returned {}", inter_data.timestamps_count); - ChannelInterruptTimestampList timestamp_list; + CHECK_AS_EXPECTED(inter_data.timestamps_count <= MAX_IRQ_TIMESTAMPS_SIZE, HAILO_DRIVER_FAIL, + "Invalid channel interrupts timestamps count returned {}", inter_data.timestamps_count); + ChannelInterruptTimestampList timestamp_list{}; + timestamp_list.count = inter_data.timestamps_count; for (size_t i = 0; i < timestamp_list.count; i++) { timestamp_list.timestamp_list[i].timestamp = std::chrono::nanoseconds(inter_data.timestamps[i].timestamp_ns); timestamp_list.timestamp_list[i].desc_num_processed = inter_data.timestamps[i].desc_num_processed; } - return std::move(timestamp_list); + return timestamp_list; } -Expected HailoRTDriver::wait_channel_interrupts(vdma::ChannelId channel_id, - VdmaChannelHandle channel_handle, const std::chrono::milliseconds &timeout) +static Expected to_irq_data(const hailo_vdma_interrupts_wait_params& params, + uint8_t engines_count) { - CHECK_AS_EXPECTED(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - CHECK_AS_EXPECTED(timeout.count() >= 0, HAILO_INVALID_ARGUMENT); + static_assert(ARRAY_ENTRIES(IrqData::channels_irq_data) == ARRAY_ENTRIES(params.irq_data), "Mismatch irq data size"); + CHECK_AS_EXPECTED(params.channels_count <= ARRAY_ENTRIES(params.irq_data), HAILO_DRIVER_FAIL, + "Invalid channels count returned from vdma_interrupts_wait"); + + IrqData irq{}; + irq.channels_count = params.channels_count; + for (uint8_t i = 0; i < params.channels_count; i++) { + const auto engine_index = params.irq_data[i].engine_index; + const auto channel_index = params.irq_data[i].channel_index; + CHECK_AS_EXPECTED(engine_index < engines_count, HAILO_DRIVER_FAIL, + "Invalid engine index {} returned from vdma_interrupts_wait, max {}", engine_index, engines_count); + CHECK_AS_EXPECTED(channel_index < MAX_VDMA_CHANNELS_PER_ENGINE, HAILO_DRIVER_FAIL, + "Invalid channel_index index {} returned from vdma_interrupts_wait", channel_index); + + irq.channels_irq_data[i].channel_id.engine_index = engine_index; + irq.channels_irq_data[i].channel_id.channel_index = channel_index; + irq.channels_irq_data[i].is_active = params.irq_data[i].is_active; + irq.channels_irq_data[i].desc_num_processed = params.irq_data[i].host_num_processed; + irq.channels_irq_data[i].host_error = params.irq_data[i].host_error; + irq.channels_irq_data[i].device_error = params.irq_data[i].device_error; + } + return irq; +} + +Expected HailoRTDriver::vdma_interrupts_wait(const ChannelsBitmap &channels_bitmap) +{ + CHECK_AS_EXPECTED(is_valid_channels_bitmap(channels_bitmap), HAILO_INVALID_ARGUMENT, "Invalid channel bitmap given"); + tCompatibleHailoIoctlData data = {}; + hailo_vdma_interrupts_wait_params& params = data.Buffer.VdmaInterruptsWait; + std::copy(channels_bitmap.begin(), channels_bitmap.end(), params.channels_bitmap_per_engine); + + if (0 > ioctl(this->m_fd, HAILO_VDMA_INTERRUPTS_WAIT, &data)) { + LOGGER__ERROR("Failed to wait interrupts for channels bitmap with errno: {}", errno); + return make_unexpected(HAILO_DRIVER_FAIL); + } + + return to_irq_data(params, static_cast(m_dma_engines_count)); +} +Expected HailoRTDriver::vdma_interrupts_read_timestamps(vdma::ChannelId channel_id) +{ tCompatibleHailoIoctlData data = {}; - hailo_vdma_channel_wait_params& params = data.Buffer.ChannelWait; + hailo_vdma_interrupts_read_timestamp_params ¶ms = data.Buffer.VdmaInterruptsReadTimestamps; params.engine_index = channel_id.engine_index; params.channel_index = channel_id.channel_index; - params.channel_handle = channel_handle; - params.timeout_ms = static_cast(timeout.count()); - params.timestamps_count = MAX_IRQ_TIMESTAMPS_SIZE; - - if (0 > ioctl(this->m_fd, HAILO_VDMA_CHANNEL_WAIT_INT, &data)) { - const auto ioctl_errno = errno; - if (ERROR_SEM_TIMEOUT == ioctl_errno) { - LOGGER__ERROR("Waiting for interrupt for channel {} timed-out", channel_id); - return make_unexpected(HAILO_TIMEOUT); - } - if (ERROR_OPERATION_ABORTED == ioctl_errno) { - LOGGER__INFO("Stream (index={}) was aborted!", channel_id); - return make_unexpected(HAILO_STREAM_ABORTED_BY_USER); - } - if (ERROR_NOT_READY == ioctl_errno) { - LOGGER__INFO("Channel (index={}) was deactivated!", channel_id); - return make_unexpected(HAILO_STREAM_NOT_ACTIVATED); - } - LOGGER__ERROR("Failed to wait interrupt for channel {} with errno: {}", channel_id, ioctl_errno); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + + if (0 > ioctl(this->m_fd, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS, &data)) { + LOGGER__ERROR("Failed to read channel interrupts timestamps errno: {}", errno); + return make_unexpected(HAILO_DRIVER_FAIL); } return create_interrupt_timestamp_list(params); @@ -712,7 +731,7 @@ hailo_status read_log(uint8_t *buffer, size_t buffer_size, size_t *read_bytes, h } Expected HailoRTDriver::vdma_buffer_map(void *user_address, size_t required_size, DmaDirection data_direction, - vdma_mapped_buffer_driver_identifier &driver_buff_handle) + const vdma_mapped_buffer_driver_identifier &driver_buff_handle) { tCompatibleHailoIoctlData data = {}; hailo_vdma_buffer_map_params& map_user_buffer_info = data.Buffer.VdmaBufferMap; @@ -724,7 +743,7 @@ Expected HailoRTDriver::vdma_buffer_map(void *user_address, size_t requi if (0 > ioctl(this->m_fd, HAILO_VDMA_BUFFER_MAP, &data)) { LOGGER__ERROR("Failed to map user buffer with errno: {}", errno); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return std::move(map_user_buffer_info.mapped_handle); @@ -737,7 +756,7 @@ hailo_status HailoRTDriver::vdma_buffer_unmap(VdmaBufferHandle handle) unmap_user_buffer_info.mapped_handle = handle; if (0 > ioctl(this->m_fd, HAILO_VDMA_BUFFER_UNMAP, &data)) { LOGGER__ERROR("Failed to unmap user buffer with errno: {}", errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; @@ -753,7 +772,7 @@ Expected> HailoRTDriver::descriptors_list_create( if (0 > ioctl(this->m_fd, HAILO_DESC_LIST_CREATE, &data)) { LOGGER__ERROR("Failed to create descriptors list with errno: {}", errno); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } return std::move(std::make_pair(create_desc_info.desc_handle, create_desc_info.dma_address)); @@ -766,14 +785,14 @@ hailo_status HailoRTDriver::descriptors_list_release(uintptr_t desc_handle) release_desc_info = desc_handle; if (0 > ioctl(this->m_fd, HAILO_DESC_LIST_RELEASE, &data)) { LOGGER__ERROR("Failed to release descriptors list with errno: {}", errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; } hailo_status HailoRTDriver::descriptors_list_bind_vdma_buffer(uintptr_t desc_handle, VdmaBufferHandle buffer_handle, - uint16_t desc_page_size, uint8_t channel_index, size_t offset) + uint16_t desc_page_size, uint8_t channel_index, uint32_t starting_desc) { tCompatibleHailoIoctlData data = {}; hailo_desc_list_bind_vdma_buffer_params& config_info = data.Buffer.DescListBind; @@ -781,43 +800,11 @@ hailo_status HailoRTDriver::descriptors_list_bind_vdma_buffer(uintptr_t desc_han config_info.desc_handle = desc_handle; config_info.desc_page_size = desc_page_size; config_info.channel_index = channel_index; - config_info.offset = offset; + config_info.starting_desc = starting_desc; if (0 > ioctl(this->m_fd, HAILO_DESC_LIST_BIND_VDMA_BUFFER, &data)) { LOGGER__ERROR("Failed to bind vdma buffer to descriptors list with errno: {}", errno); - return HAILO_PCIE_DRIVER_FAIL; - } - - return HAILO_SUCCESS; -} - -hailo_status HailoRTDriver::vdma_channel_abort(vdma::ChannelId channel_id, VdmaChannelHandle channel_handle) -{ - CHECK(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - tCompatibleHailoIoctlData data = {}; - hailo_vdma_channel_abort_params& params = data.Buffer.ChannelAbort; - params.engine_index = channel_id.engine_index; - params.channel_index = channel_id.channel_index; - params.channel_handle = channel_handle; - if (0 > ioctl(this->m_fd, HAILO_VDMA_CHANNEL_ABORT, &data)) { - LOGGER__ERROR("Failed to abort vdma channel (index={}) with errno: {}", channel_id, errno); - return HAILO_PCIE_DRIVER_FAIL; - } - - return HAILO_SUCCESS; -} - -hailo_status HailoRTDriver::vdma_channel_clear_abort(vdma::ChannelId channel_id, VdmaChannelHandle channel_handle) -{ - CHECK(is_valid_channel_id(channel_id), HAILO_INVALID_ARGUMENT, "Invalid channel id {} given", channel_id); - tCompatibleHailoIoctlData data = {}; - hailo_vdma_channel_clear_abort_params& params = data.Buffer.ChannelClearAbort; - params.engine_index = channel_id.engine_index; - params.channel_index = channel_id.channel_index; - params.channel_handle = channel_handle; - if (0 > ioctl(this->m_fd, HAILO_VDMA_CHANNEL_CLEAR_ABORT, &data)) { - LOGGER__ERROR("Failed to clear abort vdma channel (index={}) with errno: {}", channel_id, errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } return HAILO_SUCCESS; @@ -835,10 +822,10 @@ hailo_status HailoRTDriver::read_log(uint8_t *buffer, size_t buffer_size, size_t if (0 > ioctl(this->m_fd, HAILO_READ_LOG, &data)) { LOGGER__ERROR("Failed to read log with errno:{}", errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } - CHECK(params.read_bytes <= sizeof(params.buffer), HAILO_PCIE_DRIVER_FAIL, + CHECK(params.read_bytes <= sizeof(params.buffer), HAILO_DRIVER_FAIL, "Amount of bytes read from log {} is bigger than size of buffer {}", params.read_bytes, sizeof(params.buffer)); @@ -861,7 +848,7 @@ Expected MmapBufferImpl::create_file_map(size_t length, FileDesc data.Buffer.DescListMmap.size = length; if (0 > ioctl(file, HAILO_NON_LINUX_DESC_LIST_MMAP, &data)) { LOGGER__ERROR("Failed to map physical memory with errno: {}", errno); - return make_unexpected(HAILO_PCIE_DRIVER_FAIL); + return make_unexpected(HAILO_DRIVER_FAIL); } // this mapping will be deleted automatically with the physical allocation return MmapBufferImpl(data.Buffer.DescListMmap.user_address, length, false); @@ -895,7 +882,7 @@ hailo_status HailoRTDriver::mark_as_used() tCompatibleHailoIoctlData data = {}; if (0 > ioctl(this->m_fd, HAILO_MARK_AS_IN_USE, &data)) { LOGGER__ERROR("Failed to mark device as in use with errno: {}", errno); - return HAILO_PCIE_DRIVER_FAIL; + return HAILO_DRIVER_FAIL; } if (data.Buffer.MarkAsInUse.in_use) { return HAILO_DEVICE_IN_USE; diff --git a/hailort/libhailort/src/pcie_stream.cpp b/hailort/libhailort/src/pcie_stream.cpp deleted file mode 100644 index c7f46e8..0000000 --- a/hailort/libhailort/src/pcie_stream.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file pcie_stream.cpp - **/ - -#include "pcie_stream.hpp" - -namespace hailort -{ - -PcieInputStream::PcieInputStream( - PcieDevice &device, - std::shared_ptr channel, - const LayerInfo &edge_layer, - EventPtr network_group_activated_event, - uint16_t batch_size, - const std::chrono::milliseconds &transfer_timeout, - hailo_status &status) : - VdmaInputStream(device, std::move(channel), edge_layer, network_group_activated_event, - batch_size, transfer_timeout, HAILO_STREAM_INTERFACE_PCIE, status) - {} - -Expected> PcieInputStream::create(Device &device, - std::shared_ptr channel, const LayerInfo &edge_layer, - uint16_t batch_size, EventPtr network_group_activated_event) -{ - hailo_status status = HAILO_UNINITIALIZED; - - PcieDevice *pcie_device = reinterpret_cast(&device); - std::unique_ptr local_stream(new (std::nothrow) PcieInputStream(*pcie_device, - std::move(channel), edge_layer, std::move(network_group_activated_event), batch_size, - DEFAULT_TRANSFER_TIMEOUT, status)); - CHECK((nullptr != local_stream), make_unexpected(HAILO_OUT_OF_HOST_MEMORY)); - CHECK_SUCCESS_AS_EXPECTED(status); - - return local_stream; -} - -Expected> PcieOutputStream::create(Device &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event) -{ - hailo_status status = HAILO_UNINITIALIZED; - PcieDevice *pcie_device = reinterpret_cast(&device); - - std::unique_ptr local_stream(new (std::nothrow) PcieOutputStream(*pcie_device, - std::move(channel), edge_layer, std::move(network_group_activated_event), - batch_size, DEFAULT_TRANSFER_TIMEOUT, status)); - CHECK((nullptr != local_stream), make_unexpected(HAILO_OUT_OF_HOST_MEMORY)); - CHECK_SUCCESS_AS_EXPECTED(status); - - return local_stream; -} - -PcieOutputStream::PcieOutputStream( - PcieDevice &device, - std::shared_ptr channel, - const LayerInfo &edge_layer, - EventPtr network_group_activated_event, - uint16_t batch_size, - const std::chrono::milliseconds &transfer_timeout, - hailo_status &status) : - VdmaOutputStream(device, std::move(channel), edge_layer, - network_group_activated_event, batch_size, transfer_timeout, status) - {} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/pcie_stream.hpp b/hailort/libhailort/src/pcie_stream.hpp deleted file mode 100644 index 5bd5035..0000000 --- a/hailort/libhailort/src/pcie_stream.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file pcie_stream.hpp - * @brief Stream object for PCIe device - **/ - -#ifndef _HAILO_PCIE_STREAM_H_ -#define _HAILO_PCIE_STREAM_H_ - -#include "vdma_stream.hpp" -#include "pcie_device.hpp" - -namespace hailort -{ - -class PcieInputStream : public VdmaInputStream { -public: - PcieInputStream(PcieInputStream &&other) = default; - virtual ~PcieInputStream() = default; - - static Expected> create(Device &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event); - - virtual hailo_stream_interface_t get_interface() const override { return HAILO_STREAM_INTERFACE_PCIE; } - -private: - PcieInputStream( - PcieDevice &device, - std::shared_ptr channel, - const LayerInfo &edge_layer, - EventPtr network_group_activated_event, - uint16_t batch_size, - const std::chrono::milliseconds &transfer_timeout, - hailo_status &status); -}; - -class PcieOutputStream : public VdmaOutputStream { -public: - PcieOutputStream(PcieOutputStream &&other) = default; - virtual ~PcieOutputStream() = default; - - static Expected> create(Device &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event); - - virtual hailo_stream_interface_t get_interface() const override { return HAILO_STREAM_INTERFACE_PCIE; } - -private: - explicit PcieOutputStream( - PcieDevice &device, - std::shared_ptr channel, - const LayerInfo &edge_layer, - EventPtr network_group_activated_event, - uint16_t batch_size, - const std::chrono::milliseconds &transfer_timeout, - hailo_status &status); -}; - -} /* namespace hailort */ - -#endif /* _HAILO_PCIE_STREAM_H_ */ diff --git a/hailort/libhailort/src/pipeline_multiplexer.cpp b/hailort/libhailort/src/pipeline_multiplexer.cpp deleted file mode 100644 index 4d350a7..0000000 --- a/hailort/libhailort/src/pipeline_multiplexer.cpp +++ /dev/null @@ -1,417 +0,0 @@ -/** - * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file pipeline_multiplexer.cpp - * @brief: Pipeline Multiplexer - **/ - -#include "pipeline_multiplexer.hpp" -#include "common/utils.hpp" -#include "hailo/hailort_common.hpp" - -namespace hailort -{ - -PipelineMultiplexer::PipelineMultiplexer() : - m_next_to_write(0), - m_order_queue(), - m_currently_writing(INVALID_NETWORK_GROUP_HANDLE), - m_written_streams_count(0), - m_read_streams_count(0), - m_next_to_read_after_drain(INVALID_NETWORK_GROUP_HANDLE) -{} - -bool PipelineMultiplexer::should_use_multiplexer() -{ - auto disable_multiplexer_env = std::getenv(DISABLE_MULTIPLEXER_ENV_VAR); - if ((nullptr != disable_multiplexer_env) && (strnlen(disable_multiplexer_env, 2) == 1) && (strncmp(disable_multiplexer_env, "1", 1) == 0)) { - return false; - } - return true; -} - -hailo_status PipelineMultiplexer::add_network_group_instance(multiplexer_ng_handle_t network_group_handle, ConfiguredNetworkGroup &network_group) -{ - std::unique_lock lock(m_writing_mutex); - std::unique_lock read_lock(m_reading_mutex); - assert(!contains(m_should_ng_stop, network_group_handle)); - - m_should_ng_stop[network_group_handle] = false; - - m_input_streams_count = static_cast(network_group.get_input_streams().size()); - m_output_streams_count = static_cast(network_group.get_output_streams().size()); - - m_write_barriers[network_group_handle] = make_shared_nothrow(m_input_streams_count); - CHECK(nullptr != m_write_barriers[network_group_handle], HAILO_OUT_OF_HOST_MEMORY); - m_is_waiting_to_write[network_group_handle] = false; - - for (auto &output_stream : network_group.get_output_streams()) { - m_is_stream_reading[network_group_handle][output_stream.get().name()] = false; - } - - return HAILO_SUCCESS; -} - -void PipelineMultiplexer::set_output_vstreams_names(multiplexer_ng_handle_t network_group_handle, const std::vector &output_vstreams) -{ - std::unique_lock lock(m_writing_mutex); - for (const auto &output_vstream : output_vstreams) { - m_can_output_vstream_read[network_group_handle][output_vstream.name()] = true; - } - m_can_network_group_read[network_group_handle] = true; -} - -bool PipelineMultiplexer::has_more_than_one_ng_instance() const -{ - return instances_count() > 1; -} - -size_t PipelineMultiplexer::instances_count() const -{ - return m_should_ng_stop.size(); -} - -hailo_status PipelineMultiplexer::wait_for_write(multiplexer_ng_handle_t network_group_handle) -{ - std::shared_ptr barrier; - { - std::unique_lock lock(m_writing_mutex); - assert(contains(m_write_barriers, network_group_handle)); - barrier = m_write_barriers[network_group_handle]; - } - // TODO: This has no timeout - // TODO: HRT-8634 - barrier->arrive_and_wait(); - { - std::unique_lock lock(m_writing_mutex); - assert(contains(m_should_ng_stop, network_group_handle)); - assert(contains(m_is_waiting_to_write, network_group_handle)); - - m_is_waiting_to_write[network_group_handle] = true; - m_writing_cv.wait(lock, [this, network_group_handle] { - if (!has_more_than_one_ng_instance() || !should_use_multiplexer()) { - return true; - } - - if (m_should_ng_stop[network_group_handle]) { - return true; - } - - if (m_currently_writing == network_group_handle) { - return true; - } - - if (!can_network_group_read(network_group_handle)) { - return false; - } - - if (INVALID_NETWORK_GROUP_HANDLE == m_currently_writing) { - if ((m_next_to_write != network_group_handle) && m_is_waiting_to_write[m_next_to_write] && can_network_group_read(m_next_to_write)) { - return false; - } - - return true; - } - - return false; - }); - m_is_waiting_to_write[network_group_handle] = false; - - if (m_should_ng_stop[network_group_handle]) { - return HAILO_STREAM_ABORTED_BY_USER; - } - - if (INVALID_NETWORK_GROUP_HANDLE == m_currently_writing) { - m_currently_writing = network_group_handle; - m_next_to_write = m_currently_writing; - } - } - m_writing_cv.notify_all(); - - return HAILO_SUCCESS; -} - -bool PipelineMultiplexer::can_network_group_read(multiplexer_ng_handle_t network_group_handle) -{ - if (m_should_ng_stop[network_group_handle]) { - return false; - } - - if (!contains(m_can_network_group_read, network_group_handle)) { - return true; - } - - return m_can_network_group_read[network_group_handle]; -} - -hailo_status PipelineMultiplexer::signal_write_finish(multiplexer_ng_handle_t network_group_handle) -{ - std::unique_lock lock(m_writing_mutex); - m_written_streams_count++; - if (m_written_streams_count == m_input_streams_count) { - m_written_streams_count = 0; - m_currently_writing = INVALID_NETWORK_GROUP_HANDLE; - - m_next_to_write++; - m_next_to_write %= static_cast(instances_count()); - - { - std::unique_lock reading_lock(m_reading_mutex); - m_order_queue.push_back(network_group_handle); - } - m_reading_cv.notify_all(); - - lock.unlock(); - m_writing_cv.notify_all(); - } - - return HAILO_SUCCESS; -} - -Expected PipelineMultiplexer::wait_for_read(multiplexer_ng_handle_t network_group_handle, const std::string &stream_name, - const std::chrono::milliseconds &timeout) -{ - std::unique_lock lock(m_reading_mutex); - uint32_t drain_frames = 0; - - assert(contains(m_should_ng_stop, network_group_handle)); - assert(contains(m_is_stream_reading, network_group_handle)); - assert(contains(m_is_stream_reading[network_group_handle], stream_name)); - - auto wait_res = m_reading_cv.wait_for(lock, timeout, [this, network_group_handle, stream_name, &drain_frames] { - if (m_should_ng_stop[network_group_handle]) { - return true; - } - - if (m_is_stream_reading[network_group_handle][stream_name]) { - return false; - } - - if (m_next_to_read_after_drain == network_group_handle) { - drain_frames = m_num_frames_to_drain[stream_name]; - return true; - } - - if (m_order_queue.empty()) { - return false; - } - - if (m_order_queue.front() != network_group_handle) { - if (!m_should_ng_stop[m_order_queue.front()]) { - return false; - } - - uint32_t max_drain_count = get_frame_count_to_drain(network_group_handle); - if (0 == max_drain_count) { - return false; - } - - drain_frames = drain_aborted_in_order_queue(network_group_handle, stream_name, max_drain_count); - } - - return true; - }); - CHECK_AS_EXPECTED(wait_res, HAILO_TIMEOUT, "{} (D2H) failed with status={}, timeout={}ms", stream_name, HAILO_TIMEOUT, timeout.count()); - - if (m_should_ng_stop[network_group_handle]) { - return make_unexpected(HAILO_STREAM_ABORTED_BY_USER); - } - - m_is_stream_reading[network_group_handle][stream_name] = true; - - return drain_frames; -} - -uint32_t PipelineMultiplexer::get_frame_count_to_drain(multiplexer_ng_handle_t network_group_handle) -{ - uint32_t drain_count = 0; - for (const auto &handle : m_order_queue) { - if (!m_should_ng_stop[handle]) { - if (handle == network_group_handle) { - // Current instance is in the front after draining - break; - } else { - // Someone else should drain these frames, the current instance won't be in front after draining - return 0; - } - } - - drain_count++; - } - - return drain_count; -} - -uint32_t PipelineMultiplexer::drain_aborted_in_order_queue(multiplexer_ng_handle_t network_group_handle, const std::string &stream_name, - uint32_t max_drain_count) -{ - // In case of multiple outputs where one or more already read the frame we need to drain one less frame - for (auto &name_flag_pair : m_is_stream_reading[m_order_queue.front()]) { - if (name_flag_pair.second) { - m_num_frames_to_drain[name_flag_pair.first] = max_drain_count - 1; - } else { - m_num_frames_to_drain[name_flag_pair.first] = max_drain_count; - } - } - - m_next_to_read_after_drain = network_group_handle; - m_read_streams_count = 0; - for (uint32_t i = 0; i < max_drain_count; i++) { - for (auto &name_flag_pair : m_is_stream_reading[m_order_queue.front()]) { - name_flag_pair.second = false; - } - m_order_queue.pop_front(); - } - - return m_num_frames_to_drain[stream_name]; -} - -hailo_status PipelineMultiplexer::signal_read_finish(multiplexer_ng_handle_t network_group_handle) -{ - std::unique_lock lock(m_reading_mutex); - assert(contains(m_is_stream_reading, network_group_handle)); - - if (m_should_ng_stop[network_group_handle]) { - return HAILO_STREAM_ABORTED_BY_USER; - } - - m_read_streams_count++; - if (m_read_streams_count == m_output_streams_count) { - m_read_streams_count = 0; - m_order_queue.pop_front(); - for (auto &name_flag_pair : m_is_stream_reading[network_group_handle]) { - name_flag_pair.second = false; - } - - m_next_to_read_after_drain = INVALID_NETWORK_GROUP_HANDLE; - - lock.unlock(); - m_reading_cv.notify_all(); - } - - return HAILO_SUCCESS; -} - -hailo_status PipelineMultiplexer::enable_network_group(multiplexer_ng_handle_t network_group_handle) -{ - { - std::unique_lock write_lock(m_writing_mutex); - std::unique_lock read_lock(m_reading_mutex); - assert(contains(m_should_ng_stop, network_group_handle)); - if (!m_should_ng_stop[network_group_handle]) { - return HAILO_SUCCESS; - } - - m_should_ng_stop[network_group_handle] = false; - } - - m_writing_cv.notify_all(); - m_reading_cv.notify_all(); - - return HAILO_SUCCESS; -} - -hailo_status PipelineMultiplexer::disable_network_group(multiplexer_ng_handle_t network_group_handle) -{ - { - std::unique_lock write_lock(m_writing_mutex); - std::unique_lock read_lock(m_reading_mutex); - assert(contains(m_should_ng_stop, network_group_handle)); - if (m_should_ng_stop[network_group_handle]) { - return HAILO_SUCCESS; - } - - m_should_ng_stop[network_group_handle] = true; - if (m_currently_writing == network_group_handle) { - m_currently_writing = INVALID_NETWORK_GROUP_HANDLE; - } - - assert(contains(m_write_barriers, network_group_handle)); - m_write_barriers[network_group_handle]->terminate(); - } - - m_writing_cv.notify_all(); - m_reading_cv.notify_all(); - - return HAILO_SUCCESS; -} - -void PipelineMultiplexer::RunOnceForStream::add_instance() -{ - std::unique_lock lock(m_mutex); - m_was_called[static_cast(m_was_called.size())] = false; -} - -void PipelineMultiplexer::RunOnceForStream::set_callback(std::function callback) -{ - std::unique_lock lock(m_mutex); - m_callback = callback; -} - -hailo_status PipelineMultiplexer::RunOnceForStream::run(multiplexer_ng_handle_t network_group_handle) -{ - std::unique_lock lock(m_mutex); - assert(contains(m_was_called, network_group_handle)); - - m_was_called[network_group_handle] = true; - for (auto &handle_flag_pair : m_was_called) { - if (!handle_flag_pair.second) { - return HAILO_SUCCESS; - } - } - - for (auto &handle_flag_pair : m_was_called) { - handle_flag_pair.second = false; - } - - return m_callback(); -} - -hailo_status PipelineMultiplexer::register_run_once_for_stream(const std::string &stream_name, run_once_for_stream_handle_t handle, - std::function callback) -{ - std::unique_lock lock(m_register_run_once_mutex); - if (!contains(m_run_once_db[stream_name], handle)) { - m_run_once_db[stream_name][handle] = make_shared_nothrow(); - CHECK(nullptr != m_run_once_db[stream_name][handle], HAILO_OUT_OF_HOST_MEMORY); - - m_run_once_db[stream_name][handle]->set_callback(callback); - } - - m_run_once_db[stream_name][handle]->add_instance(); - - return HAILO_SUCCESS; -} - -hailo_status PipelineMultiplexer::run_once_for_stream(const std::string &stream_name, run_once_for_stream_handle_t run_once_handle, - multiplexer_ng_handle_t network_group_handle) -{ - return m_run_once_db[stream_name][run_once_handle]->run(network_group_handle); -} - -void PipelineMultiplexer::set_can_output_vstream_read(multiplexer_ng_handle_t network_group_handle, const std::string &vstream_name, bool can_read) -{ - { - std::unique_lock lock(m_writing_mutex); - assert(contains(m_can_output_vstream_read, network_group_handle)); - assert(contains(m_can_output_vstream_read[network_group_handle], vstream_name)); - assert(contains(m_can_network_group_read, network_group_handle)); - - m_can_output_vstream_read[network_group_handle][vstream_name] = can_read; - - if (can_read != m_can_network_group_read[network_group_handle]) { - m_can_network_group_read[network_group_handle] = true; - for (const auto &name_bool_pair : m_can_output_vstream_read[network_group_handle]) { - if (!name_bool_pair.second) { - m_can_network_group_read[network_group_handle] = false; - break; - } - } - } - } - m_writing_cv.notify_all(); -} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/rpc_client_utils.hpp b/hailort/libhailort/src/rpc_client_utils.hpp deleted file mode 100644 index 7336248..0000000 --- a/hailort/libhailort/src/rpc_client_utils.hpp +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file hailort_common.hpp - * @brief Utility functions for rpc client communication - **/ - -#ifndef _HAILO_HAILORT_RPC_CLIENT_UTILS_HPP_ -#define _HAILO_HAILORT_RPC_CLIENT_UTILS_HPP_ - -#include "hailo/hailort.h" -#include "hailo/expected.hpp" -#include "hailort_defaults.hpp" -#include "common/async_thread.hpp" -#include "hailort_rpc_client.hpp" -#include "rpc/rpc_definitions.hpp" -#include - -namespace hailort -{ - -class HailoRtRpcClientUtils final -{ -public: - static HailoRtRpcClientUtils& get_instance() - { - static HailoRtRpcClientUtils instance; - return instance; - } - - hailo_status init_client_service_communication() - { - std::unique_lock lock(m_mutex); - if (!m_initialized) { - - auto channel = grpc::CreateChannel(hailort::HAILO_DEFAULT_UDS_ADDR, grpc::InsecureChannelCredentials()); - auto client = make_shared_nothrow(channel); - CHECK(client != nullptr, HAILO_OUT_OF_HOST_MEMORY); - m_initialized = true; - auto reply = client->get_service_version(); - CHECK_EXPECTED_AS_STATUS(reply); - hailo_version_t client_version = {}; - auto status = hailo_get_library_version(&client_version); - CHECK_SUCCESS(status); - auto service_version = reply.value(); - auto are_equal = [](auto version1, auto version2) { - return version1.major == version2.major - && version1.minor == version2.minor - && version1.revision == version2.revision; - }; - CHECK(are_equal(service_version, client_version), HAILO_INVALID_SERVICE_VERSION, "Invalid libhailort version on service: " - "client version {}.{}.{}, service version {}.{}.{}", - service_version.major, service_version.minor, service_version.revision, - client_version.major, client_version.minor, client_version.revision); - - m_keep_alive_thread = make_unique_nothrow>([client] () { - auto pid = getpid(); - auto status = client->client_keep_alive(pid); - CHECK_SUCCESS(status); - return HAILO_SUCCESS; - }); - - } - return HAILO_SUCCESS; - } - -private: - ~HailoRtRpcClientUtils() - { - m_keep_alive_thread.release(); - } - - std::mutex m_mutex; - AsyncThreadPtr m_keep_alive_thread; - bool m_initialized = false; -}; - -} /* namespace hailort */ - -#endif /* _HAILO_HAILORT_RPC_CLIENT_UTILS_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/scheduled_stream.hpp b/hailort/libhailort/src/scheduled_stream.hpp deleted file mode 100644 index 1cec64d..0000000 --- a/hailort/libhailort/src/scheduled_stream.hpp +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file scheduled_stream.hpp - * @brief Internal stream implementation for scheduled streams - * - **/ - -#ifndef HAILO_SCHEDULED_STREAM_HPP_ -#define HAILO_SCHEDULED_STREAM_HPP_ - -#include "stream_internal.hpp" -#include "hailo/hailort.h" -#include "vdevice_internal.hpp" -#include "vdma_device.hpp" -#include "vdevice_stream.hpp" -#include "hailo/expected.hpp" - -namespace hailort -{ - -class ScheduledInputStream : public InputVDeviceBaseStream { -public: - ScheduledInputStream(ScheduledInputStream &&other) : - InputVDeviceBaseStream(std::move(other)), - m_network_group_handle(std::move(other.m_network_group_handle)), - m_network_group_scheduler(std::move(other.m_network_group_scheduler)) - {} - - explicit ScheduledInputStream( - std::vector> &&streams, - const scheduler_ng_handle_t &network_group_handle, - EventPtr &&network_group_activated_event, - const LayerInfo &layer_info, - NetworkGroupSchedulerWeakPtr network_group_scheduler, - hailo_status &status) : - InputVDeviceBaseStream(std::move(streams), std::move(network_group_activated_event), layer_info, status), - m_network_group_handle(network_group_handle), - m_network_group_scheduler(network_group_scheduler) - {} - - virtual hailo_status abort() override; - virtual hailo_status clear_abort() override; - virtual bool is_scheduled() override { return true; }; - - virtual void notify_all() override - { - auto scheduler = m_network_group_scheduler.lock(); - if (nullptr == scheduler) { - LOGGER__CRITICAL("Failed to acquire scheduler"); - return; - } - scheduler->notify_all(); - - for (auto &stream : m_streams) { - stream.get().notify_all(); - } - } - -protected: - virtual Expected sync_write_raw_buffer(const MemoryView &buffer, - const std::function &should_cancel = []() { return false; }); - - Expected sync_write_raw_buffer_impl(const MemoryView &buffer, scheduler_ng_handle_t network_group_handle, - const std::function &should_cancel); - - scheduler_ng_handle_t m_network_group_handle; - NetworkGroupSchedulerWeakPtr m_network_group_scheduler; - -private: - hailo_status abort_impl(scheduler_ng_handle_t network_group_handle); - hailo_status clear_abort_impl(scheduler_ng_handle_t network_group_handle); -}; - -class ScheduledOutputStream : public OutputVDeviceBaseStream { -public: - ScheduledOutputStream(ScheduledOutputStream &&other) : - OutputVDeviceBaseStream(std::move(other)), - m_network_group_handle(std::move(other.m_network_group_handle)), - m_network_group_scheduler(std::move(other.m_network_group_scheduler)) - {} - - explicit ScheduledOutputStream( - std::vector> &&streams, - const scheduler_ng_handle_t &network_group_handle, - const LayerInfo &layer_info, - EventPtr &&network_group_activated_event, - NetworkGroupSchedulerWeakPtr network_group_scheduler, - hailo_status &status) : - OutputVDeviceBaseStream(std::move(streams), layer_info, std::move(network_group_activated_event), status), - m_network_group_handle(network_group_handle), - m_network_group_scheduler(network_group_scheduler) - {} - - virtual hailo_status abort() override; - virtual hailo_status clear_abort() override; - virtual bool is_scheduled() override { return true; }; - -protected: - virtual hailo_status read(MemoryView buffer) override; - hailo_status read_impl(MemoryView buffer, scheduler_ng_handle_t network_group_handle); - - scheduler_ng_handle_t m_network_group_handle; - NetworkGroupSchedulerWeakPtr m_network_group_scheduler; - -private: - hailo_status abort_impl(scheduler_ng_handle_t network_group_handle); - hailo_status clear_abort_impl(scheduler_ng_handle_t network_group_handle); -}; - -} /* namespace hailort */ - -#endif /* HAILO_SCHEDULED_STREAM_HPP_ */ diff --git a/hailort/libhailort/src/scheduler_oracle.cpp b/hailort/libhailort/src/scheduler_oracle.cpp deleted file mode 100644 index a375bbe..0000000 --- a/hailort/libhailort/src/scheduler_oracle.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file scheduler_oracle.cpp - * @brief: - **/ - -#include "scheduler_oracle.hpp" -#include "tracer_macros.hpp" - -namespace hailort -{ - -bool NetworkGroupSchedulerOracle::choose_next_model(NetworkGroupScheduler &scheduler, uint32_t device_id) -{ - auto cngs_size = scheduler.m_cngs.size(); - auto& device_info = scheduler.m_devices[device_id]; - for (uint32_t i = 0; i < cngs_size; i++) { - uint32_t index = scheduler.m_last_choosen_network_group + i + 1; - index %= static_cast(cngs_size); - auto ready_info = scheduler.is_network_group_ready(index, true, device_id); - if (ready_info.is_ready) { - TRACE(ChooseNetworkGroupTrace, "", index, ready_info.threshold, ready_info.timeout); - device_info->is_switching_network_group = true; - device_info->next_network_group_handle = index; - scheduler.m_last_choosen_network_group = index; - return true; - } - } - return false; -} - -// TODO: return device handle instead index -uint32_t NetworkGroupSchedulerOracle::get_avail_device(NetworkGroupScheduler &scheduler, scheduler_ng_handle_t network_group_handle) -{ - const bool check_threshold = false; - - // Check if should be next - /* Checking (INVALID_NETWORK_GROUP_HANDLE == m_current_network_group) for activating the first time the scheduler is running. - In this case we don't want to check threshold. */ - for (auto active_device_info : scheduler.m_devices) { - if (active_device_info->is_switching_network_group && scheduler.has_ng_drained_everything(active_device_info->current_network_group_handle, active_device_info->device_id) && - (((INVALID_NETWORK_GROUP_HANDLE == active_device_info->current_network_group_handle) && - scheduler.is_network_group_ready(network_group_handle, check_threshold, active_device_info->device_id).is_ready) || - (active_device_info->next_network_group_handle == network_group_handle))) { - return active_device_info->device_id; - } - } - - // Check if device Idle with this network active - for (auto active_device_info : scheduler.m_devices) { - if ((active_device_info->current_network_group_handle == network_group_handle) && !active_device_info->is_switching_network_group && - scheduler.has_ng_drained_everything(active_device_info->current_network_group_handle, active_device_info->device_id) && - scheduler.is_network_group_ready(network_group_handle, check_threshold, active_device_info->device_id).is_ready) { - scheduler.m_last_choosen_network_group = network_group_handle; - return active_device_info->device_id; - } - } - - // Check if device Idle - for (auto active_device_info : scheduler.m_devices) { - if (!active_device_info->is_switching_network_group && scheduler.has_ng_drained_everything(active_device_info->current_network_group_handle, active_device_info->device_id) && - scheduler.is_network_group_ready(network_group_handle, check_threshold, active_device_info->device_id).is_ready) { - scheduler.m_last_choosen_network_group = network_group_handle; - return active_device_info->device_id; - } - } - - return INVALID_DEVICE_ID; -} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/scheduler_oracle.hpp b/hailort/libhailort/src/scheduler_oracle.hpp deleted file mode 100644 index df1bedf..0000000 --- a/hailort/libhailort/src/scheduler_oracle.hpp +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file scheduler_oracle.hpp - * @brief - **/ - -#ifndef _HAILO_SCHEDULER_ORACLE_HPP_ -#define _HAILO_SCHEDULER_ORACLE_HPP_ - -#include "hailo/hailort.h" -#include "hailo/expected.hpp" -#include "hailo/network_group.hpp" -#include "common/utils.hpp" -#include "network_group_scheduler.hpp" - -namespace hailort -{ - -class NetworkGroupSchedulerOracle -{ -public: - static bool choose_next_model(NetworkGroupScheduler &scheduler, uint32_t device_id); - static uint32_t get_avail_device(NetworkGroupScheduler &scheduler, scheduler_ng_handle_t network_group_handle); - -private: - NetworkGroupSchedulerOracle() {} -}; - -} /* namespace hailort */ - -#endif /* _HAILO_SCHEDULER_ORACLE_HPP_ */ diff --git a/hailort/libhailort/src/service/CMakeLists.txt b/hailort/libhailort/src/service/CMakeLists.txt new file mode 100644 index 0000000..fd42ad3 --- /dev/null +++ b/hailort/libhailort/src/service/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/hailort_rpc_client.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/network_group_client.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/hailort_rpc_client.cpp b/hailort/libhailort/src/service/hailort_rpc_client.cpp similarity index 84% rename from hailort/libhailort/src/hailort_rpc_client.cpp rename to hailort/libhailort/src/service/hailort_rpc_client.cpp index 4dc03cb..30340aa 100644 --- a/hailort/libhailort/src/hailort_rpc_client.cpp +++ b/hailort/libhailort/src/service/hailort_rpc_client.cpp @@ -7,9 +7,10 @@ * @brief Implementation of the hailort rpc client **/ -#include "hailort_rpc_client.hpp" #include "common/utils.hpp" -#include "hef_internal.hpp" + +#include "hef/hef_internal.hpp" +#include "hailort_rpc_client.hpp" #include @@ -17,10 +18,10 @@ namespace hailort { -hailo_status HailoRtRpcClient::client_keep_alive(uint32_t process_id) +hailo_status HailoRtRpcClient::client_keep_alive(uint32_t pid) { keepalive_Request request; - request.set_process_id(process_id); + request.set_pid(pid); empty reply; grpc::ClientContext context; grpc::Status status = m_stub->client_keep_alive(&context, request, &reply); @@ -65,6 +66,18 @@ Expected HailoRtRpcClient::VDevice_create(const hailo_vdevice_params_t return reply.handle(); } +Expected HailoRtRpcClient::VDevice_dup_handle(uint32_t pid, uint32_t handle) +{ + dup_handle_Request request; + request.set_pid(pid); + request.set_handle(handle); + dup_handle_Reply reply; + grpc::ClientContext context; + grpc::Status status = m_stub->VDevice_dup_handle(&context, request, &reply); + CHECK_GRPC_STATUS_AS_EXPECTED(status); + return reply.handle(); +} + hailo_status HailoRtRpcClient::VDevice_release(uint32_t handle) { Release_Request request; @@ -193,6 +206,30 @@ hailo_status HailoRtRpcClient::OutputVStream_release(uint32_t handle) return HAILO_SUCCESS; } +Expected HailoRtRpcClient::InputVStream_dup_handle(uint32_t pid, uint32_t handle) +{ + dup_handle_Request request; + request.set_pid(pid); + request.set_handle(handle); + dup_handle_Reply reply; + grpc::ClientContext context; + grpc::Status status = m_stub->InputVStream_dup_handle(&context, request, &reply); + CHECK_GRPC_STATUS_AS_EXPECTED(status); + return reply.handle(); +} + +Expected HailoRtRpcClient::OutputVStream_dup_handle(uint32_t pid, uint32_t handle) +{ + dup_handle_Request request; + request.set_pid(pid); + request.set_handle(handle); + dup_handle_Reply reply; + grpc::ClientContext context; + grpc::Status status = m_stub->OutputVStream_dup_handle(&context, request, &reply); + CHECK_GRPC_STATUS_AS_EXPECTED(status); + return reply.handle(); +} + Expected> HailoRtRpcClient::VDevice_configure(uint32_t vdevice_handle, const Hef &hef, uint32_t pid, const NetworkGroupsParamsMap &configure_params) { @@ -222,6 +259,7 @@ Expected> HailoRtRpcClient::VDevice_configure(uint32_t vde auto stream_params = name_stream_params_pair.second; proto_stream_params->set_stream_interface(stream_params.stream_interface); proto_stream_params->set_direction(stream_params.direction); + proto_stream_params->set_flags(stream_params.flags); } // Init network params map @@ -280,6 +318,18 @@ Expected HailoRtRpcClient::VDevice_get_default_streams return static_cast(reply.stream_interface()); } +Expected HailoRtRpcClient::ConfiguredNetworkGroup_dup_handle(uint32_t pid, uint32_t handle) +{ + dup_handle_Request request; + request.set_pid(pid); + request.set_handle(handle); + dup_handle_Reply reply; + grpc::ClientContext context; + grpc::Status status = m_stub->ConfiguredNetworkGroup_dup_handle(&context, request, &reply); + CHECK_GRPC_STATUS_AS_EXPECTED(status); + return reply.handle(); +} + hailo_status HailoRtRpcClient::ConfiguredNetworkGroup_release(uint32_t handle) { Release_Request request; @@ -302,16 +352,16 @@ std::map get_group(const ProtoNamedVStreamP auto proto_params = named_params.params(); auto proto_user_buffer_format = proto_params.user_buffer_format(); hailo_format_t user_buffer_format = { - .type = static_cast(proto_user_buffer_format.type()), - .order = static_cast(proto_user_buffer_format.order()), - .flags = static_cast(proto_user_buffer_format.flags()) + static_cast(proto_user_buffer_format.type()), + static_cast(proto_user_buffer_format.order()), + static_cast(proto_user_buffer_format.flags()) }; hailo_vstream_params_t params = { - .user_buffer_format = user_buffer_format, - .timeout_ms = proto_params.timeout_ms(), - .queue_size = proto_params.queue_size(), - .vstream_stats_flags = static_cast(proto_params.vstream_stats_flags()), - .pipeline_elements_stats_flags = static_cast(proto_params.pipeline_elements_stats_flags()) + user_buffer_format, + proto_params.timeout_ms(), + proto_params.queue_size(), + static_cast(proto_params.vstream_stats_flags()), + static_cast(proto_params.pipeline_elements_stats_flags()) }; result.insert({name, params}); } @@ -387,16 +437,16 @@ Expected> HailoRtRpcClient::Config auto proto_params = reply.vstream_params_map().vstream_params_map(i).params(); auto proto_user_buffer_format = proto_params.user_buffer_format(); hailo_format_t user_buffer_format = { - .type = static_cast(proto_user_buffer_format.type()), - .order = static_cast(proto_user_buffer_format.order()), - .flags = static_cast(proto_user_buffer_format.flags()) + static_cast(proto_user_buffer_format.type()), + static_cast(proto_user_buffer_format.order()), + static_cast(proto_user_buffer_format.flags()) }; hailo_vstream_params_t params = { - .user_buffer_format = user_buffer_format, - .timeout_ms = proto_params.timeout_ms(), - .queue_size = proto_params.queue_size(), - .vstream_stats_flags = static_cast(proto_params.vstream_stats_flags()), - .pipeline_elements_stats_flags = static_cast(proto_params.pipeline_elements_stats_flags()) + user_buffer_format, + proto_params.timeout_ms(), + proto_params.queue_size(), + static_cast(proto_params.vstream_stats_flags()), + static_cast(proto_params.pipeline_elements_stats_flags()) }; result.insert({name, params}); } @@ -462,38 +512,38 @@ Expected> HailoRtRpcClient::ConfiguredNetworkGr result.reserve(reply.stream_infos().size()); for (auto proto_stream_info : reply.stream_infos()) { hailo_3d_image_shape_t shape{ - .height = proto_stream_info.stream_shape().shape().height(), - .width = proto_stream_info.stream_shape().shape().width(), - .features = proto_stream_info.stream_shape().shape().features(), + proto_stream_info.stream_shape().shape().height(), + proto_stream_info.stream_shape().shape().width(), + proto_stream_info.stream_shape().shape().features(), }; hailo_3d_image_shape_t hw_shape{ - .height = proto_stream_info.stream_shape().hw_shape().height(), - .width = proto_stream_info.stream_shape().hw_shape().width(), - .features = proto_stream_info.stream_shape().hw_shape().features(), + proto_stream_info.stream_shape().hw_shape().height(), + proto_stream_info.stream_shape().hw_shape().width(), + proto_stream_info.stream_shape().hw_shape().features(), }; hailo_nms_defuse_info_t nms_defuse_info{ - .class_group_index = proto_stream_info.nms_info().defuse_info().class_group_index(), - .original_name = {0} + proto_stream_info.nms_info().defuse_info().class_group_index(), + {0} }; strcpy(nms_defuse_info.original_name, proto_stream_info.nms_info().defuse_info().original_name().c_str()); hailo_nms_info_t nms_info{ - .number_of_classes = proto_stream_info.nms_info().number_of_classes(), - .max_bboxes_per_class = proto_stream_info.nms_info().max_bboxes_per_class(), - .bbox_size = proto_stream_info.nms_info().bbox_size(), - .chunks_per_frame = proto_stream_info.nms_info().chunks_per_frame(), - .is_defused = proto_stream_info.nms_info().is_defused(), - .defuse_info = nms_defuse_info, + proto_stream_info.nms_info().number_of_classes(), + proto_stream_info.nms_info().max_bboxes_per_class(), + proto_stream_info.nms_info().bbox_size(), + proto_stream_info.nms_info().chunks_per_frame(), + proto_stream_info.nms_info().is_defused(), + nms_defuse_info, }; hailo_format_t format{ - .type = static_cast(proto_stream_info.format().type()), - .order = static_cast(proto_stream_info.format().order()), - .flags = static_cast(proto_stream_info.format().flags()) + static_cast(proto_stream_info.format().type()), + static_cast(proto_stream_info.format().order()), + static_cast(proto_stream_info.format().flags()) }; hailo_quant_info_t quant_info{ - .qp_zp = proto_stream_info.quant_info().qp_zp(), - .qp_scale = proto_stream_info.quant_info().qp_scale(), - .limvals_min = proto_stream_info.quant_info().limvals_min(), - .limvals_max = proto_stream_info.quant_info().limvals_max() + proto_stream_info.quant_info().qp_zp(), + proto_stream_info.quant_info().qp_scale(), + proto_stream_info.quant_info().limvals_min(), + proto_stream_info.quant_info().limvals_max() }; hailo_stream_info_t stream_info; if (format.order == HAILO_FORMAT_ORDER_HAILO_NMS) { @@ -562,30 +612,30 @@ hailo_vstream_info_t deserialize_vstream_info(const ProtoVStreamInfo &info_proto strcpy(info.network_name, info_proto.network_name().c_str()); info.direction = static_cast(info_proto.direction()); hailo_format_t format = { - .type = static_cast(info_proto.format().type()), - .order = static_cast(info_proto.format().order()), - .flags = static_cast(info_proto.format().flags()) + static_cast(info_proto.format().type()), + static_cast(info_proto.format().order()), + static_cast(info_proto.format().flags()) }; info.format = format; if (format.order == HAILO_FORMAT_ORDER_HAILO_NMS) { hailo_nms_shape_t nms_shape = { - .number_of_classes = info_proto.nms_shape().number_of_classes(), - .max_bboxes_per_class = info_proto.nms_shape().max_bbox_per_class() + info_proto.nms_shape().number_of_classes(), + info_proto.nms_shape().max_bbox_per_class() }; info.nms_shape = nms_shape; } else { hailo_3d_image_shape_t shape = { - .height = info_proto.shape().height(), - .width = info_proto.shape().width(), - .features = info_proto.shape().features() + info_proto.shape().height(), + info_proto.shape().width(), + info_proto.shape().features() }; info.shape = shape; } hailo_quant_info_t quant_info = { - .qp_zp = info_proto.quant_info().qp_zp(), - .qp_scale = info_proto.quant_info().qp_scale(), - .limvals_min = info_proto.quant_info().limvals_min(), - .limvals_max = info_proto.quant_info().limvals_max() + info_proto.quant_info().qp_zp(), + info_proto.quant_info().qp_scale(), + info_proto.quant_info().limvals_min(), + info_proto.quant_info().limvals_max() }; info.quant_info = quant_info; return info; @@ -650,6 +700,19 @@ Expected> HailoRtRpcClient::ConfiguredNetworkG return deserialize_vstream_infos(reply); } +Expected HailoRtRpcClient::ConfiguredNetworkGroup_is_scheduled(uint32_t handle) +{ + ConfiguredNetworkGroup_is_scheduled_Request request; + ConfiguredNetworkGroup_is_scheduled_Reply reply; + request.set_handle(handle); + grpc::ClientContext context; + grpc::Status status = m_stub->ConfiguredNetworkGroup_is_scheduled(&context, request, &reply); + CHECK_GRPC_STATUS_AS_EXPECTED(status); + assert(reply.status() < HAILO_STATUS_COUNT); + CHECK_SUCCESS_AS_EXPECTED(static_cast(reply.status())); + return reply.is_scheduled(); +} + hailo_status HailoRtRpcClient::ConfiguredNetworkGroup_set_scheduler_timeout(uint32_t handle, const std::chrono::milliseconds &timeout, const std::string &network_name) { @@ -682,6 +745,22 @@ hailo_status HailoRtRpcClient::ConfiguredNetworkGroup_set_scheduler_threshold(ui return static_cast(reply.status()); } +hailo_status HailoRtRpcClient::ConfiguredNetworkGroup_set_scheduler_priority(uint32_t handle, uint8_t priority, + const std::string &network_name) +{ + ConfiguredNetworkGroup_set_scheduler_priority_Request request; + request.set_handle(handle); + request.set_priority(priority); + request.set_network_name(network_name); + + ConfiguredNetworkGroup_set_scheduler_priority_Reply reply; + grpc::ClientContext context; + grpc::Status status = m_stub->ConfiguredNetworkGroup_set_scheduler_priority(&context, request, &reply); + CHECK_GRPC_STATUS(status); + assert(reply.status() < HAILO_STATUS_COUNT); + return static_cast(reply.status()); +} + Expected HailoRtRpcClient::ConfiguredNetworkGroup_get_latency_measurement(uint32_t handle, const std::string &network_name) { @@ -695,7 +774,7 @@ Expected HailoRtRpcClient::ConfiguredNetworkGroup_get_ assert(reply.status() < HAILO_STATUS_COUNT); CHECK_SUCCESS_AS_EXPECTED(static_cast(reply.status())); LatencyMeasurementResult result{ - .avg_hw_latency = std::chrono::nanoseconds(reply.avg_hw_latency()) + std::chrono::nanoseconds(reply.avg_hw_latency()) }; return result; } @@ -724,37 +803,28 @@ Expected HailoRtRpcClient::ConfiguredNetworkGroup_get_co assert(reply.status() < HAILO_STATUS_COUNT); CHECK_SUCCESS_AS_EXPECTED(static_cast(reply.status())); auto proto_configure_params = reply.params(); - ConfigureNetworkParams network_configure_params; + ConfigureNetworkParams network_configure_params{}; network_configure_params.batch_size = static_cast(proto_configure_params.batch_size()); network_configure_params.power_mode = static_cast(proto_configure_params.power_mode()); network_configure_params.latency = static_cast(proto_configure_params.latency()); for (auto &proto_name_streams_params_pair : proto_configure_params.stream_params_map()) { auto proto_streams_params = proto_name_streams_params_pair.params(); auto stream_direction = static_cast(proto_streams_params.direction()); - hailo_stream_parameters_t stream_params; + hailo_stream_parameters_t stream_params{}; + stream_params.stream_interface = static_cast(proto_streams_params.stream_interface()); + stream_params.direction = stream_direction; + stream_params.flags = static_cast(proto_streams_params.flags()); if (stream_direction == HAILO_H2D_STREAM) { - stream_params = { - .stream_interface = static_cast(proto_streams_params.stream_interface()), - .direction = stream_direction, - {.pcie_input_params = { - .reserved = 0 - }} - }; + stream_params.pcie_input_params = {0}; } else { - stream_params = { - .stream_interface = static_cast(proto_streams_params.stream_interface()), - .direction = stream_direction, - {.pcie_output_params = { - .reserved = 0 - }} - }; + stream_params.pcie_output_params = {0}; } network_configure_params.stream_params_by_name.insert({proto_name_streams_params_pair.name(), stream_params}); } for (auto &proto_name_network_params_pair : proto_configure_params.network_params_map()) { auto proto_network_params = proto_name_network_params_pair.params(); hailo_network_parameters_t net_params { - .batch_size = static_cast(proto_network_params.batch_size()) + static_cast(proto_network_params.batch_size()) }; network_configure_params.network_params_by_name.insert({proto_name_network_params_pair.name(), net_params}); @@ -952,9 +1022,9 @@ Expected HailoRtRpcClient::InputVStream_get_user_buffer_format(u auto user_buffer_format_proto = reply.user_buffer_format(); hailo_format_t format{ - .type = static_cast(user_buffer_format_proto.type()), - .order = static_cast(user_buffer_format_proto.order()), - .flags = static_cast(user_buffer_format_proto.flags()) + static_cast(user_buffer_format_proto.type()), + static_cast(user_buffer_format_proto.order()), + static_cast(user_buffer_format_proto.flags()) }; return format; @@ -973,9 +1043,9 @@ Expected HailoRtRpcClient::OutputVStream_get_user_buffer_format( auto user_buffer_format_proto = reply.user_buffer_format(); hailo_format_t format{ - .type = static_cast(user_buffer_format_proto.type()), - .order = static_cast(user_buffer_format_proto.order()), - .flags = static_cast(user_buffer_format_proto.flags()) + static_cast(user_buffer_format_proto.type()), + static_cast(user_buffer_format_proto.order()), + static_cast(user_buffer_format_proto.flags()) }; return format; diff --git a/hailort/libhailort/src/hailort_rpc_client.hpp b/hailort/libhailort/src/service/hailort_rpc_client.hpp similarity index 89% rename from hailort/libhailort/src/hailort_rpc_client.hpp rename to hailort/libhailort/src/service/hailort_rpc_client.hpp index eaafbc7..5b393fd 100644 --- a/hailort/libhailort/src/hailort_rpc_client.hpp +++ b/hailort/libhailort/src/service/hailort_rpc_client.hpp @@ -10,8 +10,9 @@ #ifndef HAILO_HAILORT_RPC_CLIENT_HPP_ #define HAILO_HAILORT_RPC_CLIENT_HPP_ +#include "hailo/hailort.h" #include "hailo/expected.hpp" -#include "hailo/hailort.hpp" + #if defined(_MSC_VER) #pragma warning(push) #pragma warning(disable: 4244 4267 4127) @@ -28,6 +29,7 @@ #endif #include + namespace hailort { @@ -36,14 +38,17 @@ public: HailoRtRpcClient(std::shared_ptr channel) : m_stub(ProtoHailoRtRpc::NewStub(channel)) {} - hailo_status client_keep_alive(uint32_t process_id); + hailo_status client_keep_alive(uint32_t pid); Expected get_service_version(); Expected VDevice_create(const hailo_vdevice_params_t ¶ms, uint32_t pid); + Expected VDevice_dup_handle(uint32_t pid, uint32_t handle); hailo_status VDevice_release(uint32_t handle); Expected> VDevice_get_physical_devices_ids(uint32_t handle); Expected VDevice_get_default_streams_interface(uint32_t handle); Expected> VDevice_configure(uint32_t vdevice_handle, const Hef &hef, uint32_t pid, const NetworkGroupsParamsMap &configure_params={}); + + Expected ConfiguredNetworkGroup_dup_handle(uint32_t pid, uint32_t handle); hailo_status ConfiguredNetworkGroup_release(uint32_t handle); Expected> ConfiguredNetworkGroup_make_input_vstream_params(uint32_t handle, bool quantized, hailo_format_type_t format_type, uint32_t timeout_ms, uint32_t queue_size, @@ -62,15 +67,19 @@ public: Expected> ConfiguredNetworkGroup_get_input_vstream_infos(uint32_t handle, std::string network_name); Expected> ConfiguredNetworkGroup_get_output_vstream_infos(uint32_t handle, std::string network_name); Expected> ConfiguredNetworkGroup_get_all_vstream_infos(uint32_t handle, std::string network_name); + Expected ConfiguredNetworkGroup_is_scheduled(uint32_t handle); hailo_status ConfiguredNetworkGroup_set_scheduler_timeout(uint32_t handle, const std::chrono::milliseconds &timeout, const std::string &network_name); hailo_status ConfiguredNetworkGroup_set_scheduler_threshold(uint32_t handle, uint32_t threshold, const std::string &network_name); + hailo_status ConfiguredNetworkGroup_set_scheduler_priority(uint32_t handle, uint8_t priority, const std::string &network_name); Expected ConfiguredNetworkGroup_get_latency_measurement(uint32_t handle, const std::string &network_name); Expected ConfiguredNetworkGroup_is_multi_context(uint32_t handle); Expected ConfiguredNetworkGroup_get_config_params(uint32_t handle); Expected> InputVStreams_create(uint32_t net_group_handle, const std::map &inputs_params, uint32_t pid); + Expected InputVStream_dup_handle(uint32_t pid, uint32_t handle); + Expected OutputVStream_dup_handle(uint32_t pid, uint32_t handle); hailo_status InputVStream_release(uint32_t handle); Expected> OutputVStreams_create(uint32_t net_group_handle, const std::map &output_params, uint32_t pid); @@ -105,4 +114,4 @@ private: } -#endif // HAILO_HAILORT_CLIENT_RPC_HPP_ \ No newline at end of file +#endif // HAILO_HAILORT_RPC_CLIENT_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/network_group_client.cpp b/hailort/libhailort/src/service/network_group_client.cpp similarity index 83% rename from hailort/libhailort/src/network_group_client.cpp rename to hailort/libhailort/src/service/network_group_client.cpp index d522693..bb2db9c 100644 --- a/hailort/libhailort/src/network_group_client.cpp +++ b/hailort/libhailort/src/service/network_group_client.cpp @@ -7,11 +7,17 @@ * @brief: Network group client object **/ -#include "context_switch/network_group_internal.hpp" -#include "common/utils.hpp" -#include "hailort_defaults.hpp" #include "hailo/vstream.hpp" -#include "vstream_internal.hpp" +#include "hailo/hailort_defaults.hpp" + +#include "common/utils.hpp" +#include "common/os_utils.hpp" + +#include "network_group/network_group_internal.hpp" +#include "net_flow/pipeline/vstream_internal.hpp" +#include "rpc/rpc_definitions.hpp" +#include "rpc_client_utils.hpp" + namespace hailort { @@ -36,12 +42,40 @@ ConfiguredNetworkGroupClient::~ConfiguredNetworkGroupClient() } } +hailo_status ConfiguredNetworkGroupClient::before_fork() +{ + m_client.reset(); + return HAILO_SUCCESS; +} + +hailo_status ConfiguredNetworkGroupClient::create_client() +{ + auto expected_client = HailoRtRpcClientUtils::create_client(); + CHECK_EXPECTED_AS_STATUS(expected_client); + m_client = expected_client.release(); + return HAILO_SUCCESS; +} + +hailo_status ConfiguredNetworkGroupClient::after_fork_in_parent() +{ + return create_client(); +} + +hailo_status ConfiguredNetworkGroupClient::after_fork_in_child() +{ + auto status = create_client(); + CHECK_SUCCESS(status); + auto expected_dup_handle = m_client->ConfiguredNetworkGroup_dup_handle(OsUtils::get_curr_pid(), m_handle); + CHECK_EXPECTED_AS_STATUS(expected_dup_handle); + m_handle = expected_dup_handle.value(); + return HAILO_SUCCESS; +} + Expected> ConfiguredNetworkGroupClient::activate( - const hailo_activate_network_group_params_t &network_group_params) + const hailo_activate_network_group_params_t &/* network_group_params */) { - (void)network_group_params; - LOGGER__ERROR("ConfiguredNetworkGroup::activate function is not supported when using multi-process service, please use HailoRT Scheduler."); - return make_unexpected(HAILO_INVALID_OPERATION); + LOGGER__WARNING("ConfiguredNetworkGroup::activate function is not supported when using multi-process service or HailoRT Scheduler."); + return make_unexpected(HAILO_NOT_IMPLEMENTED); } /* Network group base functions */ @@ -125,7 +159,7 @@ Expected ConfiguredNetworkGroupClient::get_output_ hailo_status ConfiguredNetworkGroupClient::wait_for_activation(const std::chrono::milliseconds&) { - LOGGER__ERROR("ConfiguredNetworkGroup::wait_for_activation function is not supported when using multi-process service"); + LOGGER__WARNING("ConfiguredNetworkGroup::wait_for_activation function is not supported when using multi-process service or HailoRT Scheduler."); return HAILO_NOT_IMPLEMENTED; } @@ -185,6 +219,16 @@ Expected> ConfiguredNetworkGroupClient::get_al return m_client->ConfiguredNetworkGroup_get_all_vstream_infos(m_handle, network_name); } +bool ConfiguredNetworkGroupClient::is_scheduled() const +{ + auto reply = m_client->ConfiguredNetworkGroup_is_scheduled(m_handle); + if (reply.status() != HAILO_SUCCESS) { + LOGGER__ERROR("is_scheduled failed with status {}", reply.status()); + return false; + } + return reply.value(); +} + hailo_status ConfiguredNetworkGroupClient::set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) { return m_client->ConfiguredNetworkGroup_set_scheduler_timeout(m_handle, timeout, network_name); @@ -195,6 +239,11 @@ hailo_status ConfiguredNetworkGroupClient::set_scheduler_threshold(uint32_t thre return m_client->ConfiguredNetworkGroup_set_scheduler_threshold(m_handle, threshold, network_name); } +hailo_status ConfiguredNetworkGroupClient::set_scheduler_priority(uint8_t priority, const std::string &network_name) +{ + return m_client->ConfiguredNetworkGroup_set_scheduler_priority(m_handle, priority, network_name); +} + AccumulatorPtr ConfiguredNetworkGroupClient::get_activation_time_accumulator() const { LOGGER__ERROR("ConfiguredNetworkGroup::get_activation_time_accumulator function is not supported when using multi-process service"); @@ -229,7 +278,7 @@ const ConfigureNetworkParams ConfiguredNetworkGroupClient::get_config_params() c Expected> ConfiguredNetworkGroupClient::create_input_vstreams(const std::map &inputs_params) { - auto reply = m_client->InputVStreams_create(m_handle, inputs_params, getpid()); + auto reply = m_client->InputVStreams_create(m_handle, inputs_params, OsUtils::get_curr_pid()); CHECK_EXPECTED(reply); auto input_vstreams_handles = reply.release(); std::vector vstreams; @@ -246,7 +295,7 @@ Expected> ConfiguredNetworkGroupClient::create_input_v Expected> ConfiguredNetworkGroupClient::create_output_vstreams(const std::map &outputs_params) { - auto reply = m_client->OutputVStreams_create(m_handle, outputs_params, getpid()); + auto reply = m_client->OutputVStreams_create(m_handle, outputs_params, OsUtils::get_curr_pid()); CHECK_EXPECTED(reply); auto output_vstreams_handles = reply.release(); std::vector vstreams; diff --git a/hailort/libhailort/src/service/rpc_client_utils.hpp b/hailort/libhailort/src/service/rpc_client_utils.hpp new file mode 100644 index 0000000..965be9c --- /dev/null +++ b/hailort/libhailort/src/service/rpc_client_utils.hpp @@ -0,0 +1,152 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file hailort_common.hpp + * @brief Utility functions for rpc client communication + **/ + +#ifndef _HAILO_HAILORT_RPC_CLIENT_UTILS_HPP_ +#define _HAILO_HAILORT_RPC_CLIENT_UTILS_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" +#include "hailo/hailort_defaults.hpp" + +#include "common/async_thread.hpp" +#include "common/os_utils.hpp" + +#include "hailort_rpc_client.hpp" +#include "rpc/rpc_definitions.hpp" + +#include + +namespace hailort +{ + +class HailoRtRpcClientUtils final +{ +public: + static HailoRtRpcClientUtils& get_instance() + { + static HailoRtRpcClientUtils instance; + return instance; + } + + HailoRtRpcClientUtils() + : m_mutex(std::make_shared()) + , m_forking(false) + {} + + static Expected> create_client() + { + auto channel = grpc::CreateChannel(HAILORT_SERVICE_DEFAULT_ADDR, grpc::InsecureChannelCredentials()); + CHECK_AS_EXPECTED(channel != nullptr, HAILO_INTERNAL_FAILURE); + auto client = make_unique_nothrow(channel); + CHECK_NOT_NULL_AS_EXPECTED(client, HAILO_INTERNAL_FAILURE); + return client; + } + + hailo_status init_client_service_communication() + { + std::unique_lock lock(*m_mutex); + if (!m_initialized) { + // Create client + auto channel = grpc::CreateChannel(hailort::HAILORT_SERVICE_DEFAULT_ADDR, grpc::InsecureChannelCredentials()); + auto client = make_unique_nothrow(channel); + CHECK(client != nullptr, HAILO_OUT_OF_HOST_MEMORY); + + // Check service version + auto reply = client->get_service_version(); + CHECK_EXPECTED_AS_STATUS(reply); + hailo_version_t client_version = {}; + auto status = hailo_get_library_version(&client_version); + CHECK_SUCCESS(status); + auto service_version = reply.value(); + auto are_equal = [](auto version1, auto version2) { + return version1.major == version2.major + && version1.minor == version2.minor + && version1.revision == version2.revision; + }; + CHECK(are_equal(service_version, client_version), HAILO_INVALID_SERVICE_VERSION, "Invalid libhailort version on service: " + "client version {}.{}.{}, service version {}.{}.{}", + service_version.major, service_version.minor, service_version.revision, + client_version.major, client_version.minor, client_version.revision); + + // Set pid + m_pid = OsUtils::get_curr_pid(); + + // Trigger client keep-alive + m_keep_alive_thread = make_unique_nothrow>([this] () { + return this->keep_alive(); + }); + CHECK(nullptr != m_keep_alive_thread, HAILO_OUT_OF_HOST_MEMORY); + m_initialized = true; + } + return HAILO_SUCCESS; + } + + hailo_status before_fork() + { + m_forking = true; + return m_keep_alive_thread->get(); + } + + hailo_status after_fork_in_parent() + { + m_forking = false; + std::unique_lock lock(*m_mutex); + if (m_initialized) { + // Trigger client keep-alive + m_keep_alive_thread = make_unique_nothrow>([this] () { + return this->keep_alive(); + }); + } + return HAILO_SUCCESS; + } + + hailo_status after_fork_in_child() + { + m_forking = false; + m_mutex = std::make_shared(); + std::unique_lock lock(*m_mutex); + if (m_initialized) { + m_pid = OsUtils::get_curr_pid(); + // Trigger client keep-alive + m_keep_alive_thread = make_unique_nothrow>([this] () { + return this->keep_alive(); + }); + } + return HAILO_SUCCESS; + } + +private: + ~HailoRtRpcClientUtils() + { + m_keep_alive_thread.release(); + } + + hailo_status keep_alive() + { + auto channel = grpc::CreateChannel(hailort::HAILORT_SERVICE_DEFAULT_ADDR, grpc::InsecureChannelCredentials()); + auto client = make_unique_nothrow(channel); + CHECK(client != nullptr, HAILO_OUT_OF_HOST_MEMORY); + while (!m_forking) { + auto status = client->client_keep_alive(m_pid); + CHECK_SUCCESS(status); + std::this_thread::sleep_for(hailort::HAILO_KEEPALIVE_INTERVAL / 2); + } + return HAILO_SUCCESS; + } + + std::shared_ptr m_mutex; + AsyncThreadPtr m_keep_alive_thread; + bool m_initialized = false; + std::atomic m_forking; + uint32_t m_pid; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_HAILORT_RPC_CLIENT_UTILS_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/stream_common/CMakeLists.txt b/hailort/libhailort/src/stream_common/CMakeLists.txt new file mode 100644 index 0000000..001d29e --- /dev/null +++ b/hailort/libhailort/src/stream_common/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/stream.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/stream_internal.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/stream.cpp b/hailort/libhailort/src/stream_common/stream.cpp similarity index 79% rename from hailort/libhailort/src/stream.cpp rename to hailort/libhailort/src/stream_common/stream.cpp index 32e5951..df20eee 100644 --- a/hailort/libhailort/src/stream.cpp +++ b/hailort/libhailort/src/stream_common/stream.cpp @@ -34,6 +34,16 @@ hailo_status InputStream::write(const MemoryView &buffer) return sync_write_all_raw_buffer_no_transform_impl(const_cast(buffer.data()), 0, buffer.size()); } +hailo_status InputStream::wait_for_ready(size_t /* transfer_size */, std::chrono::milliseconds /* timeout */) +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status InputStream::write_async(std::shared_ptr /* buffer */, const TransferDoneCallback &/* user_callback */, void */* opaque */) +{ + return HAILO_NOT_IMPLEMENTED; +} + std::string InputStream::to_string() const { std::stringstream string_stream; @@ -42,10 +52,11 @@ std::string InputStream::to_string() const return string_stream.str(); } -OutputStream::OutputStream(OutputStream &&other) : m_stream_info(std::move(other.get_info())), - m_dataflow_manager_id(std::move(other.m_dataflow_manager_id)), - m_invalid_frames_count(static_cast(other.m_invalid_frames_count)) -{} +EventPtr &InputStream::get_network_group_activated_event() +{ + LOGGER__WARNING("VDevice InputStream::get_network_group_activated_event() is deprecated."); + return get_core_op_activated_event(); +} hailo_status OutputStream::read_nms(void *buffer, size_t offset, size_t size) { @@ -100,7 +111,7 @@ hailo_status OutputStream::read_nms(void *buffer, size_t offset, size_t size) hailo_status OutputStream::read(MemoryView buffer) { CHECK((buffer.size() % get_info().hw_frame_size) == 0, HAILO_INVALID_ARGUMENT, - "When read size {} must be a multiple of hw size {}", buffer.size(), get_info().hw_frame_size); + "Read size {} must be a multiple of hw size {}", buffer.size(), get_info().hw_frame_size); if (get_info().format.order == HAILO_FORMAT_ORDER_HAILO_NMS){ return read_nms(buffer.data(), 0, buffer.size()); @@ -109,6 +120,17 @@ hailo_status OutputStream::read(MemoryView buffer) } } +hailo_status OutputStream::wait_for_ready(size_t /* transfer_size */, std::chrono::milliseconds /* timeout */) +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status OutputStream::read_async(std::shared_ptr /* buffer */, const TransferDoneCallback &/* user_callback */, void */* opaque */) +{ + return HAILO_NOT_IMPLEMENTED; +} + + std::string OutputStream::to_string() const { std::stringstream string_stream; @@ -127,5 +149,11 @@ void OutputStream::increase_invalid_frames_count(uint32_t value) m_invalid_frames_count = m_invalid_frames_count + value; } +EventPtr &OutputStream::get_network_group_activated_event() +{ + LOGGER__WARNING("VDevice OutputStream::get_network_group_activated_event() is deprecated."); + return get_core_op_activated_event(); +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/stream_internal.cpp b/hailort/libhailort/src/stream_common/stream_internal.cpp similarity index 66% rename from hailort/libhailort/src/stream_internal.cpp rename to hailort/libhailort/src/stream_common/stream_internal.cpp index 372b339..b3fb244 100644 --- a/hailort/libhailort/src/stream_internal.cpp +++ b/hailort/libhailort/src/stream_common/stream_internal.cpp @@ -7,26 +7,29 @@ * @brief Implementation of InputStreamBase and OutputStreamBase **/ -#include "stream_internal.hpp" #include "hailo/hailort.h" #include "hailo/expected.hpp" -#include "common/logger_macros.hpp" #include "hailo/transform.hpp" + #include "common/utils.hpp" +#include "common/logger_macros.hpp" + +#include "stream_common/stream_internal.hpp" + namespace hailort { InputStreamBase::InputStreamBase(const hailo_stream_info_t &stream_info, - const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, const EventPtr &network_group_activated_event) : - m_nn_stream_config(nn_stream_config), m_network_group_activated_event(network_group_activated_event) + const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, const EventPtr &core_op_activated_event) : + m_nn_stream_config(nn_stream_config), m_core_op_activated_event(core_op_activated_event) { m_stream_info = stream_info; } -EventPtr &InputStreamBase::get_network_group_activated_event() +EventPtr &InputStreamBase::get_core_op_activated_event() { - return m_network_group_activated_event; + return m_core_op_activated_event; } bool InputStreamBase::is_scheduled() @@ -35,15 +38,15 @@ bool InputStreamBase::is_scheduled() } OutputStreamBase::OutputStreamBase(const LayerInfo &layer_info, const hailo_stream_info_t &stream_info, - const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, const EventPtr &network_group_activated_event) : - m_nn_stream_config(nn_stream_config), m_layer_info(layer_info), m_network_group_activated_event(network_group_activated_event) + const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, const EventPtr &core_op_activated_event) : + m_nn_stream_config(nn_stream_config), m_layer_info(layer_info), m_core_op_activated_event(core_op_activated_event) { m_stream_info = stream_info; } -EventPtr &OutputStreamBase::get_network_group_activated_event() +EventPtr &OutputStreamBase::get_core_op_activated_event() { - return m_network_group_activated_event; + return m_core_op_activated_event; } bool OutputStreamBase::is_scheduled() diff --git a/hailort/libhailort/src/stream_internal.hpp b/hailort/libhailort/src/stream_common/stream_internal.hpp similarity index 76% rename from hailort/libhailort/src/stream_internal.hpp rename to hailort/libhailort/src/stream_common/stream_internal.hpp index 5ea1a1e..27eff6b 100644 --- a/hailort/libhailort/src/stream_internal.hpp +++ b/hailort/libhailort/src/stream_common/stream_internal.hpp @@ -11,18 +11,14 @@ * * InputStream (External "interface") * |-- InputStreamBase (Base class) - * |-- VdmaInputStream (Base class for vdma streams) - * | |-- PcieInputStream - * | |-- CoreInputStream + * |-- VdmaInputStream * |-- EthernetInputStream * |-- MipiInputStream * * * OutputStream (External "interface") * |-- OutputStreamBase (Base class) - * |-- VdmaOutputStream (Base class for vdma streams) - * | |-- PcieOutputStream - * | |-- CoreOutputStream + * |-- VdmaOutputStream * |-- EthernetOutputStream * **/ @@ -33,10 +29,12 @@ #include "hailo/stream.hpp" #include "hailo/event.hpp" #include "hailo/hailort_common.hpp" -#include "hef_internal.hpp" -#include "control_protocol.hpp" -#include "layer_info.hpp" -#include "vdma_channel.hpp" + +#include "hef/hef_internal.hpp" +#include "device_common/control_protocol.hpp" +#include "hef/layer_info.hpp" +#include "vdma/channel/boundary_channel.hpp" + namespace hailort { @@ -61,10 +59,6 @@ class InputStreamBase : public InputStream public: virtual ~InputStreamBase() = default; - InputStreamBase(const InputStreamBase&) = delete; - InputStreamBase& operator=(const InputStreamBase&) = delete; - InputStreamBase(InputStreamBase&&) = default; - virtual const CONTROL_PROTOCOL__nn_stream_config_t &get_nn_stream_config() { return m_nn_stream_config; @@ -86,12 +80,17 @@ public: return make_unexpected(HAILO_INVALID_OPERATION); } + virtual hailo_status register_interrupt_callback(const vdma::ProcessingCompleteCallback &/*callback*/) + { + return HAILO_INVALID_OPERATION; + } + CONTROL_PROTOCOL__nn_stream_config_t m_nn_stream_config; protected: explicit InputStreamBase(const LayerInfo &layer_info, hailo_stream_interface_t stream_interface, - EventPtr &&network_group_activated_event, hailo_status &status) : - m_network_group_activated_event(std::move(network_group_activated_event)) + EventPtr &&core_op_activated_event, hailo_status &status) : + m_core_op_activated_event(std::move(core_op_activated_event)) { m_stream_info = LayerInfoUtils::get_stream_info_from_layer_info(layer_info); @@ -108,15 +107,15 @@ protected: } InputStreamBase(const hailo_stream_info_t &stream_info, - const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, const EventPtr &network_group_activated_event); + const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, const EventPtr &core_op_activated_event); - virtual EventPtr &get_network_group_activated_event() override; + virtual EventPtr &get_core_op_activated_event() override; virtual bool is_scheduled() override; private: friend class InputStreamWrapper; - EventPtr m_network_group_activated_event; + EventPtr m_core_op_activated_event; }; @@ -125,10 +124,6 @@ class OutputStreamBase : public OutputStream public: virtual ~OutputStreamBase() = default; - OutputStreamBase(const OutputStreamBase&) = delete; - OutputStreamBase& operator=(const OutputStreamBase&) = delete; - OutputStreamBase(OutputStreamBase&&) = default; - virtual const CONTROL_PROTOCOL__nn_stream_config_t &get_nn_stream_config() { return m_nn_stream_config; @@ -149,7 +144,7 @@ public: return make_unexpected(HAILO_INVALID_OPERATION); } - virtual hailo_status register_for_d2h_interrupts(const std::function &/*callback*/) + virtual hailo_status register_interrupt_callback(const vdma::ProcessingCompleteCallback &/*callback*/) { return HAILO_INVALID_OPERATION; } @@ -158,8 +153,8 @@ public: protected: explicit OutputStreamBase(const LayerInfo &layer_info, - EventPtr &&network_group_activated_event, hailo_status &status) : - m_layer_info(layer_info), m_network_group_activated_event(std::move(network_group_activated_event)) + EventPtr &&core_op_activated_event, hailo_status &status) : + m_layer_info(layer_info), m_core_op_activated_event(std::move(core_op_activated_event)) { m_stream_info = LayerInfoUtils::get_stream_info_from_layer_info(m_layer_info); @@ -175,9 +170,9 @@ protected: } OutputStreamBase(const LayerInfo &layer_info, const hailo_stream_info_t &stream_info, - const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, const EventPtr &network_group_activated_event); + const CONTROL_PROTOCOL__nn_stream_config_t &nn_stream_config, const EventPtr &core_op_activated_event); - virtual EventPtr &get_network_group_activated_event() override; + virtual EventPtr &get_core_op_activated_event() override; virtual bool is_scheduled() override; LayerInfo m_layer_info; @@ -185,7 +180,7 @@ protected: private: friend class OutputStreamWrapper; - EventPtr m_network_group_activated_event; + EventPtr m_core_op_activated_event; }; } /* namespace hailort */ diff --git a/hailort/libhailort/src/tracer.hpp b/hailort/libhailort/src/tracer.hpp deleted file mode 100644 index 76caf36..0000000 --- a/hailort/libhailort/src/tracer.hpp +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file tracer.hpp - * @brief Tracing mechanism for HailoRT + FW events - **/ - -#ifndef _HAILO_TRACER_HPP_ -#define _HAILO_TRACER_HPP_ - -#include "hailo/hailort.h" -#include "common/logger_macros.hpp" -#include "network_group_scheduler.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace hailort -{ - -struct Trace -{ - Trace(const std::string &name) - : name(name) - {} - - virtual ~Trace() = default; - - uint64_t timestamp = 0; - std::string name; -}; - -struct InitTrace : Trace -{ - InitTrace() : Trace("init") {} -}; - -struct AddNetworkGroupTrace : Trace -{ - AddNetworkGroupTrace(const std::string &device_id, const std::string &network_group_name, uint64_t timeout, uint32_t threshold, scheduler_ng_handle_t handle) - : Trace("add_network_group"), device_id(device_id), network_group_name(network_group_name), timeout(timeout), threshold(threshold), network_group_handle(handle) - {} - - std::string device_id; - std::string network_group_name; - uint64_t timeout = 0; - uint32_t threshold = 0; - scheduler_ng_handle_t network_group_handle = INVALID_NETWORK_GROUP_HANDLE; -}; - -struct CreateNetworkGroupInputStreamsTrace : Trace -{ - CreateNetworkGroupInputStreamsTrace(const std::string &device_id, const std::string &network_group_name, const std::string &stream_name, uint32_t queue_size) - : Trace("create_input_stream"), device_id(device_id), network_group_name(network_group_name), stream_name(stream_name), queue_size(queue_size) - {} - - std::string device_id; - std::string network_group_name; - std::string stream_name; - uint32_t queue_size; -}; - -struct CreateNetworkGroupOutputStreamsTrace : Trace -{ - CreateNetworkGroupOutputStreamsTrace(const std::string &device_id, const std::string &network_group_name, const std::string &stream_name, uint32_t queue_size) - : Trace("create_output_stream"), device_id(device_id), network_group_name(network_group_name), stream_name(stream_name), queue_size(queue_size) - {} - - std::string device_id; - std::string network_group_name; - std::string stream_name; - uint32_t queue_size; -}; - -struct WriteFrameTrace : Trace -{ - WriteFrameTrace(const std::string &device_id, scheduler_ng_handle_t network_group_handle, const std::string &queue_name) - : Trace("wrte_frame"), device_id(device_id), network_group_handle(network_group_handle), queue_name(queue_name) - {} - - std::string device_id; - scheduler_ng_handle_t network_group_handle; - std::string queue_name; -}; - -struct InputVdmaEnqueueTrace : Trace -{ - InputVdmaEnqueueTrace(const std::string &device_id, scheduler_ng_handle_t network_group_handle, const std::string &queue_name) - : Trace("input_vdma_enqueue"), device_id(device_id), network_group_handle(network_group_handle), queue_name(queue_name) - {} - - std::string device_id; - scheduler_ng_handle_t network_group_handle; - std::string queue_name; -}; - -struct ReadFrameTrace : Trace -{ - ReadFrameTrace(const std::string &device_id, scheduler_ng_handle_t network_group_handle, const std::string &queue_name) - : Trace("read_frame"), device_id(device_id), network_group_handle(network_group_handle), queue_name(queue_name) - {} - - std::string device_id; - scheduler_ng_handle_t network_group_handle; - std::string queue_name; -}; - -struct OutputVdmaEnqueueTrace : Trace -{ - OutputVdmaEnqueueTrace(const std::string &device_id, scheduler_ng_handle_t network_group_handle, const std::string &queue_name, uint32_t frames) - : Trace("output_vdma_enqueue"), device_id(device_id), network_group_handle(network_group_handle), queue_name(queue_name), frames(frames) - {} - - std::string device_id; - scheduler_ng_handle_t network_group_handle; - std::string queue_name; - uint32_t frames = 0; -}; - -struct ChooseNetworkGroupTrace : Trace -{ - ChooseNetworkGroupTrace(const std::string &device_id, scheduler_ng_handle_t handle, bool threshold, bool timeout) - : Trace("choose_network_group"), device_id(device_id), network_group_handle(handle), threshold(threshold), timeout(timeout) - {} - - std::string device_id; - scheduler_ng_handle_t network_group_handle; - bool threshold = false; - bool timeout = false; -}; - -struct SwitchNetworkGroupTrace : Trace -{ - SwitchNetworkGroupTrace(const std::string &device_id, scheduler_ng_handle_t handle) - : Trace("switch_network_group"), device_id(device_id), network_group_handle(handle) - {} - - std::string device_id; - scheduler_ng_handle_t network_group_handle; -}; - -class Handler -{ -public: - virtual ~Handler() = default; - - virtual void handle_trace(const InitTrace&) {}; - virtual void handle_trace(const AddNetworkGroupTrace&) {}; - virtual void handle_trace(const CreateNetworkGroupInputStreamsTrace&) {}; - virtual void handle_trace(const CreateNetworkGroupOutputStreamsTrace&) {}; - virtual void handle_trace(const WriteFrameTrace&) {}; - virtual void handle_trace(const InputVdmaEnqueueTrace&) {}; - virtual void handle_trace(const ReadFrameTrace&) {}; - virtual void handle_trace(const OutputVdmaEnqueueTrace&) {}; - virtual void handle_trace(const ChooseNetworkGroupTrace&) {}; - virtual void handle_trace(const SwitchNetworkGroupTrace&) {}; -}; - -struct JSON; - -class SchedulerProfilerHandler : public Handler -{ -public: - SchedulerProfilerHandler(SchedulerProfilerHandler const&) = delete; - void operator=(SchedulerProfilerHandler const&) = delete; - - SchedulerProfilerHandler(int64_t &start_time); - ~SchedulerProfilerHandler(); - - virtual void handle_trace(const AddNetworkGroupTrace&) override; - virtual void handle_trace(const CreateNetworkGroupInputStreamsTrace&) override; - virtual void handle_trace(const CreateNetworkGroupOutputStreamsTrace&) override; - virtual void handle_trace(const WriteFrameTrace&) override; - virtual void handle_trace(const InputVdmaEnqueueTrace&) override; - virtual void handle_trace(const ReadFrameTrace&) override; - virtual void handle_trace(const OutputVdmaEnqueueTrace&) override; - virtual void handle_trace(const ChooseNetworkGroupTrace&) override; - virtual void handle_trace(const SwitchNetworkGroupTrace&) override; - -private: - void log(JSON json); - bool comma(); - - std::shared_ptr m_file_sink; - std::shared_ptr m_profiler_logger; - std::atomic m_first_write; -}; - -class Tracer -{ -public: - template - static void trace(Args... trace_args) - { - auto &tracer = get_instance(); - tracer.execute_trace(trace_args...); - } - -private: - Tracer(); - - static Tracer& get_instance() - { - static Tracer tracer; - return tracer; - } - - template - void execute_trace(Args... trace_args) - { - if (!m_should_trace) { - return; - } - - TraceType trace_struct(trace_args...); - auto curr_time = std::chrono::high_resolution_clock::now(); - trace_struct.timestamp = std::chrono::duration_cast(curr_time - this->m_start_time).count(); - for (auto &handler : this->m_handlers) { - handler->handle_trace(trace_struct); - } - } - - bool m_should_trace = false; - std::chrono::high_resolution_clock::time_point m_start_time; - std::vector> m_handlers; -}; - -} - -#endif \ No newline at end of file diff --git a/hailort/libhailort/src/transform/CMakeLists.txt b/hailort/libhailort/src/transform/CMakeLists.txt new file mode 100644 index 0000000..7c0f9c7 --- /dev/null +++ b/hailort/libhailort/src/transform/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/transform.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/transform.cpp b/hailort/libhailort/src/transform/transform.cpp similarity index 94% rename from hailort/libhailort/src/transform.cpp rename to hailort/libhailort/src/transform/transform.cpp index 799d22b..a983c95 100644 --- a/hailort/libhailort/src/transform.cpp +++ b/hailort/libhailort/src/transform/transform.cpp @@ -12,15 +12,18 @@ #include "hailo/expected.hpp" #include "hailo/hailort_common.hpp" #include "hailo/quantization.hpp" -#include "hailort_defaults.hpp" +#include "hailo/hailort_defaults.hpp" + #include "common/compiler_extensions_compat.hpp" #include "common/logger_macros.hpp" #include "common/utils.hpp" -#include "transform_internal.hpp" + +#include "transform/transform_internal.hpp" #include #include + namespace hailort { @@ -286,6 +289,56 @@ void transform__h2d_NV12_to_NV12(const T *src_ptr, hailo_3d_image_shape_t *src_i } } +template +void transform__h2d_I420_to_YYYYUV(const T *src_ptr, hailo_3d_image_shape_t *src_image_shape, T *dst_ptr, hailo_3d_image_shape_t *dst_image_shape) +{ + /* Validate arguments */ + ASSERT(NULL != src_ptr); + ASSERT(NULL != dst_ptr); + uint32_t rows_count = src_image_shape->height * src_image_shape->features; + ASSERT(0 == (rows_count % 3)); + ASSERT(0 == (src_image_shape->width % 2)); + ASSERT(dst_image_shape->width >= src_image_shape->width); + + auto padding_size_y = (dst_image_shape->width - src_image_shape->width); + auto padding_size_uv = (dst_image_shape->width / 2) - (src_image_shape->width / 2); + + uint32_t y_plane_rows_count = static_cast(rows_count / 1.5); + + size_t src_offset_y = 0; + size_t src_offset_u = (y_plane_rows_count * src_image_shape->width); + size_t src_offset_v = src_offset_u + (static_cast((y_plane_rows_count / 2) * (src_image_shape->width / 2))); + size_t dst_offset = 0; + + for(uint32_t h = 0; h < y_plane_rows_count; h += 2) { + // Copy Y + for (auto j = 0; j < 2; j++) { + memcpy(dst_ptr + dst_offset, src_ptr + src_offset_y, (src_image_shape->width * sizeof(T))); + src_offset_y += (src_image_shape->width); + dst_offset += (src_image_shape->width); + // add padding + memset((dst_ptr + dst_offset), 0, (padding_size_y * sizeof(T))); + dst_offset += padding_size_y; + } + + // Copy U/2 + memcpy(dst_ptr + dst_offset, (src_ptr + src_offset_u), ((src_image_shape->width / 2) * sizeof(T))); + src_offset_u += (src_image_shape->width / 2); + dst_offset += (src_image_shape->width / 2); + // Add padding + memset((dst_ptr + dst_offset), 0, (padding_size_uv * sizeof(T))); + dst_offset += padding_size_uv; + + // Copy V/2 + memcpy(dst_ptr + dst_offset, (src_ptr + src_offset_v), ((src_image_shape->width / 2) * sizeof(T))); + src_offset_v += (src_image_shape->width / 2); + dst_offset += (src_image_shape->width / 2); + // Add padding + memset((dst_ptr + dst_offset), 0, (padding_size_uv * sizeof(T))); + dst_offset += padding_size_uv; + } +} + template void transform__h2d_NHWC_to_NHCW(const T *src_ptr, hailo_3d_image_shape_t *src_image_shape, T *dst_ptr, hailo_3d_image_shape_t *dst_image_shape) @@ -312,7 +365,7 @@ void transform__h2d_NHWC_to_NHCW(const T *src_ptr, hailo_3d_image_shape_t *src_i /* pad width to 8 elemnts */ if (pad_size != 0) { dst_offset = r * dst_row_size + f * dst_image_shape->width + src_image_shape->width; - memset(dst_ptr + dst_offset, 0, pad_size); + memset(dst_ptr + dst_offset, 0, pad_size * sizeof(T)); } } } @@ -805,7 +858,7 @@ hailo_status InputTransformContext::quantize_stream(const void *src_ptr, void *q switch (m_src_format.type) { case HAILO_FORMAT_TYPE_UINT8: - if (m_dst_format.type == HAILO_FORMAT_TYPE_UINT8) { + if (HAILO_FORMAT_TYPE_UINT8 == m_dst_format.type) { Quantization::quantize_input_buffer((uint8_t*)src_ptr, (uint8_t*)quant_buffer, shape_size, m_dst_quant_info); } else { @@ -813,7 +866,10 @@ hailo_status InputTransformContext::quantize_stream(const void *src_ptr, void *q } break; case HAILO_FORMAT_TYPE_UINT16: - if (m_dst_format.type == HAILO_FORMAT_TYPE_UINT16) { + if (HAILO_FORMAT_TYPE_UINT8 == m_dst_format.type) { + Quantization::quantize_input_buffer((uint16_t*)src_ptr, (uint8_t *)quant_buffer, shape_size, m_dst_quant_info); + } + else if (HAILO_FORMAT_TYPE_UINT16 == m_dst_format.type) { Quantization::quantize_input_buffer((uint16_t*)src_ptr, (uint16_t *)quant_buffer, shape_size, m_dst_quant_info); } else { @@ -821,10 +877,10 @@ hailo_status InputTransformContext::quantize_stream(const void *src_ptr, void *q } break; case HAILO_FORMAT_TYPE_FLOAT32: - if (m_dst_format.type == HAILO_FORMAT_TYPE_UINT8) { + if (HAILO_FORMAT_TYPE_UINT8 == m_dst_format.type) { Quantization::quantize_input_buffer((float32_t*)src_ptr, (uint8_t*)quant_buffer, shape_size, m_dst_quant_info); } - else if (m_dst_format.type == HAILO_FORMAT_TYPE_UINT16) { + else if (HAILO_FORMAT_TYPE_UINT16 == m_dst_format.type) { Quantization::quantize_input_buffer((float32_t*)src_ptr, (uint16_t*)quant_buffer, shape_size, m_dst_quant_info); } else { @@ -852,7 +908,10 @@ hailo_status FrameOutputTransformContext::quantize_stream(const void *dst_ptr) } break; case HAILO_FORMAT_TYPE_UINT16: - if (HAILO_FORMAT_TYPE_UINT16 == m_src_format.type) { + if (HAILO_FORMAT_TYPE_UINT8 == m_src_format.type) { + Quantization::dequantize_output_buffer_in_place((uint16_t*)dst_ptr, shape_size, m_dst_quant_info); + } + else if (HAILO_FORMAT_TYPE_UINT16 == m_src_format.type) { Quantization::dequantize_output_buffer_in_place((uint16_t*)dst_ptr, shape_size, m_dst_quant_info); } else { @@ -862,20 +921,20 @@ hailo_status FrameOutputTransformContext::quantize_stream(const void *dst_ptr) case HAILO_FORMAT_TYPE_FLOAT32: /* if output layer is argmax - do not rescale */ if (HAILO_FORMAT_ORDER_NHW != m_dst_format.order) { - if (m_src_format.type == HAILO_FORMAT_TYPE_UINT8) { + if (HAILO_FORMAT_TYPE_UINT8 == m_src_format.type) { Quantization::dequantize_output_buffer_in_place((float32_t*)dst_ptr, shape_size, m_dst_quant_info); } - else if (m_src_format.type == HAILO_FORMAT_TYPE_UINT16) { + else if (HAILO_FORMAT_TYPE_UINT16 == m_src_format.type) { Quantization::dequantize_output_buffer_in_place((float32_t*)dst_ptr, shape_size, m_dst_quant_info); } else { return HAILO_INVALID_OPERATION; } } else { - if (m_src_format.type == HAILO_FORMAT_TYPE_UINT8) { + if (HAILO_FORMAT_TYPE_UINT8 == m_src_format.type) { cast_elements_inplace((float32_t*)dst_ptr, shape_size); } - else if (m_src_format.type == HAILO_FORMAT_TYPE_UINT16) { + else if (HAILO_FORMAT_TYPE_UINT16 == m_src_format.type) { cast_elements_inplace((float32_t*)dst_ptr, shape_size); } else { @@ -1059,18 +1118,34 @@ hailo_status reorder_input_stream(const void *src_ptr, hailo_3d_image_shape_t sr (HAILO_FORMAT_ORDER_HAILO_YYUV) == dst_format.order) || ((HAILO_FORMAT_ORDER_NV21 == src_format.order) && (HAILO_FORMAT_ORDER_HAILO_YYVU) == dst_format.order)) { - switch (src_format.type) { - case HAILO_FORMAT_TYPE_UINT8: - transform__h2d_NV12_to_NV12((uint8_t*)src_ptr, &src_image_shape, (uint8_t*)dst_ptr, &dst_image_shape); - break; - case HAILO_FORMAT_TYPE_UINT16: - transform__h2d_NV12_to_NV12((uint16_t*)src_ptr, &src_image_shape, (uint16_t*)dst_ptr, &dst_image_shape); - break; - default: - LOGGER__ERROR("Invalid src-buffer's type format {}", src_format.type); - return HAILO_INVALID_ARGUMENT; - } - return HAILO_SUCCESS; + switch (src_format.type) { + case HAILO_FORMAT_TYPE_UINT8: + transform__h2d_NV12_to_NV12((uint8_t*)src_ptr, &src_image_shape, (uint8_t*)dst_ptr, &dst_image_shape); + break; + case HAILO_FORMAT_TYPE_UINT16: + transform__h2d_NV12_to_NV12((uint16_t*)src_ptr, &src_image_shape, (uint16_t*)dst_ptr, &dst_image_shape); + break; + default: + LOGGER__ERROR("Invalid src-buffer's type format {}", src_format.type); + return HAILO_INVALID_ARGUMENT; + } + return HAILO_SUCCESS; + } + + if (((HAILO_FORMAT_ORDER_I420 == src_format.order) && + (HAILO_FORMAT_ORDER_HAILO_YYYYUV) == dst_format.order)) { + switch (src_format.type) { + case HAILO_FORMAT_TYPE_UINT8: + transform__h2d_I420_to_YYYYUV((uint8_t*)src_ptr, &src_image_shape, (uint8_t*)dst_ptr, &dst_image_shape); + break; + case HAILO_FORMAT_TYPE_UINT16: + transform__h2d_I420_to_YYYYUV((uint16_t*)src_ptr, &src_image_shape, (uint16_t*)dst_ptr, &dst_image_shape); + break; + default: + LOGGER__ERROR("Invalid src-buffer's type format {}", src_format.type); + return HAILO_INVALID_ARGUMENT; + } + return HAILO_SUCCESS; } if ((HAILO_FORMAT_ORDER_RGB4 == src_format.order) && @@ -1439,21 +1514,22 @@ hailo_status validate_input_transform_params(hailo_3d_image_shape_t src_image_sh return HAILO_INVALID_ARGUMENT; } - if ((HAILO_FORMAT_FLAGS_QUANTIZED & src_format.flags) && (HAILO_FORMAT_TYPE_FLOAT32 == src_format.type)) { - LOGGER__ERROR("float32 data isn't quantized"); - return HAILO_INVALID_ARGUMENT; - } - /* Check for overscale transformation*/ CHECK((hailo_format_type_t::HAILO_FORMAT_TYPE_AUTO == src_format.type) || (src_format.type >= dst_format.type), HAILO_INVALID_ARGUMENT, "Overscale transformation is not supported"); /* Check device type */ if (!((HAILO_FORMAT_TYPE_UINT16 == dst_format.type) || (HAILO_FORMAT_TYPE_UINT8 == dst_format.type))) { - LOGGER__ERROR("unsupported device type {}", dst_format.type); + LOGGER__ERROR("Unsupported device-side format_type {}", dst_format.type); return HAILO_INVALID_ARGUMENT; } + /* Check for scaled type without quantization flag*/ + CHECK(!(HAILO_FORMAT_FLAGS_QUANTIZED & src_format.flags) || + ((src_format.type == dst_format.type) || (hailo_format_type_t::HAILO_FORMAT_TYPE_AUTO == src_format.type)), + HAILO_INVALID_ARGUMENT, "src-data-type ({}) is bigger than dst-data-type ({}), and must be marked as not quantized", + src_format.type, dst_format.type); + /* Check reorder flags - where no reorder is needed */ if ((HAILO_FORMAT_ORDER_FCR == src_format.order) && (HAILO_FORMAT_ORDER_FCR == dst_format.order)) { @@ -1495,7 +1571,7 @@ hailo_status validate_output_transform_params(hailo_3d_image_shape_t src_image_s /* Check device type */ if (!((HAILO_FORMAT_TYPE_UINT16 == src_format.type) || (HAILO_FORMAT_TYPE_UINT8 == src_format.type))) { - LOGGER__ERROR("unsupported device type {}", dst_format.type); + LOGGER__ERROR("Unsupported device-side format_type {}", dst_format.type); return HAILO_INVALID_ARGUMENT; } @@ -1503,6 +1579,12 @@ hailo_status validate_output_transform_params(hailo_3d_image_shape_t src_image_s CHECK((hailo_format_type_t::HAILO_FORMAT_TYPE_AUTO == dst_format.type) || (src_format.type <= dst_format.type), HAILO_INVALID_ARGUMENT, "Underscale transformation is not supported"); + /* Check for scaled type without quantization flag*/ + CHECK(!(HAILO_FORMAT_FLAGS_QUANTIZED & dst_format.flags) || + ((src_format.type == dst_format.type) || (hailo_format_type_t::HAILO_FORMAT_TYPE_AUTO == dst_format.type)), + HAILO_INVALID_ARGUMENT, "dst-data-type ({}) is bigger than src-data-type ({}), and must be marked as not quantized", + dst_format.type, src_format.type); + /* Check reorder flags - where no reorder is needed */ if ((HAILO_FORMAT_ORDER_BAYER_RGB == src_format.order) && (HAILO_FORMAT_ORDER_BAYER_RGB == dst_format.order)) { @@ -1846,7 +1928,7 @@ hailo_status NMSOutputTransformContext::transform(const MemoryView src, MemoryVi // NMS has to be uint16 or float32 switch (m_dst_format.type) { case HAILO_FORMAT_TYPE_UINT16: - if (m_src_format.type == HAILO_FORMAT_TYPE_UINT16) { + if (HAILO_FORMAT_TYPE_UINT16 == m_src_format.type) { Quantization::dequantize_output_buffer_nms((uint16_t*)m_quant_buffer.data(), (uint16_t*)dst.data(), shape_size, m_dst_quant_info, m_nms_info.number_of_classes); } @@ -1855,7 +1937,7 @@ hailo_status NMSOutputTransformContext::transform(const MemoryView src, MemoryVi } break; case HAILO_FORMAT_TYPE_FLOAT32: - if (m_src_format.type == HAILO_FORMAT_TYPE_UINT16) { + if (HAILO_FORMAT_TYPE_UINT16 == m_src_format.type) { Quantization::dequantize_output_buffer_nms((uint16_t*)m_quant_buffer.data(), (float32_t*)dst.data(), shape_size, m_dst_quant_info, m_nms_info.number_of_classes); } diff --git a/hailort/libhailort/src/transform_internal.hpp b/hailort/libhailort/src/transform/transform_internal.hpp similarity index 97% rename from hailort/libhailort/src/transform_internal.hpp rename to hailort/libhailort/src/transform/transform_internal.hpp index f439776..b8ef52d 100644 --- a/hailort/libhailort/src/transform_internal.hpp +++ b/hailort/libhailort/src/transform/transform_internal.hpp @@ -16,12 +16,14 @@ #include "hailo/buffer.hpp" #include "hailo/hef.hpp" #include "hailo/transform.hpp" -#include "stream_internal.hpp" -#include "layer_info.hpp" + +#include "stream_common/stream_internal.hpp" +#include "hef/layer_info.hpp" #include #include + namespace hailort { diff --git a/hailort/libhailort/src/utils/CMakeLists.txt b/hailort/libhailort/src/utils/CMakeLists.txt new file mode 100644 index 0000000..c16a9a5 --- /dev/null +++ b/hailort/libhailort/src/utils/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/hailort_common.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/hailort_logger.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/buffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/sensor_config_utils.cpp +) + +if(HAILO_BUILD_PROFILER) + add_subdirectory(profiler) +endif() + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/buffer.cpp b/hailort/libhailort/src/utils/buffer.cpp similarity index 100% rename from hailort/libhailort/src/buffer.cpp rename to hailort/libhailort/src/utils/buffer.cpp diff --git a/hailort/libhailort/src/event_internal.hpp b/hailort/libhailort/src/utils/event_internal.hpp similarity index 100% rename from hailort/libhailort/src/event_internal.hpp rename to hailort/libhailort/src/utils/event_internal.hpp diff --git a/hailort/libhailort/src/hailort_common.cpp b/hailort/libhailort/src/utils/hailort_common.cpp similarity index 100% rename from hailort/libhailort/src/hailort_common.cpp rename to hailort/libhailort/src/utils/hailort_common.cpp diff --git a/hailort/libhailort/src/hailort_logger.cpp b/hailort/libhailort/src/utils/hailort_logger.cpp similarity index 78% rename from hailort/libhailort/src/hailort_logger.cpp rename to hailort/libhailort/src/utils/hailort_logger.cpp index 4e29c26..3eda92f 100644 --- a/hailort/libhailort/src/hailort_logger.cpp +++ b/hailort/libhailort/src/utils/hailort_logger.cpp @@ -7,10 +7,11 @@ * @brief Implements logger used by hailort. **/ -#include "hailort_logger.hpp" #include "common/utils.hpp" #include "common/filesystem.hpp" +#include "utils/hailort_logger.hpp" + #include #include #include @@ -25,6 +26,7 @@ #include #endif + namespace hailort { @@ -33,9 +35,13 @@ namespace hailort #define HAILORT_NAME ("HailoRT") #define HAILORT_LOGGER_FILENAME ("hailort.log") #define HAILORT_MAX_NUMBER_OF_LOG_FILES (1) // There will be 2 log files - 1 spare -#define HAILORT_CONSOLE_LOGGER_PATTERN ("[%n] [%^%l%$] %v") // Console logger will print: [hailort logger file name] [log level] msg -#define HAILORT_MAIN_FILE_LOGGER_PATTERN ("[%Y-%m-%d %X.%e] [%P] [%n] [%l] [%s:%#] [%!] %v") //File logger will print: [timestamp] [PID] [hailort] [log level] [source file:line number] [function name] msg -#define HAILORT_LOCAL_FILE_LOGGER_PATTERN ("[%Y-%m-%d %X.%e] [%n] [%l] [%s:%#] [%!] %v") //File logger will print: [timestamp] [hailort] [log level] [source file:line number] [function name] msg +#ifdef NDEBUG +#define HAILORT_CONSOLE_LOGGER_PATTERN ("[%n] [%^%l%$] %v") // Console logger will print: [hailort] [log level] msg +#else +#define HAILORT_CONSOLE_LOGGER_PATTERN ("[%Y-%m-%d %X.%e] [%P] [%t] [%n] [%^%l%$] [%s:%#] [%!] %v") // Console logger will print: [timestamp] [PID] [TID] [hailort] [log level] [source file:line number] [function name] msg +#endif +#define HAILORT_MAIN_FILE_LOGGER_PATTERN ("[%Y-%m-%d %X.%e] [%P] [%t] [%n] [%l] [%s:%#] [%!] %v") // File logger will print: [timestamp] [PID] [TID] [hailort] [log level] [source file:line number] [function name] msg +#define HAILORT_LOCAL_FILE_LOGGER_PATTERN ("[%Y-%m-%d %X.%e] [%t] [%n] [%l] [%s:%#] [%!] %v") // File logger will print: [timestamp] [TID] [hailort] [log level] [source file:line number] [function name] msg #define HAILORT_ANDROID_LOGGER_PATTERN ("%v") // Android logger will print only message (additional info are built-in) #define HAILORT_LOGGER_PATH_ENV_VAR ("HAILORT_LOGGER_PATH") @@ -84,6 +90,30 @@ std::string HailoRTLogger::get_main_log_path() const auto hailo_dir_path = std::string(local_app_data_path) + PATH_SEPARATOR + "Hailo"; const auto full_path = hailo_dir_path + PATH_SEPARATOR + "HailoRT"; + +#ifdef HAILO_SUPPORT_MULTI_PROCESS + TCHAR program_data_path[MAX_PATH]; + auto ret_val = SHGetFolderPath(NULL, CSIDL_COMMON_APPDATA, NULL, 0, program_data_path); + if (!SUCCEEDED(ret_val)) { + std::cerr << "Cannot resolve ProgramData directory path" << std::endl; + return ""; + } + + const auto hailort_service_dir_path = std::string(program_data_path) + PATH_SEPARATOR + "HailoRT_Service"; + auto create_status = Filesystem::create_directory(hailort_service_dir_path); + if (HAILO_SUCCESS != create_status) { + std::cerr << "Cannot create directory at path " << hailort_service_dir_path << std::endl; + return ""; + } + + const auto hailort_service_full_path = std::string(program_data_path) + PATH_SEPARATOR + "HailoRT_Service" + PATH_SEPARATOR + "logs"; + create_status = Filesystem::create_directory(hailort_service_full_path); + if (HAILO_SUCCESS != create_status) { + std::cerr << "Cannot create directory at path " << hailort_service_full_path << std::endl; + return ""; + } +#endif + #else const auto hailo_dir_path = Filesystem::get_home_directory() + PATH_SEPARATOR + ".hailo"; const auto full_path = hailo_dir_path + PATH_SEPARATOR + "hailort"; diff --git a/hailort/libhailort/src/hailort_logger.hpp b/hailort/libhailort/src/utils/hailort_logger.hpp similarity index 100% rename from hailort/libhailort/src/hailort_logger.hpp rename to hailort/libhailort/src/utils/hailort_logger.hpp diff --git a/hailort/libhailort/src/utils/profiler/CMakeLists.txt b/hailort/libhailort/src/utils/profiler/CMakeLists.txt new file mode 100644 index 0000000..f5c91fa --- /dev/null +++ b/hailort/libhailort/src/utils/profiler/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/tracer.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/tracer.cpp b/hailort/libhailort/src/utils/profiler/tracer.cpp similarity index 80% rename from hailort/libhailort/src/tracer.cpp rename to hailort/libhailort/src/utils/profiler/tracer.cpp index 28982a7..175f67f 100644 --- a/hailort/libhailort/src/tracer.cpp +++ b/hailort/libhailort/src/utils/profiler/tracer.cpp @@ -8,19 +8,19 @@ * **/ - -#include "tracer.hpp" #include "common/utils.hpp" -#include "hailort_logger.hpp" + +#include "utils/hailort_logger.hpp" +#include "utils/profiler/tracer.hpp" #include #include #include #include - #include #include + #define SCHEDULER_PROFILER_NAME ("SchedulerProfiler") #define SCHEDULER_PROFILER_LOGGER_FILENAME ("scheduler_profiler.json") #define SCHEDULER_PROFILER_LOGGER_PATTERN ("%v") @@ -38,7 +38,7 @@ Tracer::Tracer() m_should_trace = ((nullptr != should_trace_env) && (strnlen(should_trace_env, 2) == 1) && (strncmp(should_trace_env, "1", 1) == 0)); if (m_should_trace) { m_start_time = std::chrono::high_resolution_clock::now(); - int64_t time_since_epoch = std::chrono::duration_cast(m_start_time.time_since_epoch()).count(); + int64_t time_since_epoch = std::chrono::duration_cast(m_start_time.time_since_epoch()).count(); m_handlers.push_back(std::make_unique(time_since_epoch)); } } @@ -55,7 +55,7 @@ SchedulerProfilerHandler::SchedulerProfilerHandler(int64_t &start_time) m_file_sink->set_level(spdlog::level::level_enum::info); m_file_sink->set_pattern(SCHEDULER_PROFILER_LOGGER_PATTERN); std::stringstream ss; - ss << "{\"ms_since_epoch_zero_time\": \"" << start_time << "\",\n\"scheduler_actions\": [\n"; + ss << "{\"ns_since_epoch_zero_time\": \"" << start_time << "\",\n\"scheduler_actions\": [\n"; m_profiler_logger->info(ss.str()); #else (void)start_time; @@ -124,38 +124,38 @@ void SchedulerProfilerHandler::log(JSON json) m_profiler_logger->info("{}{}", comma() ? ",\n" : "", json_to_string(json)); } -void SchedulerProfilerHandler::handle_trace(const AddNetworkGroupTrace &trace) +void SchedulerProfilerHandler::handle_trace(const AddCoreOpTrace &trace) { log(JSON({ {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"network_group_name", json_to_string(trace.network_group_name)}, - {"network_group_handle", json_to_string(trace.network_group_handle)}, + {"core_op_name", json_to_string(trace.core_op_name)}, + {"core_op_handle", json_to_string(trace.core_op_handle)}, {"timeout", json_to_string((uint64_t)trace.timeout)}, {"threshold", json_to_string((uint64_t)trace.threshold)} })); } -void SchedulerProfilerHandler::handle_trace(const CreateNetworkGroupInputStreamsTrace &trace) +void SchedulerProfilerHandler::handle_trace(const CreateCoreOpInputStreamsTrace &trace) { log(JSON({ {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"network_group_name", json_to_string(trace.network_group_name)}, + {"core_op_name", json_to_string(trace.core_op_name)}, {"stream_name", json_to_string(trace.stream_name)}, {"queue_size", json_to_string(trace.queue_size)} })); } -void SchedulerProfilerHandler::handle_trace(const CreateNetworkGroupOutputStreamsTrace &trace) +void SchedulerProfilerHandler::handle_trace(const CreateCoreOpOutputStreamsTrace &trace) { log(JSON({ {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"network_group_name", json_to_string(trace.network_group_name)}, + {"core_op_name", json_to_string(trace.core_op_name)}, {"stream_name", json_to_string(trace.stream_name)}, {"queue_size", json_to_string(trace.queue_size)} })); @@ -167,18 +167,18 @@ void SchedulerProfilerHandler::handle_trace(const WriteFrameTrace &trace) {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"network_group_handle", json_to_string(trace.network_group_handle)}, + {"core_op_handle", json_to_string(trace.core_op_handle)}, {"queue_name", json_to_string(trace.queue_name)} })); } -void SchedulerProfilerHandler::handle_trace(const InputVdmaEnqueueTrace &trace) +void SchedulerProfilerHandler::handle_trace(const InputVdmaDequeueTrace &trace) { log(JSON({ {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"network_group_handle", json_to_string(trace.network_group_handle)}, + {"core_op_handle", json_to_string(trace.core_op_handle)}, {"queue_name", json_to_string(trace.queue_name)} })); } @@ -189,7 +189,7 @@ void SchedulerProfilerHandler::handle_trace(const ReadFrameTrace &trace) {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"network_group_handle", json_to_string(trace.network_group_handle)}, + {"core_op_handle", json_to_string(trace.core_op_handle)}, {"queue_name", json_to_string(trace.queue_name)} })); } @@ -200,31 +200,32 @@ void SchedulerProfilerHandler::handle_trace(const OutputVdmaEnqueueTrace &trace) {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"network_group_handle", json_to_string(trace.network_group_handle)}, + {"core_op_handle", json_to_string(trace.core_op_handle)}, {"queue_name", json_to_string(trace.queue_name)}, {"frames", json_to_string(trace.frames)} })); } -void SchedulerProfilerHandler::handle_trace(const ChooseNetworkGroupTrace &trace) +void SchedulerProfilerHandler::handle_trace(const ChooseCoreOpTrace &trace) { log(JSON({ {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"chosen_network_group_handle", json_to_string(trace.network_group_handle)}, + {"chosen_core_op_handle", json_to_string(trace.core_op_handle)}, {"threshold", json_to_string(trace.threshold)}, - {"timeout", json_to_string(trace.timeout)} + {"timeout", json_to_string(trace.timeout)}, + {"priority", json_to_string(trace.priority)} })); } -void SchedulerProfilerHandler::handle_trace(const SwitchNetworkGroupTrace &trace) +void SchedulerProfilerHandler::handle_trace(const SwitchCoreOpTrace &trace) { log(JSON({ {"action", json_to_string(trace.name)}, {"timestamp", json_to_string(trace.timestamp)}, {"device_id", json_to_string(trace.device_id)}, - {"network_group_handle", json_to_string(trace.network_group_handle)} + {"core_op_handle", json_to_string(trace.core_op_handle)} })); } diff --git a/hailort/libhailort/src/utils/profiler/tracer.hpp b/hailort/libhailort/src/utils/profiler/tracer.hpp new file mode 100644 index 0000000..369079f --- /dev/null +++ b/hailort/libhailort/src/utils/profiler/tracer.hpp @@ -0,0 +1,242 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file tracer.hpp + * @brief Tracing mechanism for HailoRT + FW events + **/ + +#ifndef _HAILO_TRACER_HPP_ +#define _HAILO_TRACER_HPP_ + +#include "hailo/hailort.h" +#include "common/logger_macros.hpp" + +#include "vdevice/scheduler/scheduler_base.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace hailort +{ + +struct Trace +{ + Trace(const std::string &name) + : name(name) + {} + + virtual ~Trace() = default; + + uint64_t timestamp = 0; + std::string name; +}; + +struct InitTrace : Trace +{ + InitTrace() : Trace("init") {} +}; + +struct AddCoreOpTrace : Trace +{ + AddCoreOpTrace(const std::string &device_id, const std::string &core_op_name, uint64_t timeout, uint32_t threshold, scheduler_core_op_handle_t handle) + : Trace("add_core_op"), device_id(device_id), core_op_name(core_op_name), timeout(timeout), threshold(threshold), core_op_handle(handle) + {} + + std::string device_id; + std::string core_op_name; + uint64_t timeout = 0; + uint32_t threshold = 0; + scheduler_core_op_handle_t core_op_handle = INVALID_CORE_OP_HANDLE; +}; + +struct CreateCoreOpInputStreamsTrace : Trace +{ + CreateCoreOpInputStreamsTrace(const std::string &device_id, const std::string &core_op_name, const std::string &stream_name, uint32_t queue_size) + : Trace("create_input_stream"), device_id(device_id), core_op_name(core_op_name), stream_name(stream_name), queue_size(queue_size) + {} + + std::string device_id; + std::string core_op_name; + std::string stream_name; + uint32_t queue_size; +}; + +struct CreateCoreOpOutputStreamsTrace : Trace +{ + CreateCoreOpOutputStreamsTrace(const std::string &device_id, const std::string &core_op_name, const std::string &stream_name, uint32_t queue_size) + : Trace("create_output_stream"), device_id(device_id), core_op_name(core_op_name), stream_name(stream_name), queue_size(queue_size) + {} + + std::string device_id; + std::string core_op_name; + std::string stream_name; + uint32_t queue_size; +}; + +struct WriteFrameTrace : Trace +{ + WriteFrameTrace(const std::string &device_id, scheduler_core_op_handle_t core_op_handle, const std::string &queue_name) + : Trace("write_frame"), device_id(device_id), core_op_handle(core_op_handle), queue_name(queue_name) + {} + + std::string device_id; + scheduler_core_op_handle_t core_op_handle; + std::string queue_name; +}; + +struct InputVdmaDequeueTrace : Trace +{ + InputVdmaDequeueTrace(const std::string &device_id, scheduler_core_op_handle_t core_op_handle, const std::string &queue_name) + : Trace("input_vdma_dequeue"), device_id(device_id), core_op_handle(core_op_handle), queue_name(queue_name) + {} + + std::string device_id; + scheduler_core_op_handle_t core_op_handle; + std::string queue_name; +}; + +struct ReadFrameTrace : Trace +{ + ReadFrameTrace(const std::string &device_id, scheduler_core_op_handle_t core_op_handle, const std::string &queue_name) + : Trace("read_frame"), device_id(device_id), core_op_handle(core_op_handle), queue_name(queue_name) + {} + + std::string device_id; + scheduler_core_op_handle_t core_op_handle; + std::string queue_name; +}; + +struct OutputVdmaEnqueueTrace : Trace +{ + OutputVdmaEnqueueTrace(const std::string &device_id, scheduler_core_op_handle_t core_op_handle, const std::string &queue_name, uint32_t frames) + : Trace("output_vdma_enqueue"), device_id(device_id), core_op_handle(core_op_handle), queue_name(queue_name), frames(frames) + {} + + std::string device_id; + scheduler_core_op_handle_t core_op_handle; + std::string queue_name; + uint32_t frames = 0; +}; + +struct ChooseCoreOpTrace : Trace +{ + ChooseCoreOpTrace(const std::string &device_id, scheduler_core_op_handle_t handle, bool threshold, bool timeout, core_op_priority_t priority) + : Trace("choose_core_op"), device_id(device_id), core_op_handle(handle), threshold(threshold), timeout(timeout), priority(priority) + {} + + std::string device_id; + scheduler_core_op_handle_t core_op_handle; + bool threshold = false; + bool timeout = false; + core_op_priority_t priority; +}; + +struct SwitchCoreOpTrace : Trace +{ + SwitchCoreOpTrace(const std::string &device_id, scheduler_core_op_handle_t handle) + : Trace("switch_core_op"), device_id(device_id), core_op_handle(handle) + {} + + std::string device_id; + scheduler_core_op_handle_t core_op_handle; +}; + +class Handler +{ +public: + virtual ~Handler() = default; + + virtual void handle_trace(const InitTrace&) {}; + virtual void handle_trace(const AddCoreOpTrace&) {}; + virtual void handle_trace(const CreateCoreOpInputStreamsTrace&) {}; + virtual void handle_trace(const CreateCoreOpOutputStreamsTrace&) {}; + virtual void handle_trace(const WriteFrameTrace&) {}; + virtual void handle_trace(const InputVdmaDequeueTrace&) {}; + virtual void handle_trace(const ReadFrameTrace&) {}; + virtual void handle_trace(const OutputVdmaEnqueueTrace&) {}; + virtual void handle_trace(const ChooseCoreOpTrace&) {}; + virtual void handle_trace(const SwitchCoreOpTrace&) {}; +}; + +struct JSON; + +class SchedulerProfilerHandler : public Handler +{ +public: + SchedulerProfilerHandler(SchedulerProfilerHandler const&) = delete; + void operator=(SchedulerProfilerHandler const&) = delete; + + SchedulerProfilerHandler(int64_t &start_time); + ~SchedulerProfilerHandler(); + + virtual void handle_trace(const AddCoreOpTrace&) override; + virtual void handle_trace(const CreateCoreOpInputStreamsTrace&) override; + virtual void handle_trace(const CreateCoreOpOutputStreamsTrace&) override; + virtual void handle_trace(const WriteFrameTrace&) override; + virtual void handle_trace(const InputVdmaDequeueTrace&) override; + virtual void handle_trace(const ReadFrameTrace&) override; + virtual void handle_trace(const OutputVdmaEnqueueTrace&) override; + virtual void handle_trace(const ChooseCoreOpTrace&) override; + virtual void handle_trace(const SwitchCoreOpTrace&) override; + +private: + void log(JSON json); + bool comma(); + + std::shared_ptr m_file_sink; + std::shared_ptr m_profiler_logger; + std::atomic m_first_write; +}; + +class Tracer +{ +public: + template + static void trace(Args... trace_args) + { + auto &tracer = get_instance(); + tracer.execute_trace(trace_args...); + } + +private: + Tracer(); + + static Tracer& get_instance() + { + static Tracer tracer; + return tracer; + } + + template + void execute_trace(Args... trace_args) + { + if (!m_should_trace) { + return; + } + + TraceType trace_struct(trace_args...); + auto curr_time = std::chrono::high_resolution_clock::now(); + trace_struct.timestamp = std::chrono::duration_cast(curr_time - this->m_start_time).count(); + for (auto &handler : this->m_handlers) { + handler->handle_trace(trace_struct); + } + } + + bool m_should_trace = false; + std::chrono::high_resolution_clock::time_point m_start_time; + std::vector> m_handlers; +}; + +} + +#endif \ No newline at end of file diff --git a/hailort/libhailort/src/tracer_macros.hpp b/hailort/libhailort/src/utils/profiler/tracer_macros.hpp similarity index 100% rename from hailort/libhailort/src/tracer_macros.hpp rename to hailort/libhailort/src/utils/profiler/tracer_macros.hpp diff --git a/hailort/libhailort/src/sensor_config_utils.cpp b/hailort/libhailort/src/utils/sensor_config_utils.cpp similarity index 99% rename from hailort/libhailort/src/sensor_config_utils.cpp rename to hailort/libhailort/src/utils/sensor_config_utils.cpp index 3f67079..3d41acf 100644 --- a/hailort/libhailort/src/sensor_config_utils.cpp +++ b/hailort/libhailort/src/utils/sensor_config_utils.cpp @@ -7,14 +7,16 @@ * @brief Utilities for sensor_config operations **/ -#include "sensor_config_utils.hpp" #include "common/string_utils.hpp" #include "common/utils.hpp" +#include "utils/sensor_config_utils.hpp" + #include #include #include + namespace hailort { diff --git a/hailort/libhailort/src/sensor_config_utils.hpp b/hailort/libhailort/src/utils/sensor_config_utils.hpp similarity index 100% rename from hailort/libhailort/src/sensor_config_utils.hpp rename to hailort/libhailort/src/utils/sensor_config_utils.hpp diff --git a/hailort/libhailort/src/shared_resource_manager.hpp b/hailort/libhailort/src/utils/shared_resource_manager.hpp similarity index 100% rename from hailort/libhailort/src/shared_resource_manager.hpp rename to hailort/libhailort/src/utils/shared_resource_manager.hpp diff --git a/hailort/libhailort/src/thread_safe_map.hpp b/hailort/libhailort/src/utils/thread_safe_map.hpp similarity index 100% rename from hailort/libhailort/src/thread_safe_map.hpp rename to hailort/libhailort/src/utils/thread_safe_map.hpp diff --git a/hailort/libhailort/src/thread_safe_queue.hpp b/hailort/libhailort/src/utils/thread_safe_queue.hpp similarity index 99% rename from hailort/libhailort/src/thread_safe_queue.hpp rename to hailort/libhailort/src/utils/thread_safe_queue.hpp index 28ec72e..6d8646f 100644 --- a/hailort/libhailort/src/thread_safe_queue.hpp +++ b/hailort/libhailort/src/utils/thread_safe_queue.hpp @@ -11,10 +11,12 @@ #define HAILO_THREAD_SAFE_QUEUE_HPP_ #include "hailo/expected.hpp" -#include "common/utils.hpp" #include "hailo/event.hpp" + +#include "common/utils.hpp" #include "common/logger_macros.hpp" -#include "event_internal.hpp" + +#include "utils/event_internal.hpp" // Define __unix__ for inclusion of readerwriterqueue.h because readerwriterqueue is implemented over POSIX standards // but checks __unix__ - otherwise QNX returns unsupported platform (need HAILO_UNDEF_UNIX_FLAG in order to undefine @@ -37,6 +39,7 @@ #include #include + namespace hailort { diff --git a/hailort/libhailort/src/vdevice/CMakeLists.txt b/hailort/libhailort/src/vdevice/CMakeLists.txt new file mode 100644 index 0000000..cacefd2 --- /dev/null +++ b/hailort/libhailort/src/vdevice/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/vdevice.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_core_op.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/pipeline_multiplexer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_stream.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdevice_stream_multiplexer_wrapper.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/scheduler/network_group_scheduler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/scheduler/scheduler_oracle.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/scheduler/scheduled_core_op_state.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/scheduler/multi_device_scheduled_stream.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/vdevice/pipeline_multiplexer.cpp b/hailort/libhailort/src/vdevice/pipeline_multiplexer.cpp new file mode 100644 index 0000000..3526662 --- /dev/null +++ b/hailort/libhailort/src/vdevice/pipeline_multiplexer.cpp @@ -0,0 +1,454 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file pipeline_multiplexer.cpp + * @brief: Pipeline Multiplexer + **/ + +#include "hailo/hailort_common.hpp" +#include "hailo/vstream.hpp" + +#include "common/utils.hpp" + +#include "vdevice/pipeline_multiplexer.hpp" + + +namespace hailort +{ + +PipelineMultiplexer::PipelineMultiplexer() : + m_should_core_op_stop(), + m_input_streams_count(0), + m_output_streams_count(0), + m_next_to_write(0), + m_order_queue(), + m_currently_writing(INVALID_CORE_OP_HANDLE), + m_written_streams_count(0), + m_read_streams_count(0), + m_next_to_read_after_drain(INVALID_CORE_OP_HANDLE) +{} + +bool PipelineMultiplexer::should_use_multiplexer() +{ + auto disable_multiplexer_env = std::getenv(DISABLE_MULTIPLEXER_ENV_VAR); + if ((nullptr != disable_multiplexer_env) && (strnlen(disable_multiplexer_env, 2) == 1) && (strncmp(disable_multiplexer_env, "1", 1) == 0)) { + LOGGER__WARNING("Usage of '{}' env variable is deprecated.", DISABLE_MULTIPLEXER_ENV_VAR); + return false; + } + return true; +} + +hailo_status PipelineMultiplexer::add_core_op_instance(multiplexer_core_op_handle_t core_op_handle, CoreOp &core_op) +{ + std::unique_lock lock(m_writing_mutex); + std::unique_lock read_lock(m_reading_mutex); + assert(!contains(m_should_core_op_stop, core_op_handle)); + + auto is_first_instance = (0 == instances_count()); + + auto stream_infos = core_op.get_all_stream_infos(); + CHECK_EXPECTED_AS_STATUS(stream_infos); + + for (const auto &stream_info : stream_infos.value()) { + m_should_core_op_stop[core_op_handle][stream_info.name] = false; + if (is_first_instance) { + // To be filled only on first instance + if (HAILO_H2D_STREAM == stream_info.direction) { + m_input_streams_count++; + } else { + m_output_streams_count++; + m_is_stream_reading[stream_info.name] = false; + } + } + } + + m_write_barriers[core_op_handle] = make_shared_nothrow(m_input_streams_count); + CHECK(nullptr != m_write_barriers[core_op_handle], HAILO_OUT_OF_HOST_MEMORY); + m_is_waiting_to_write[core_op_handle] = false; + + return HAILO_SUCCESS; +} + +void PipelineMultiplexer::set_output_vstreams_names(multiplexer_core_op_handle_t core_op_handle, const std::vector &output_vstreams) +{ + std::unique_lock lock(m_writing_mutex); + for (const auto &output_vstream : output_vstreams) { + m_can_output_vstream_read[core_op_handle][output_vstream.name()] = true; + } + m_can_core_op_read[core_op_handle] = true; +} + +bool PipelineMultiplexer::has_more_than_one_core_op_instance() const +{ + return instances_count() > 1; +} + +size_t PipelineMultiplexer::instances_count() const +{ + return m_should_core_op_stop.size(); +} + +bool PipelineMultiplexer::should_core_op_stop(multiplexer_core_op_handle_t core_op_handle) +{ + for (const auto &name_flag_pair : m_should_core_op_stop[core_op_handle]) { + if (name_flag_pair.second) { + return true; + } + } + + return false; +} + +hailo_status PipelineMultiplexer::wait_for_write(multiplexer_core_op_handle_t core_op_handle) +{ + std::shared_ptr barrier; + { + std::unique_lock lock(m_writing_mutex); + assert(contains(m_write_barriers, core_op_handle)); + barrier = m_write_barriers[core_op_handle]; + } + // TODO: This has no timeout + // TODO: HRT-8634 + barrier->arrive_and_wait(); + { + std::unique_lock lock(m_writing_mutex); + assert(contains(m_should_core_op_stop, core_op_handle)); + assert(contains(m_is_waiting_to_write, core_op_handle)); + + m_is_waiting_to_write[core_op_handle] = true; + hailo_status status = HAILO_SUCCESS; + m_writing_cv.wait(lock, [this, core_op_handle, &status] { + if (!has_more_than_one_core_op_instance() || !should_use_multiplexer()) { + return true; + } + + if (should_core_op_stop(core_op_handle)) { + status = HAILO_STREAM_ABORTED_BY_USER; + return true; // return true so that the wait will finish + } + + if (m_currently_writing == core_op_handle) { + return true; + } + + if (!can_core_op_read(core_op_handle)) { + return false; + } + + if (INVALID_CORE_OP_HANDLE == m_currently_writing) { + if ((m_next_to_write != core_op_handle) && m_is_waiting_to_write[m_next_to_write] && can_core_op_read(m_next_to_write)) { + return false; + } + + return true; + } + + return false; + }); + m_is_waiting_to_write[core_op_handle] = false; + + if (HAILO_STREAM_ABORTED_BY_USER == status) { + return status; + } + CHECK_SUCCESS(status); + + if (INVALID_CORE_OP_HANDLE == m_currently_writing) { + m_currently_writing = core_op_handle; + m_next_to_write = m_currently_writing; + } + } + m_writing_cv.notify_all(); + + return HAILO_SUCCESS; +} + +bool PipelineMultiplexer::can_core_op_read(multiplexer_core_op_handle_t core_op_handle) +{ + if (should_core_op_stop(core_op_handle)) { + return false; + } + + if (!contains(m_can_core_op_read, core_op_handle)) { + return true; + } + + return m_can_core_op_read[core_op_handle]; +} + +hailo_status PipelineMultiplexer::signal_write_finish(multiplexer_core_op_handle_t core_op_handle, bool did_write_fail) +{ + { + std::unique_lock lock(m_writing_mutex); + m_written_streams_count++; + if (m_written_streams_count == m_input_streams_count) { + m_written_streams_count = 0; + m_currently_writing = INVALID_CORE_OP_HANDLE; + m_next_to_write++; + m_next_to_write %= static_cast(instances_count()); + + if (!did_write_fail) { + std::unique_lock reading_lock(m_reading_mutex); + m_order_queue.push_back(core_op_handle); + } + m_reading_cv.notify_all(); + } + } + + m_writing_cv.notify_all(); + return HAILO_SUCCESS; +} + +Expected PipelineMultiplexer::wait_for_read(multiplexer_core_op_handle_t core_op_handle, const std::string &stream_name, + const std::chrono::milliseconds &timeout) +{ + uint32_t drain_frames = 0; + + { + std::unique_lock lock(m_reading_mutex); + + assert(contains(m_should_core_op_stop, core_op_handle)); + assert(contains(m_is_stream_reading, stream_name)); + + hailo_status status = HAILO_SUCCESS; + auto wait_res = m_reading_cv.wait_for(lock, timeout, [this, core_op_handle, stream_name, &drain_frames, &status] { + if (should_core_op_stop(core_op_handle)) { + status = HAILO_STREAM_ABORTED_BY_USER; + return true; // return true so that the wait will finish + } + if (m_is_stream_reading[stream_name]) { + return false; + } + + if (m_next_to_read_after_drain == core_op_handle) { + drain_frames = m_num_frames_to_drain[stream_name]; + return true; + } + + if (m_order_queue.empty()) { + return false; + } + + if (m_order_queue.front() != core_op_handle) { + if (!should_core_op_stop(m_order_queue.front())) { + return false; + } + + // This means the NG that is currently writing was aborted so we have to wait for it to finish processing its frames + if ((INVALID_CORE_OP_HANDLE != m_currently_writing) && (m_currently_writing != core_op_handle)) { + return false; + } + + uint32_t max_drain_count = get_frame_count_to_drain(core_op_handle); + if (0 == max_drain_count) { + return false; + } + + drain_frames = drain_aborted_in_order_queue(core_op_handle, stream_name, max_drain_count); + } + + return true; + }); + CHECK_AS_EXPECTED(wait_res, HAILO_TIMEOUT, "{} (D2H) failed with status={}, timeout={}ms", stream_name, HAILO_TIMEOUT, timeout.count()); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + return make_unexpected(status); + } + CHECK_SUCCESS_AS_EXPECTED(status); + + m_is_stream_reading[stream_name] = true; + } + + m_reading_cv.notify_all(); + return drain_frames; +} + +uint32_t PipelineMultiplexer::get_frame_count_to_drain(multiplexer_core_op_handle_t core_op_handle) +{ + uint32_t drain_count = 0; + for (const auto &handle : m_order_queue) { + if (!should_core_op_stop(handle)) { + if (handle == core_op_handle) { + // Current instance is in the front after draining + break; + } else { + // Someone else should drain these frames, the current instance won't be in front after draining + return 0; + } + } + + drain_count++; + } + + return drain_count; +} + +uint32_t PipelineMultiplexer::drain_aborted_in_order_queue(multiplexer_core_op_handle_t core_op_handle, const std::string &stream_name, + uint32_t max_drain_count) +{ + // In case of multiple outputs where one or more already read the frame we need to drain one less frame + for (auto &name_flag_pair : m_is_stream_reading) { + if (name_flag_pair.second) { + m_num_frames_to_drain[name_flag_pair.first] = max_drain_count - 1; + } else { + m_num_frames_to_drain[name_flag_pair.first] = max_drain_count; + } + } + + m_next_to_read_after_drain = core_op_handle; + m_read_streams_count = 0; + for (uint32_t i = 0; i < max_drain_count; i++) { + for (auto &name_flag_pair : m_is_stream_reading) { + name_flag_pair.second = false; + } + m_order_queue.pop_front(); + } + + return m_num_frames_to_drain[stream_name]; +} + +hailo_status PipelineMultiplexer::signal_read_finish() +{ + std::unique_lock lock(m_reading_mutex); + + m_read_streams_count++; + if (m_read_streams_count == m_output_streams_count) { + m_read_streams_count = 0; + m_order_queue.pop_front(); + for (auto &name_flag_pair : m_is_stream_reading) { + name_flag_pair.second = false; + } + + m_next_to_read_after_drain = INVALID_CORE_OP_HANDLE; + + lock.unlock(); + m_reading_cv.notify_all(); + } + + return HAILO_SUCCESS; +} + +hailo_status PipelineMultiplexer::enable_stream(multiplexer_core_op_handle_t core_op_handle, const std::string &stream_name) +{ + { + std::unique_lock write_lock(m_writing_mutex); + std::unique_lock read_lock(m_reading_mutex); + assert(contains(m_should_core_op_stop, core_op_handle)); + assert(contains(m_should_core_op_stop[core_op_handle], stream_name)); + + if (!m_should_core_op_stop[core_op_handle][stream_name]) { + return HAILO_SUCCESS; + } + + m_should_core_op_stop[core_op_handle][stream_name] = false; + + // TODO: should we 'enable' barrier? + } + + m_writing_cv.notify_all(); + m_reading_cv.notify_all(); + + return HAILO_SUCCESS; +} + +hailo_status PipelineMultiplexer::disable_stream(multiplexer_core_op_handle_t core_op_handle, const std::string &stream_name) +{ + { + std::unique_lock write_lock(m_writing_mutex); + std::unique_lock read_lock(m_reading_mutex); + assert(contains(m_should_core_op_stop, core_op_handle)); + assert(contains(m_should_core_op_stop[core_op_handle], stream_name)); + + if (m_should_core_op_stop[core_op_handle][stream_name]) { + return HAILO_SUCCESS; + } + + m_should_core_op_stop[core_op_handle][stream_name] = true; + + assert(contains(m_write_barriers, core_op_handle)); + m_write_barriers[core_op_handle]->terminate(); + } + + m_writing_cv.notify_all(); + m_reading_cv.notify_all(); + + return HAILO_SUCCESS; +} + +void PipelineMultiplexer::RunOnceForStream::add_instance() +{ + std::unique_lock lock(m_mutex); + m_calls_count[static_cast(m_calls_count.size())] = 0; +} + +void PipelineMultiplexer::RunOnceForStream::set_callback(std::function callback) +{ + std::unique_lock lock(m_mutex); + m_callback = callback; +} + +hailo_status PipelineMultiplexer::RunOnceForStream::run(multiplexer_core_op_handle_t core_op_handle) +{ + std::unique_lock lock(m_mutex); + assert(contains(m_calls_count, core_op_handle)); + + m_calls_count[core_op_handle]++; + for (auto &handle_flag_pair : m_calls_count) { + if (0 == handle_flag_pair.second) { + return HAILO_SUCCESS; + } + } + + for (auto &handle_flag_pair : m_calls_count) { + handle_flag_pair.second--; + } + + return m_callback(); +} + +hailo_status PipelineMultiplexer::register_run_once_for_stream(const std::string &stream_name, run_once_for_stream_handle_t handle, + std::function callback) +{ + std::unique_lock lock(m_register_run_once_mutex); + if (!contains(m_run_once_db[stream_name], handle)) { + m_run_once_db[stream_name][handle] = make_shared_nothrow(); + CHECK(nullptr != m_run_once_db[stream_name][handle], HAILO_OUT_OF_HOST_MEMORY); + + m_run_once_db[stream_name][handle]->set_callback(callback); + } + + m_run_once_db[stream_name][handle]->add_instance(); + + return HAILO_SUCCESS; +} + +hailo_status PipelineMultiplexer::run_once_for_stream(const std::string &stream_name, run_once_for_stream_handle_t run_once_handle, + multiplexer_core_op_handle_t core_op_handle) +{ + return m_run_once_db[stream_name][run_once_handle]->run(core_op_handle); +} + +void PipelineMultiplexer::set_can_output_vstream_read(multiplexer_core_op_handle_t core_op_handle, const std::string &vstream_name, bool can_read) +{ + { + std::unique_lock lock(m_writing_mutex); + assert(contains(m_can_output_vstream_read, core_op_handle)); + assert(contains(m_can_output_vstream_read[core_op_handle], vstream_name)); + assert(contains(m_can_core_op_read, core_op_handle)); + + m_can_output_vstream_read[core_op_handle][vstream_name] = can_read; + + if (can_read != m_can_core_op_read[core_op_handle]) { + m_can_core_op_read[core_op_handle] = true; + for (const auto &name_bool_pair : m_can_output_vstream_read[core_op_handle]) { + if (!name_bool_pair.second) { + m_can_core_op_read[core_op_handle] = false; + break; + } + } + } + } + m_writing_cv.notify_all(); +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/pipeline_multiplexer.hpp b/hailort/libhailort/src/vdevice/pipeline_multiplexer.hpp similarity index 52% rename from hailort/libhailort/src/pipeline_multiplexer.hpp rename to hailort/libhailort/src/vdevice/pipeline_multiplexer.hpp index 3187724..e781697 100644 --- a/hailort/libhailort/src/pipeline_multiplexer.hpp +++ b/hailort/libhailort/src/vdevice/pipeline_multiplexer.hpp @@ -12,19 +12,21 @@ #define _HAILO_PIPELINE_MULTIPLEXER_HPP_ #include "hailo/event.hpp" -#include "hailo/network_group.hpp" -#include "network_group_scheduler.hpp" + #include "common/barrier.hpp" +#include "vdevice/scheduler/network_group_scheduler.hpp" + #include #include + namespace hailort { #define DISABLE_MULTIPLEXER_ENV_VAR "HAILO_DISABLE_MULTIPLEXER" -using multiplexer_ng_handle_t = uint32_t; +using multiplexer_core_op_handle_t = uint32_t; using run_once_for_stream_handle_t = uint32_t; class PipelineMultiplexer @@ -38,54 +40,57 @@ public: PipelineMultiplexer &operator=(PipelineMultiplexer &&other) = delete; PipelineMultiplexer(PipelineMultiplexer &&other) = delete; - hailo_status add_network_group_instance(multiplexer_ng_handle_t network_group_handle, ConfiguredNetworkGroup &network_group); - void set_output_vstreams_names(multiplexer_ng_handle_t network_group_handle, const std::vector &output_vstreams); - bool has_more_than_one_ng_instance() const; + hailo_status add_core_op_instance(multiplexer_core_op_handle_t core_op_handle, CoreOp &core_op); + void set_output_vstreams_names(multiplexer_core_op_handle_t core_op_handle, const std::vector &output_vstreams); + bool has_more_than_one_core_op_instance() const; size_t instances_count() const; - hailo_status wait_for_write(multiplexer_ng_handle_t network_group_handle); - hailo_status signal_write_finish(multiplexer_ng_handle_t network_group_handle); - Expected wait_for_read(multiplexer_ng_handle_t network_group_handle, const std::string &stream_name, + hailo_status wait_for_write(multiplexer_core_op_handle_t core_op_handle); + hailo_status signal_write_finish(multiplexer_core_op_handle_t core_op_handle, bool did_write_fail); + Expected wait_for_read(multiplexer_core_op_handle_t core_op_handle, const std::string &stream_name, const std::chrono::milliseconds &timeout); - hailo_status signal_read_finish(multiplexer_ng_handle_t network_group_handle); - hailo_status enable_network_group(multiplexer_ng_handle_t network_group_handle); - hailo_status disable_network_group(multiplexer_ng_handle_t network_group_handle); + hailo_status signal_read_finish(); + hailo_status enable_stream(multiplexer_core_op_handle_t core_op_handle, const std::string &stream_name); + hailo_status disable_stream(multiplexer_core_op_handle_t core_op_handle, const std::string &stream_name); hailo_status register_run_once_for_stream(const std::string &stream_name, run_once_for_stream_handle_t handle, std::function callback); hailo_status run_once_for_stream(const std::string &stream_name, run_once_for_stream_handle_t run_once_handle, - multiplexer_ng_handle_t network_group_handle); + multiplexer_core_op_handle_t core_op_handle); - void set_can_output_vstream_read(multiplexer_ng_handle_t network_group_handle, const std::string &vstream_name, bool can_read); + void set_can_output_vstream_read(multiplexer_core_op_handle_t core_op_handle, const std::string &vstream_name, bool can_read); static bool should_use_multiplexer(); private: - std::unordered_map m_should_ng_stop; - std::unordered_map m_is_waiting_to_write; + + bool should_core_op_stop(multiplexer_core_op_handle_t core_op_handle); + + std::unordered_map> m_should_core_op_stop; + std::unordered_map m_is_waiting_to_write; uint32_t m_input_streams_count; uint32_t m_output_streams_count; - multiplexer_ng_handle_t m_next_to_write; - std::unordered_map> m_write_barriers; - std::deque m_order_queue; + multiplexer_core_op_handle_t m_next_to_write; + std::unordered_map> m_write_barriers; + std::deque m_order_queue; std::mutex m_writing_mutex; std::condition_variable m_writing_cv; - multiplexer_ng_handle_t m_currently_writing; + multiplexer_core_op_handle_t m_currently_writing; std::atomic_uint32_t m_written_streams_count; - std::unordered_map> m_is_stream_reading; + std::unordered_map m_is_stream_reading; std::mutex m_reading_mutex; std::condition_variable m_reading_cv; std::atomic_uint32_t m_read_streams_count; std::unordered_map m_num_frames_to_drain; - multiplexer_ng_handle_t m_next_to_read_after_drain; + multiplexer_core_op_handle_t m_next_to_read_after_drain; - std::unordered_map> m_can_output_vstream_read; - std::unordered_map m_can_network_group_read; + std::unordered_map> m_can_output_vstream_read; + std::unordered_map m_can_core_op_read; - bool can_network_group_read(multiplexer_ng_handle_t network_group_handle); - uint32_t get_frame_count_to_drain(multiplexer_ng_handle_t network_group_handle); - uint32_t drain_aborted_in_order_queue(multiplexer_ng_handle_t network_group_handle, const std::string &stream_name, uint32_t max_drain_count); + bool can_core_op_read(multiplexer_core_op_handle_t core_op_handle); + uint32_t get_frame_count_to_drain(multiplexer_core_op_handle_t core_op_handle); + uint32_t drain_aborted_in_order_queue(multiplexer_core_op_handle_t core_op_handle, const std::string &stream_name, uint32_t max_drain_count); class RunOnceForStream final { @@ -95,9 +100,9 @@ private: private: void add_instance(); void set_callback(std::function callback); - hailo_status run(multiplexer_ng_handle_t network_group_handle); + hailo_status run(multiplexer_core_op_handle_t core_op_handle); - std::unordered_map m_was_called; + std::unordered_map m_calls_count; std::function m_callback; std::mutex m_mutex; diff --git a/hailort/libhailort/src/multi_device_scheduled_stream.cpp b/hailort/libhailort/src/vdevice/scheduler/multi_device_scheduled_stream.cpp similarity index 60% rename from hailort/libhailort/src/multi_device_scheduled_stream.cpp rename to hailort/libhailort/src/vdevice/scheduler/multi_device_scheduled_stream.cpp index 3f8949a..7ae77e9 100644 --- a/hailort/libhailort/src/multi_device_scheduled_stream.cpp +++ b/hailort/libhailort/src/vdevice/scheduler/multi_device_scheduled_stream.cpp @@ -9,17 +9,26 @@ * TODO: doc **/ -#include "multi_device_scheduled_stream.hpp" +#include "vdevice/scheduler/multi_device_scheduled_stream.hpp" namespace hailort { hailo_status MultiDeviceScheduledInputStream::send_pending_buffer(size_t device_index) { - auto buffer = dequeue(); + auto buffer = m_queue->front(get_timeout()); // Counting on scheduler to not allow paralle calls to this function + if (HAILO_STREAM_ABORTED_BY_USER == buffer.status()) { + LOGGER__INFO("'front' was aborted."); + return buffer.status(); + } CHECK_EXPECTED_AS_STATUS(buffer); auto status = m_streams[device_index].get().write_buffer_only(buffer.value()); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("send_pending_buffer was aborted."); + return status; + } CHECK_SUCCESS(status); + m_queue->pop(); // Release buffer to free the queue for other dequeues VdmaInputStream &vdma_input = static_cast(m_streams[device_index].get()); return vdma_input.send_pending_buffer(); @@ -28,29 +37,29 @@ hailo_status MultiDeviceScheduledInputStream::send_pending_buffer(size_t device_ Expected MultiDeviceScheduledInputStream::sync_write_raw_buffer(const MemoryView &buffer, const std::function &should_cancel) { - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK_AS_EXPECTED(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK_AS_EXPECTED(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto status = network_group_scheduler->wait_for_write(m_network_group_handle, name(), get_timeout(), should_cancel); + auto status = core_ops_scheduler->wait_for_write(m_core_op_handle, name(), get_timeout(), should_cancel); if (HAILO_STREAM_ABORTED_BY_USER == status) { LOGGER__INFO("Write to stream was aborted."); return make_unexpected(status); } CHECK_SUCCESS_AS_EXPECTED(status); - status = enqueue(buffer); + status = m_queue->push(buffer, get_timeout()); + + auto write_finish_status = core_ops_scheduler->signal_write_finish(m_core_op_handle, name(), status != HAILO_SUCCESS); if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("Enqueue was aborted."); - network_group_scheduler->mark_failed_write(m_network_group_handle, name()); + LOGGER__INFO("'push' was aborted."); return make_unexpected(status); } CHECK_SUCCESS_AS_EXPECTED(status); - status = network_group_scheduler->signal_write_finish(m_network_group_handle, name()); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - return make_unexpected(status); + if (HAILO_STREAM_ABORTED_BY_USER == write_finish_status) { + return make_unexpected(write_finish_status); } - CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_SUCCESS_AS_EXPECTED(write_finish_status); return buffer.size(); } @@ -60,16 +69,6 @@ Expected MultiDeviceScheduledInputStream::get_pending_frames_count() con return get_queue_size(); } -hailo_status MultiDeviceScheduledInputStream::enqueue(const MemoryView &buffer) -{ - return m_queue->enqueue(buffer, get_timeout()); -} - -Expected MultiDeviceScheduledInputStream::dequeue() -{ - return m_queue->dequeue(get_timeout()); -} - size_t MultiDeviceScheduledInputStream::get_queue_size() const { return m_queue->size(); @@ -87,12 +86,12 @@ hailo_status MultiDeviceScheduledInputStream::abort() } m_queue->abort(); - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto disable_status = network_group_scheduler->disable_stream(m_network_group_handle, name()); + auto disable_status = core_ops_scheduler->disable_stream(m_core_op_handle, name()); if (HAILO_SUCCESS != disable_status) { - LOGGER__ERROR("Failed to disable stream in the network group scheduler. (status: {})", disable_status); + LOGGER__ERROR("Failed to disable stream in the core-op scheduler. (status: {})", disable_status); status = disable_status; } @@ -111,12 +110,12 @@ hailo_status MultiDeviceScheduledInputStream::clear_abort() } m_queue->clear_abort(); - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto enable_status = network_group_scheduler->enable_stream(m_network_group_handle, name()); + auto enable_status = core_ops_scheduler->enable_stream(m_core_op_handle, name()); if (HAILO_SUCCESS != enable_status) { - LOGGER__ERROR("Failed to enable stream in the network group scheduler. (status: {})", enable_status); + LOGGER__ERROR("Failed to enable stream in the core-op scheduler. (status: {})", enable_status); status = enable_status; } diff --git a/hailort/libhailort/src/multi_device_scheduled_stream.hpp b/hailort/libhailort/src/vdevice/scheduler/multi_device_scheduled_stream.hpp similarity index 75% rename from hailort/libhailort/src/multi_device_scheduled_stream.hpp rename to hailort/libhailort/src/vdevice/scheduler/multi_device_scheduled_stream.hpp index 89f71a3..ac8fe41 100644 --- a/hailort/libhailort/src/multi_device_scheduled_stream.hpp +++ b/hailort/libhailort/src/vdevice/scheduler/multi_device_scheduled_stream.hpp @@ -11,13 +11,15 @@ #ifndef HAILO_MULTI_DEVICE_SCHEDULED_STREAM_HPP_ #define HAILO_MULTI_DEVICE_SCHEDULED_STREAM_HPP_ -#include "stream_internal.hpp" #include "hailo/hailort.h" -#include "vdevice_internal.hpp" -#include "vdma_device.hpp" -#include "scheduled_stream.hpp" #include "hailo/expected.hpp" +#include "stream_common/stream_internal.hpp" +#include "vdevice/vdevice_internal.hpp" +#include "vdevice/scheduler/scheduled_stream.hpp" +#include "vdma/vdma_device.hpp" + + namespace hailort { @@ -39,7 +41,7 @@ public: return ptr; } - hailo_status enqueue(const MemoryView &buff, const std::chrono::milliseconds &timeout) + hailo_status push(const MemoryView &buff, const std::chrono::milliseconds &timeout) { auto status = HAILO_SUCCESS; { @@ -55,7 +57,7 @@ public: }); CHECK(wait_res, HAILO_TIMEOUT, "Failed to enqueue frame with status={}, timeout={}ms", HAILO_TIMEOUT, timeout.count()); if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("'enqueue' was aborted by user"); + LOGGER__INFO("'push' was aborted by user"); return status; } @@ -68,14 +70,12 @@ public: return HAILO_SUCCESS; } - Expected dequeue(const std::chrono::milliseconds &timeout) + Expected front(const std::chrono::milliseconds &timeout) { auto status = HAILO_SUCCESS; - size_t last_tail = 0; { std::unique_lock lock(m_mutex); - // TODO: this validation is done in scheduler logic. can be removed? auto wait_res = m_cv.wait_for(lock, timeout, [this, &status] { if (m_should_stop) { status = HAILO_STREAM_ABORTED_BY_USER; @@ -85,19 +85,25 @@ public: }); CHECK_AS_EXPECTED(wait_res, HAILO_TIMEOUT, "Failed to dequeue frame with status={}, timeout={}ms", HAILO_TIMEOUT, timeout.count()); if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("'dequeue' was aborted by user"); + LOGGER__INFO("'front' was aborted by user"); return make_unexpected(status); } + } + m_cv.notify_all(); - last_tail = m_tail; + return MemoryView(m_queue[m_tail]); + } + + void pop() + { + { + std::unique_lock lock(m_mutex); m_tail = static_cast((m_tail + 1) % m_queue.size()); if (m_tail == m_head) { m_is_empty = true; } } m_cv.notify_all(); - - return MemoryView(m_queue[last_tail]); } size_t size() @@ -113,19 +119,24 @@ public: void abort() { - std::unique_lock lock(m_mutex); - m_should_stop = true; + { + std::unique_lock lock(m_mutex); + m_should_stop = true; + } m_cv.notify_all(); } void clear_abort() { - std::unique_lock lock(m_mutex); - m_should_stop = false; + { + std::unique_lock lock(m_mutex); + m_should_stop = false; + } m_cv.notify_all(); } - BuffersQueue(std::vector &&queue) : m_queue(std::move(queue)), m_head(0), m_tail(0), m_is_empty(true), m_should_stop(false) + BuffersQueue(std::vector &&queue) : m_queue(std::move(queue)), m_head(0), m_tail(0), + m_is_empty(true), m_should_stop(false) {} private: @@ -142,24 +153,18 @@ private: class MultiDeviceScheduledInputStream : public ScheduledInputStream { public: - MultiDeviceScheduledInputStream(MultiDeviceScheduledInputStream &&other) : - ScheduledInputStream(std::move(other)), - m_queue(std::move(other.m_queue)) - {} - - explicit MultiDeviceScheduledInputStream( + MultiDeviceScheduledInputStream( std::vector> &&streams, - const scheduler_ng_handle_t &network_group_handle, - EventPtr &&network_group_activated_event, + const scheduler_core_op_handle_t &core_op_handle, + EventPtr &&core_op_activated_event, const LayerInfo &layer_info, - NetworkGroupSchedulerWeakPtr network_group_scheduler, + CoreOpsSchedulerWeakPtr core_ops_scheduler, std::unique_ptr &&frames_queue, hailo_status &status) : - ScheduledInputStream(std::move(streams), network_group_handle, - std::move(network_group_activated_event), layer_info, network_group_scheduler, status), + ScheduledInputStream(std::move(streams), core_op_handle, + std::move(core_op_activated_event), layer_info, core_ops_scheduler, status), m_queue(std::move(frames_queue)) - { - } + {} virtual hailo_status send_pending_buffer(size_t device_index = 0) override; virtual Expected get_pending_frames_count() const override; @@ -171,8 +176,6 @@ protected: virtual hailo_status clear_abort() override; private: - hailo_status enqueue(const MemoryView &buffer); - Expected dequeue(); size_t get_queue_size() const; std::unique_ptr m_queue; diff --git a/hailort/libhailort/src/vdevice/scheduler/network_group_scheduler.cpp b/hailort/libhailort/src/vdevice/scheduler/network_group_scheduler.cpp new file mode 100644 index 0000000..cc7ace8 --- /dev/null +++ b/hailort/libhailort/src/vdevice/scheduler/network_group_scheduler.cpp @@ -0,0 +1,1006 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * TODO: Rename in a different PR + * @file network_group_scheduler.cpp + * @brief: Network scheduler + **/ + +#include "common/os_utils.hpp" + + +#include "vdevice/scheduler/network_group_scheduler.hpp" +#include "vdevice/vdevice_core_op.hpp" +#include "vdevice/scheduler/scheduler_oracle.hpp" +#include "vdevice/vdevice_stream_multiplexer_wrapper.hpp" +#include "hef/hef_internal.hpp" +#include "utils/profiler/tracer_macros.hpp" + +#include + + +namespace hailort +{ + +#define SINGLE_CONTEXT_BATCH_SIZE (1) +#define DEFAULT_BURST_SIZE (1) + +// TODO: use device handles instead device count +CoreOpsScheduler::CoreOpsScheduler(hailo_scheduling_algorithm_t algorithm, uint32_t device_count, std::vector &devices_bdf_id, + std::vector &devices_arch) : + SchedulerBase(algorithm, device_count, devices_bdf_id, devices_arch), + m_changing_current_batch_size(), + m_should_core_op_stop(), + m_before_read_write_mutex(), + m_core_ops_cvs(), + m_should_monitor(false) +#if defined(__GNUC__) + , m_mon_tmp_output() +#endif +{ + // TODO: HRT-7391 - Change scheduler monitor to work only when MON command is active + m_should_monitor = SchedulerMon::should_monitor(); + if (m_should_monitor) { + auto status = start_mon(); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to initiate hailo monitor of networks, with status {}", status); + } + } +} + +CoreOpsScheduler::~CoreOpsScheduler() +{ + for (auto device_info : m_devices) { + if (INVALID_CORE_OP_HANDLE != device_info->current_core_op_handle) { + auto current_core_op = m_scheduled_core_ops[device_info->current_core_op_handle]->get_core_op(); + auto current_core_op_bundle = std::dynamic_pointer_cast(current_core_op); + assert(nullptr != current_core_op_bundle); + auto vdma_core_op = current_core_op_bundle->get_core_op_by_device_index(device_info->device_id); + if (!vdma_core_op) { + LOGGER__ERROR("Error retrieving core-op in scheduler destructor"); + } else { + static const auto RESUME_PENDING_STREAM_TRANSFERS = true; + if (HAILO_SUCCESS != VdmaConfigManager::switch_core_op(vdma_core_op.value(), nullptr, 0, + RESUME_PENDING_STREAM_TRANSFERS)) { + LOGGER__ERROR("Error deactivating core-op when destroying scheduler"); + } + } + } + } + + if (m_should_monitor) { + m_should_monitor = false; + m_mon_shutdown_event->signal(); + if (m_mon_thread.joinable()) { + m_mon_thread.join(); + } + } +} + +Expected CoreOpsScheduler::create_round_robin(uint32_t device_count, std::vector &devices_bdf_id, std::vector &devices_arch) +{ + auto ptr = make_shared_nothrow(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN, device_count, devices_bdf_id, devices_arch); + CHECK_AS_EXPECTED(nullptr != ptr, HAILO_OUT_OF_HOST_MEMORY); + + return ptr; +} + +std::string get_curr_pid_as_str() +{ + return std::to_string(OsUtils::get_curr_pid()); +} + +hailo_status CoreOpsScheduler::start_mon() +{ +#if defined(__GNUC__) + m_last_measured_timestamp = std::chrono::steady_clock::now(); + m_mon_shutdown_event = Event::create_shared(Event::State::not_signalled); + CHECK(nullptr != m_mon_shutdown_event, HAILO_OUT_OF_HOST_MEMORY); + auto device_count = get_device_count(); + for (uint32_t i = 0; i < device_count; i++) { + m_last_measured_utilization_timestamp[i] = {}; + m_device_has_drained_everything[i] = true; + m_device_utilization[i] = 0; + } + + auto tmp_file = open_temp_mon_file(); + CHECK_EXPECTED_AS_STATUS(tmp_file); + m_mon_tmp_output = tmp_file.release(); + + m_mon_thread = std::thread([this] () + { + while (m_should_monitor) { + auto status = m_mon_shutdown_event->wait(DEFAULT_SCHEDULER_MON_INTERVAL); + if (HAILO_TIMEOUT == status) { + dump_state(); + } else if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Scheduler monitor failed with status {}", status); + return; + } + } + return; + }); + + return HAILO_SUCCESS; +#else + return HAILO_NOT_IMPLEMENTED; +#endif +} + +#if defined(__GNUC__) +Expected> CoreOpsScheduler::open_temp_mon_file() +{ + std::string file_name = get_curr_pid_as_str(); + auto tmp_file = TempFile::create(file_name, SCHEDULER_MON_TMP_DIR); + CHECK_EXPECTED(tmp_file); + + auto tmp_file_ptr = make_shared_nothrow(tmp_file.release()); + CHECK_AS_EXPECTED(nullptr != tmp_file_ptr, HAILO_OUT_OF_HOST_MEMORY); + + return tmp_file_ptr; +} + +void CoreOpsScheduler::dump_state() +{ + auto file = LockedFile::create(m_mon_tmp_output->name(), "w"); + if (HAILO_SUCCESS != file.status()) { + LOGGER__ERROR("Failed to open and lock file {}, with status: {}", m_mon_tmp_output->name(), file.status()); + return; + } + + ProtoMon mon; + mon.set_pid(get_curr_pid_as_str()); + time_dependent_events_cycle_calc(); + log_monitor_networks_infos(mon); + log_monitor_device_infos(mon); + log_monitor_frames_infos(mon); + + // Clear accumulators + for (auto &handle_core_op_utilization_pair : m_core_op_utilization) { + handle_core_op_utilization_pair.second = 0; + } + for (auto &handle_fps_pair : m_fps_accumulator) { + handle_fps_pair.second = 0; + } + for (auto &handle_device_utilization_pair: m_device_utilization) { + handle_device_utilization_pair.second = 0; + } + + if (!mon.SerializeToFileDescriptor(file->get_fd())) { + LOGGER__ERROR("Failed to SerializeToFileDescriptor(), with errno: {}", errno); + } +} +#endif + +std::string CoreOpsScheduler::get_core_op_name(const scheduler_core_op_handle_t &core_op_handle) +{ + assert(m_scheduled_core_ops.size() > core_op_handle); + return m_scheduled_core_ops[core_op_handle]->get_core_op_name(); +} + +// TODO: HRT-9804 - Change monitor to use the tracer design mechanism (curently this functions uses private members) +void CoreOpsScheduler::time_dependent_events_cycle_calc() +{ + auto curr_time = std::chrono::steady_clock::now(); + m_last_measured_time_duration = std::chrono::duration_cast>(curr_time - m_last_measured_timestamp).count(); + + for (auto device_info : m_devices) { + if (!m_device_has_drained_everything[device_info->device_id]) { + update_utilization_read_buffers_finished(device_info->device_id, device_info->current_core_op_handle, false); + } + } + + m_last_measured_timestamp = curr_time; +} + +void CoreOpsScheduler::log_monitor_device_infos(ProtoMon &mon) +{ + for (auto device_info : m_devices) { + assert(contains(m_device_utilization, device_info->device_id)); + auto curr_device_utilization = m_device_utilization[device_info->device_id]; + auto utilization_precentage = ((curr_device_utilization * 100) / m_last_measured_time_duration); + + auto device_infos = mon.add_device_infos(); + device_infos->set_device_id(device_info->device_bdf_id); + device_infos->set_utilization(utilization_precentage); + device_infos->set_device_arch(device_info->device_arch); + } +} + +void CoreOpsScheduler::log_monitor_networks_infos(ProtoMon &mon) +{ + for (uint32_t core_op_handle = 0; core_op_handle < m_core_op_utilization.size(); core_op_handle++) { + assert(contains(m_core_op_utilization, core_op_handle)); + auto curr_core_op_utilization = m_core_op_utilization[core_op_handle]; + auto utilization = ((curr_core_op_utilization * 100) / m_last_measured_time_duration); + auto outputs_count = static_cast(m_scheduled_core_ops[core_op_handle]->get_outputs_names().size()); + auto fps = static_cast((m_fps_accumulator[core_op_handle] / outputs_count) / m_last_measured_time_duration); + + auto net_info = mon.add_networks_infos(); + net_info->set_network_name(get_core_op_name(core_op_handle)); + net_info->set_utilization(utilization); + net_info->set_fps(fps); + } +} + +void CoreOpsScheduler::log_monitor_frames_infos(ProtoMon &mon) +{ + for (uint32_t core_op_handle = 0; core_op_handle < m_scheduled_core_ops.size(); core_op_handle++) { + auto net_frames_info = mon.add_net_frames_infos(); + net_frames_info->set_network_name(get_core_op_name(core_op_handle)); + + for (auto &stream_name : m_scheduled_core_ops[core_op_handle]->get_inputs_names()) { + auto stream_frames_info = net_frames_info->add_streams_frames_infos(); + stream_frames_info->set_stream_name(stream_name); + stream_frames_info->set_stream_direction(PROTO__STREAM_DIRECTION__HOST_TO_DEVICE); + auto status = set_h2d_frames_counters(core_op_handle, stream_name, *stream_frames_info); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to set stream's {} frames count, status = {}", stream_name, status); + continue; + } + } + + for (auto &stream_name : m_scheduled_core_ops[core_op_handle]->get_outputs_names()) { + auto stream_frames_info = net_frames_info->add_streams_frames_infos(); + stream_frames_info->set_stream_name(stream_name); + stream_frames_info->set_stream_direction(PROTO__STREAM_DIRECTION__DEVICE_TO_HOST); + auto status = set_d2h_frames_counters(core_op_handle, stream_name, *stream_frames_info); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to set stream's {} frames count, status = {}", stream_name, status); + continue; + } + } + } +} + +hailo_status CoreOpsScheduler::set_h2d_frames_counters(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + ProtoMonStreamFramesInfo &stream_frames_info) +{ + assert(m_scheduled_core_ops.size() > core_op_handle); + auto current_cng = m_scheduled_core_ops[core_op_handle]->get_core_op(); + + auto input_stream = current_cng->get_input_stream_by_name(stream_name); + CHECK_EXPECTED_AS_STATUS(input_stream); + + InputStreamBase &vdevice_input = static_cast(input_stream->get()); + auto buffer_frames_size = vdevice_input.get_buffer_frames_size(); + if (HAILO_SUCCESS == buffer_frames_size.status()) { + stream_frames_info.set_buffer_frames_size(static_cast(buffer_frames_size.value())); + } else { + stream_frames_info.set_buffer_frames_size(SCHEDULER_MON_NAN_VAL); + } + + auto pending_frames_count = vdevice_input.get_pending_frames_count(); + if (HAILO_SUCCESS == pending_frames_count.status()) { + stream_frames_info.set_pending_frames_count(static_cast(pending_frames_count.value())); + } else { + stream_frames_info.set_pending_frames_count(SCHEDULER_MON_NAN_VAL); + } + + return HAILO_SUCCESS; +} + +hailo_status CoreOpsScheduler::set_d2h_frames_counters(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + ProtoMonStreamFramesInfo &stream_frames_info) +{ + assert(m_scheduled_core_ops.size() > core_op_handle); + auto current_cng = m_scheduled_core_ops[core_op_handle]->get_core_op(); + + auto output_stream = current_cng->get_output_stream_by_name(stream_name); + CHECK_EXPECTED_AS_STATUS(output_stream); + + OutputStreamBase &vdevice_output = static_cast(output_stream->get()); + auto buffer_frames_size = vdevice_output.get_buffer_frames_size(); + if (HAILO_SUCCESS == buffer_frames_size.status()) { + stream_frames_info.set_buffer_frames_size(static_cast(buffer_frames_size.value())); + } else { + stream_frames_info.set_buffer_frames_size(SCHEDULER_MON_NAN_VAL); + } + + auto pending_frames_count = vdevice_output.get_pending_frames_count(); + if (HAILO_SUCCESS == pending_frames_count.status()) { + stream_frames_info.set_pending_frames_count(static_cast(pending_frames_count.value())); + } else { + stream_frames_info.set_pending_frames_count(SCHEDULER_MON_NAN_VAL); + } + + return HAILO_SUCCESS; +} + +Expected CoreOpsScheduler::add_core_op(std::shared_ptr added_cng) +{ + scheduler_core_op_handle_t core_op_handle = INVALID_CORE_OP_HANDLE; + { + std::unique_lock lock(m_before_read_write_mutex); + + core_op_handle = static_cast(m_scheduled_core_ops.size()); + TRACE(AddCoreOpTrace, "", added_cng->name(), DEFAULT_SCHEDULER_TIMEOUT.count(), DEFAULT_SCHEDULER_MIN_THRESHOLD, core_op_handle); + + auto stream_infos = added_cng->get_all_stream_infos(); + CHECK_EXPECTED(stream_infos); + + auto scheduled_core_op = ScheduledCoreOp::create(added_cng, stream_infos.value()); + CHECK_EXPECTED(scheduled_core_op); + + m_scheduled_core_ops.emplace_back(scheduled_core_op.release()); + + m_changing_current_batch_size[core_op_handle] = false; + + for (const auto &stream_info : stream_infos.value()) { + m_should_core_op_stop[core_op_handle][stream_info.name] = false; + } + + for (auto& device_info : m_devices) { + for (const auto &stream_info : stream_infos.value()) { + if (HAILO_H2D_STREAM == stream_info.direction) { + device_info->current_cycle_requested_transferred_frames_h2d[core_op_handle][stream_info.name] = 0; + } else { + device_info->current_cycle_finished_transferred_frames_d2h[core_op_handle][stream_info.name] = 0; + device_info->current_cycle_finished_read_frames_d2h[core_op_handle][stream_info.name] = 0; + } + } + } + + // Monitor members + m_core_op_utilization[core_op_handle] = 0; + m_fps_accumulator[core_op_handle] = 0; + + auto network_cvs = ScheduledCoreOpCV::create(added_cng); + CHECK_EXPECTED(network_cvs); + m_core_ops_cvs[core_op_handle] = network_cvs.release(); + m_core_op_priority[HAILO_SCHEDULER_PRIORITY_NORMAL].emplace_back(core_op_handle); + } + + return core_op_handle; +} + +bool CoreOpsScheduler::is_core_op_active(const scheduler_core_op_handle_t &core_op_handle) +{ + for (auto device_info : m_devices) { + if (core_op_handle == device_info->current_core_op_handle) { + return true; + } + } + + return false; +} + +bool CoreOpsScheduler::is_multi_device() +{ + return m_devices.size() > 1; +} + +hailo_status CoreOpsScheduler::wait_for_write(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + const std::chrono::milliseconds &timeout, const std::function &should_cancel) +{ + { + std::unique_lock lock(m_before_read_write_mutex); + + hailo_status status = HAILO_SUCCESS; + auto wait_res = m_core_ops_cvs[core_op_handle]->wait_for(stream_name, lock, timeout, [this, core_op_handle, stream_name, &should_cancel, &status] { + + if (should_cancel()) { + status = HAILO_STREAM_ABORTED_BY_USER; + return true; // return true so that the wait will finish + } + + if (should_core_op_stop(core_op_handle)) { + status = HAILO_STREAM_ABORTED_BY_USER; + return true; // return true so that the wait will finish + } + + return m_scheduled_core_ops[core_op_handle]->can_stream_write(stream_name); + }); + CHECK(wait_res, HAILO_TIMEOUT, "{} (H2D) failed with status={}, timeout={}ms", stream_name, HAILO_TIMEOUT, timeout.count()); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + return status; + } + CHECK_SUCCESS(status); + + m_scheduled_core_ops[core_op_handle]->mark_frame_sent(); + m_scheduled_core_ops[core_op_handle]->requested_write_frames().increase(stream_name); + } + + return HAILO_SUCCESS; +} + +hailo_status CoreOpsScheduler::signal_write_finish(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + bool did_write_fail) +{ + { + std::unique_lock lock(m_before_read_write_mutex); + assert(m_scheduled_core_ops.size() > core_op_handle); + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + + if (did_write_fail) { + scheduled_core_op->requested_write_frames().decrease(stream_name); + return HAILO_SUCCESS; + } + + if (should_core_op_stop(core_op_handle)) { + return HAILO_STREAM_ABORTED_BY_USER; + } + + scheduled_core_op->finished_write_frames().increase(stream_name); + scheduled_core_op->requested_write_frames().decrease(stream_name); + + auto device_id = CoreOpsSchedulerOracle::get_avail_device(*this, core_op_handle); + if (INVALID_DEVICE_ID != device_id) { + auto status = switch_core_op(core_op_handle, device_id); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("switch_core_op has failed with status=HAILO_STREAM_ABORTED_BY_USER"); + return status; + } + CHECK_SUCCESS(status); + } + + auto status = optimize_streaming_if_enabled(core_op_handle); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + return status; + } + CHECK_SUCCESS(status); + } + + return HAILO_SUCCESS; +} + +hailo_status CoreOpsScheduler::switch_core_op(const scheduler_core_op_handle_t &core_op_handle, uint32_t device_id, bool /*keep_nn_config*/) +{ + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + auto curr_device_info = m_devices[device_id]; + + // initialize current cycle maps + for (const auto &name : scheduled_core_op->get_inputs_names()) { + curr_device_info->current_cycle_requested_transferred_frames_h2d[core_op_handle][name] = 0; + } + + for (const auto &name : scheduled_core_op->get_outputs_names()) { + curr_device_info->current_cycle_finished_transferred_frames_d2h[core_op_handle][name] = 0; + curr_device_info->current_cycle_finished_read_frames_d2h[core_op_handle][name] = 0; + } + + uint16_t batch_size = SINGLE_CONTEXT_BATCH_SIZE; + uint16_t burst_size = static_cast(scheduled_core_op->finished_write_frames_min_value()); + // In multi device finished write frame could be bigger then the vdma buffers we have, can be removed after dynamic desc binding. + if (is_multi_device()) { + burst_size = std::min(burst_size, get_min_avail_buffers_count(core_op_handle, device_id)); + // We limit the max burst size to (dev_count * max_batch) to keep former behavior (this was the buffer_pool size) + // TODO: remove this limitation and work with user-controlled max_burst_size + burst_size = std::min(burst_size, static_cast(scheduled_core_op->get_max_batch_size() * get_device_count())); + } + + if (scheduled_core_op->use_dynamic_batch_flow()) { + batch_size = std::min(static_cast(scheduled_core_op->finished_write_frames_min_value()), scheduled_core_op->get_max_batch_size()); + burst_size = batch_size; + } + + bool has_same_batch_size_as_previous = (curr_device_info->current_batch_size == batch_size); + curr_device_info->current_batch_size = batch_size; + + if (curr_device_info->current_core_op_handle != core_op_handle) { + curr_device_info->is_switching_core_op = false; + } + + if ((core_op_handle != curr_device_info->current_core_op_handle) || (!has_same_batch_size_as_previous)) { + assert(m_scheduled_core_ops.size() > core_op_handle); + auto next_active_cng = scheduled_core_op->get_core_op(); + auto next_active_cng_wrapper = std::dynamic_pointer_cast(next_active_cng); + assert(nullptr != next_active_cng_wrapper); + auto next_active_cng_expected = next_active_cng_wrapper->get_core_op_by_device_index(curr_device_info->device_id); + CHECK_EXPECTED_AS_STATUS(next_active_cng_expected); + + std::shared_ptr current_active_vdma_cng = nullptr; + if (curr_device_info->current_core_op_handle != INVALID_CORE_OP_HANDLE) { + auto current_active_cng = m_scheduled_core_ops[curr_device_info->current_core_op_handle]->get_core_op(); + auto current_active_cng_bundle = std::dynamic_pointer_cast(current_active_cng); + assert(nullptr != current_active_cng_bundle); + auto current_active_cng_expected = current_active_cng_bundle->get_core_op_by_device_index(curr_device_info->device_id); + CHECK_EXPECTED_AS_STATUS(current_active_cng_expected); + current_active_vdma_cng = current_active_cng_expected.release(); + } + + TRACE(SwitchCoreOpTrace, "", core_op_handle); + static const auto RESUME_PENDING_STREAM_TRANSFERS = true; + auto status = VdmaConfigManager::switch_core_op(current_active_vdma_cng, next_active_cng_expected.value(), batch_size, + + RESUME_PENDING_STREAM_TRANSFERS); + CHECK_SUCCESS(status, "Failed switching core-op"); + // Clear the ready_to_switch flag from old activation + scheduled_core_op->mark_unready_to_switch(); + + // Register to get interrupts - has to be after core-op is activated + for (auto &output_stream : next_active_cng_expected.value()->get_output_streams()) { + OutputStreamBase &vdevice_output = static_cast(output_stream.get()); + status = vdevice_output.register_interrupt_callback( + [this, name = output_stream.get().name(), format = vdevice_output.get_layer_info().format.order, scheduled_core_op, core_op_handle, device_id] + (uint32_t frames) { + auto should_notify_next = false; + { + std::unique_lock lock(m_before_read_write_mutex); + // In order to meet performance requirement we enable switch only after first frame is arrived. + // TODO: remove this hack / move it to oracle and add another scheduling algorithm for it + scheduled_core_op->mark_ready_to_switch(); + if (hailo_format_order_t::HAILO_FORMAT_ORDER_HAILO_NMS != format) { + TRACE(OutputVdmaEnqueueTrace, "", core_op_handle, name, frames); + // TODO: Remove d2h_finished_transferred_frames and use current_cycle_finished_transferred_frames_d2h instead + scheduled_core_op->d2h_finished_transferred_frames(name) += frames; + m_devices[device_id]->current_cycle_finished_transferred_frames_d2h[core_op_handle][name] += frames; + } + + auto has_drained_everything = has_core_op_drained_everything(core_op_handle, device_id); + + if (m_should_monitor) { + update_utilization_read_buffers_finished(device_id, core_op_handle, has_drained_everything); + } + + // If ng finished and we didnt choose next lets choose without checking threshold + if (!m_devices[device_id]->is_switching_core_op && has_drained_everything) { + auto was_chosen = choose_next_core_op(device_id, true); + if (!was_chosen) { + choose_next_core_op(device_id, false); + } + } + + if (m_devices[device_id]->is_switching_core_op && has_drained_everything) { + should_notify_next = true; + } + } + // Notify stream that new frame was accepted (wait_for_read) + m_core_ops_cvs[core_op_handle]->notify_one(name); + if (should_notify_next) { + auto next_core_op = m_devices[device_id]->next_core_op_handle; + // Notify all the threads of the next ng (wait_for_read) + m_core_ops_cvs[next_core_op]->notify_all(); + } + }); + CHECK_SUCCESS(status); + } + } + + scheduled_core_op->set_last_run_timestamp(std::chrono::steady_clock::now()); // Mark timestamp on activation + curr_device_info->current_core_op_handle = core_op_handle; + + // Finished switching batch size + m_changing_current_batch_size[core_op_handle] = false; + + auto status = send_all_pending_buffers(core_op_handle, device_id, burst_size); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("send_all_pending_buffers has failed with status=HAILO_STREAM_ABORTED_BY_USER"); + return status; + } + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +hailo_status CoreOpsScheduler::send_all_pending_buffers(const scheduler_core_op_handle_t &core_op_handle, uint32_t device_id, uint32_t burst_size) +{ + auto current_device_info = m_devices[device_id]; + if ((INVALID_CORE_OP_HANDLE == current_device_info->current_core_op_handle) || (current_device_info->current_core_op_handle != core_op_handle)) { + return HAILO_SUCCESS; + } + + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + + for (size_t i = 0; i < burst_size; i++) { + auto finished_send = false; + for (const auto &name : scheduled_core_op->get_inputs_names()) { + if (scheduled_core_op->finished_write_frames(name) == 0) { + finished_send = true; + break; + } + } + if (finished_send) { + break; + } + for (const auto &name : scheduled_core_op->get_inputs_names()) { + auto status = send_pending_buffer(core_op_handle, name, device_id); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("send_pending_buffer has failed with status=HAILO_STREAM_ABORTED_BY_USER"); + return status; + } + CHECK_SUCCESS(status); + } + scheduled_core_op->push_device_index(device_id); + scheduled_core_op->set_last_device_index(device_id); + + if (m_should_monitor) { + update_utilization_send_started(device_id); + } + } + + return HAILO_SUCCESS; +} + +hailo_status CoreOpsScheduler::send_pending_buffer(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + uint32_t device_id) +{ + assert(m_scheduled_core_ops.size() > core_op_handle); + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + + auto current_cng = scheduled_core_op->get_core_op(); + auto input_stream = current_cng->get_input_stream_by_name(stream_name); + CHECK_EXPECTED_AS_STATUS(input_stream); + + VDeviceInputStreamMultiplexerWrapper &vdevice_input = static_cast(input_stream->get()); + TRACE(InputVdmaDequeueTrace, "", core_op_handle, stream_name); + auto status = vdevice_input.send_pending_buffer(device_id); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("send_pending_buffer has failed with status=HAILO_STREAM_ABORTED_BY_USER"); + return status; + } + CHECK_SUCCESS(status); + + m_devices[device_id]->current_cycle_requested_transferred_frames_h2d[core_op_handle][stream_name]++; + scheduled_core_op->finished_write_frames().decrease(stream_name); + + scheduled_core_op->h2d_finished_transferred_frames().increase(stream_name); + + if (should_core_op_stop(core_op_handle)) { + return HAILO_STREAM_ABORTED_BY_USER; + } + + return HAILO_SUCCESS; +} + +CoreOpsScheduler::ReadyInfo CoreOpsScheduler::is_core_op_ready(const scheduler_core_op_handle_t &core_op_handle, bool check_threshold) +{ + ReadyInfo result; + result.is_ready = false; + + if (should_core_op_stop(core_op_handle)) { + // Do not switch to an aborted core-op + return result; + } + + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + // Check if there arent any write requests + bool has_pending_writes = scheduled_core_op->finished_write_frames_min_value() > 0; + + // Check if there arent any read requests + bool has_pending_user_reads = false; + for (const auto &name : scheduled_core_op->get_outputs_names()) { + if (scheduled_core_op->requested_read_frames(name) > 0) { + has_pending_user_reads = true; + break; + } + } + + std::vector over_threshold; + over_threshold.reserve(scheduled_core_op->get_inputs_names().size()); + std::vector over_timeout; + over_timeout.reserve(scheduled_core_op->get_inputs_names().size()); + + if (check_threshold) { + for (const auto &name : scheduled_core_op->get_inputs_names()) { + auto threshold_exp = scheduled_core_op->get_threshold(name); + if (!threshold_exp) { + LOGGER__ERROR("Failed to get threshold for stream {}", name); + return result; + } + auto threshold = (DEFAULT_SCHEDULER_MIN_THRESHOLD == threshold_exp.value()) ? 1 : threshold_exp.value(); + auto timeout_exp = scheduled_core_op->get_timeout(); + if (!timeout_exp) { + LOGGER__ERROR("Failed to get timeout for stream {}", name); + return result; + } + auto timeout = timeout_exp.release(); + + // Check if there arent enough write requests to reach threshold and timeout didnt passed + auto write_requests = scheduled_core_op->requested_write_frames(name) + scheduled_core_op->finished_write_frames(name); + auto stream_over_threshold = write_requests >= threshold; + auto stream_over_timeout = timeout <= (std::chrono::steady_clock::now() - scheduled_core_op->get_last_run_timestamp()); + over_threshold.push_back(stream_over_threshold); + over_timeout.push_back(stream_over_timeout); + if (stream_over_threshold || stream_over_timeout) { + continue; + } else { + result.is_ready = false; + return result; + } + } + } + + result.threshold = std::all_of(over_threshold.begin(), over_threshold.end(), [](auto over) { return over; }); + result.timeout = std::all_of(over_timeout.begin(), over_timeout.end(), [](auto over) { return over; }); + result.is_ready = has_pending_writes && has_pending_user_reads; + + return result; +} + +Expected CoreOpsScheduler::wait_for_read(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + const std::chrono::milliseconds &timeout) +{ + std::unique_lock lock(m_before_read_write_mutex); + + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + scheduled_core_op->requested_read_frames().increase(stream_name); + + hailo_status status = HAILO_SUCCESS; + auto wait_res = m_core_ops_cvs[core_op_handle]->wait_for(stream_name, lock, timeout, [this, core_op_handle, scheduled_core_op, stream_name, &status] { + + if (should_core_op_stop(core_op_handle)) { + status = HAILO_STREAM_ABORTED_BY_USER; + return true; // return true so that the wait will finish + } + + auto device_id = CoreOpsSchedulerOracle::get_avail_device(*this, core_op_handle); + if (INVALID_DEVICE_ID != device_id) { + status = switch_core_op(core_op_handle, device_id); + if (HAILO_SUCCESS != status) { + return true; // return true so that the wait will finish + } + } + + return scheduled_core_op->can_stream_read(stream_name); + }); + CHECK_AS_EXPECTED(wait_res, HAILO_TIMEOUT, "{} (D2H) failed with status={}, timeout={}ms", stream_name, HAILO_TIMEOUT, timeout.count()); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + return make_unexpected(status); + } + CHECK_SUCCESS_AS_EXPECTED(status); + + scheduled_core_op->requested_read_frames().decrease(stream_name); + + return scheduled_core_op->pop_device_index(stream_name); +} + +hailo_status CoreOpsScheduler::signal_read_finish(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, uint32_t device_id) +{ + auto should_notify_next = false; + { + std::unique_lock lock(m_before_read_write_mutex); + + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + + scheduled_core_op->finished_read_frames().increase(stream_name); + m_devices[device_id]->current_cycle_finished_read_frames_d2h[core_op_handle][stream_name]++; + scheduled_core_op->d2h_finished_transferred_frames().decrease(stream_name); + m_fps_accumulator[core_op_handle]++; + + decrease_core_op_counters(core_op_handle); + + auto has_drained_everything = has_core_op_drained_everything(core_op_handle, device_id); + if (scheduled_core_op->is_nms() && has_drained_everything) { + // In NMS networks there is possibility that next wasn't choosen yet + choose_next_core_op(device_id, true); + + // If we didnt choose with treshold or timeout lets choose without treshold + if (!m_devices[device_id]->is_switching_core_op) { + choose_next_core_op(device_id, false); + } + + if (m_devices[device_id]->is_switching_core_op) { + should_notify_next = true; + } + + if (m_should_monitor) { + update_utilization_read_buffers_finished(device_id, core_op_handle, has_drained_everything); + } + } + } + + // Notify stream that frame was read and we have a space in the read buffers (wait_for_write) + m_core_ops_cvs[core_op_handle]->notify_all(); + + if (should_notify_next) { + // Notify all the threads of the next ng, for nms networks this is the only place we know the network was finished (wait_for_read) + m_core_ops_cvs[m_devices[device_id]->next_core_op_handle]->notify_all(); + } + + return HAILO_SUCCESS; +} + +void CoreOpsScheduler::decrease_core_op_counters(const scheduler_core_op_handle_t &core_op_handle) +{ + return m_scheduled_core_ops[core_op_handle]->decrease_current_core_op_counters(); +} + +bool CoreOpsScheduler::has_core_op_drained_everything(const scheduler_core_op_handle_t &core_op_handle, uint32_t device_id) +{ + if (INVALID_CORE_OP_HANDLE == core_op_handle) { + // If no core-op is running, consider it as drained + return true; + } + + if (core_op_all_streams_aborted(core_op_handle)) { + // We treat core-op as drained only if all streams are aborted - to make sure there aren't any ongoing transfers + return true; + } + + if ((!m_scheduled_core_ops[core_op_handle]->is_nms()) && (is_multi_device() || m_scheduled_core_ops[core_op_handle]->use_dynamic_batch_flow())) { + auto current_device_info = m_devices[device_id]; + auto max_transferred_h2d = get_max_value_of_unordered_map(current_device_info->current_cycle_requested_transferred_frames_h2d[core_op_handle]); + auto min_transferred_d2h = get_min_value_of_unordered_map(current_device_info->current_cycle_finished_transferred_frames_d2h[core_op_handle]); + + return (max_transferred_h2d == min_transferred_d2h); + } + + return m_scheduled_core_ops[core_op_handle]->has_core_op_drained_everything(); +} + +hailo_status CoreOpsScheduler::enable_stream(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name) +{ + { + std::unique_lock lock(m_before_read_write_mutex); + + if (!m_should_core_op_stop[core_op_handle][stream_name]) { + return HAILO_SUCCESS; + } + + m_should_core_op_stop[core_op_handle][stream_name] = false; + } + m_core_ops_cvs[core_op_handle]->notify_all(); + + return HAILO_SUCCESS; +} + +hailo_status CoreOpsScheduler::disable_stream(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name) +{ + { + std::unique_lock lock(m_before_read_write_mutex); + + if (m_should_core_op_stop[core_op_handle][stream_name]) { + return HAILO_SUCCESS; + } + + m_should_core_op_stop[core_op_handle][stream_name] = true; + } + m_core_ops_cvs[core_op_handle]->notify_all(); + + return HAILO_SUCCESS; +} + +hailo_status CoreOpsScheduler::set_timeout(const scheduler_core_op_handle_t &core_op_handle, const std::chrono::milliseconds &timeout, const std::string &/*network_name*/) +{ + // TODO: call in loop for set_timeout with the relevant stream-names (of the given network) + return m_scheduled_core_ops[core_op_handle]->set_timeout(timeout); +} + +hailo_status CoreOpsScheduler::set_threshold(const scheduler_core_op_handle_t &core_op_handle, uint32_t threshold, const std::string &/*network_name*/) +{ + // TODO: call in loop for set_timeout with the relevant stream-names (of the given network) + return m_scheduled_core_ops[core_op_handle]->set_threshold(threshold); +} + +hailo_status CoreOpsScheduler::set_priority(const scheduler_core_op_handle_t &core_op_handle, core_op_priority_t priority, const std::string &/*network_name*/) +{ + CHECK(priority <= HAILO_SCHEDULER_PRIORITY_MAX, HAILO_INVALID_ARGUMENT); + std::unique_lock lock(m_before_read_write_mutex); + auto old_priority = m_scheduled_core_ops[core_op_handle]->get_priority(); + auto &priority_vector = m_core_op_priority[old_priority]; + auto it = std::find(priority_vector.begin(), priority_vector.end(), core_op_handle); + CHECK(it != priority_vector.end(), HAILO_INTERNAL_FAILURE); + + priority_vector.erase(it); + m_scheduled_core_ops[core_op_handle]->set_priority(priority); + m_core_op_priority[priority].push_back(core_op_handle); + + return HAILO_SUCCESS; +} + +bool CoreOpsScheduler::choose_next_core_op(size_t device_id, bool check_threshold) +{ + if (!m_devices[device_id]->is_switching_core_op) { + return CoreOpsSchedulerOracle::choose_next_model(*this, m_devices[device_id]->device_id, check_threshold); + } + return false; +} + +bool CoreOpsScheduler::should_core_op_stop(const scheduler_core_op_handle_t &core_op_handle) +{ + for (const auto &name_flag_pair : m_should_core_op_stop[core_op_handle]) { + if (name_flag_pair.second) { + return true; + } + } + + return false; +} + +bool CoreOpsScheduler::core_op_all_streams_aborted(const scheduler_core_op_handle_t &core_op_handle) +{ + for (const auto &name_flag_pair : m_should_core_op_stop[core_op_handle]) { + if (!name_flag_pair.second) { + return false; + } + } + return true; +} + +void CoreOpsScheduler::notify_all() +{ + { + // Acquire mutex to make sure the notify_all will wake the blocking threads on the cv + std::unique_lock lock(m_before_read_write_mutex); + } + // TODO: consider notify only the relevant ng or stream + for (auto &cng_cvs : m_core_ops_cvs) { + cng_cvs.second->notify_all(); + } +} + +hailo_status CoreOpsScheduler::optimize_streaming_if_enabled(const scheduler_core_op_handle_t &core_op_handle) +{ + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + + if ((!scheduled_core_op->use_dynamic_batch_flow()) && !(scheduled_core_op->is_ready_to_switch() && + CoreOpsSchedulerOracle::should_stop_streaming(*this, scheduled_core_op->get_priority()))) { + for (uint32_t i = 0; i < m_devices.size(); i++) { + uint32_t index = scheduled_core_op->get_last_device_index() + i + 1; + index %= static_cast(m_devices.size()); + auto device_info = m_devices[index]; + // If multi device check for space in the vdma buffers, the send pending buffer is waitable in the current implementation. + // can be removed after dynamic descriptor binding support + if (device_info->current_core_op_handle == core_op_handle && + (!is_multi_device() || (get_min_avail_buffers_count(core_op_handle, device_info->device_id) >= DEFAULT_BURST_SIZE))) { + auto status = send_all_pending_buffers(core_op_handle, device_info->device_id, DEFAULT_BURST_SIZE); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("send_all_pending_buffers has failed with status=HAILO_STREAM_ABORTED_BY_USER"); + return status; + } + CHECK_SUCCESS(status); + } + } + } + + return HAILO_SUCCESS; +} + +uint16_t CoreOpsScheduler::get_min_avail_buffers_count(const scheduler_core_op_handle_t &core_op_handle, uint32_t device_id) +{ + auto device_info = m_devices[device_id]; + auto scheduled_core_op = m_scheduled_core_ops[core_op_handle]; + + auto max_transferred_h2d = get_max_value_of_unordered_map(device_info->current_cycle_requested_transferred_frames_h2d[core_op_handle]); + auto min_d2h_frames = scheduled_core_op->is_nms() ? get_min_value_of_unordered_map(device_info->current_cycle_finished_read_frames_d2h[core_op_handle]) : + get_min_value_of_unordered_map(device_info->current_cycle_finished_transferred_frames_d2h[core_op_handle]); + auto ongoing_frames = static_cast(max_transferred_h2d - min_d2h_frames); + + uint16_t avail_buffers = static_cast(scheduled_core_op->get_min_input_buffers_count(get_device_count()) - ongoing_frames); + + return avail_buffers; +} + +void CoreOpsScheduler::update_utilization_timers(scheduler_device_idx_t device_id, scheduler_core_op_handle_t core_op_handle) +{ + assert(contains(m_core_op_utilization, core_op_handle)); + + auto time_diff = std::chrono::duration_cast>( + std::chrono::steady_clock::now() - m_last_measured_utilization_timestamp[device_id]).count(); + + m_device_utilization[device_id] += time_diff; + m_core_op_utilization[core_op_handle] += time_diff; +} + +void CoreOpsScheduler::update_utilization_timestamp(scheduler_device_idx_t device_id) +{ + m_last_measured_utilization_timestamp[device_id] = std::chrono::steady_clock::now(); +} + +void CoreOpsScheduler::update_utilization_send_started(scheduler_device_idx_t device_id) +{ + if (m_device_has_drained_everything[device_id]) { + update_device_drained_state(device_id, false); + update_utilization_timestamp(device_id); + } +} + +void CoreOpsScheduler::update_device_drained_state(scheduler_device_idx_t device_id, bool state) +{ + m_device_has_drained_everything[device_id] = state; +} + +void CoreOpsScheduler::update_utilization_read_buffers_finished(scheduler_device_idx_t device_id, + scheduler_core_op_handle_t core_op_handle, bool is_drained_everything) +{ + update_utilization_timers(device_id, core_op_handle); + update_device_drained_state(device_id, is_drained_everything); + if (!is_drained_everything) { + update_utilization_timestamp(device_id); + } +} + +} /* namespace hailort */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdevice/scheduler/network_group_scheduler.hpp b/hailort/libhailort/src/vdevice/scheduler/network_group_scheduler.hpp new file mode 100644 index 0000000..253e2b9 --- /dev/null +++ b/hailort/libhailort/src/vdevice/scheduler/network_group_scheduler.hpp @@ -0,0 +1,144 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file network_group_scheduler.hpp + * @brief Class declaration for CoreOpsScheduler that schedules core-ops to be active depending on the scheduling algorithm. + **/ + +#ifndef _HAILO_NETWORK_GROUP_SCHEDULER_HPP_ +#define _HAILO_NETWORK_GROUP_SCHEDULER_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "common/utils.hpp" +#include "common/filesystem.hpp" + +#include "vdevice/scheduler/scheduler_mon.hpp" +#include "vdevice/scheduler/scheduled_core_op_state.hpp" +#include "vdevice/scheduler/scheduled_core_op_cv.hpp" +#include "vdevice/scheduler/scheduler_base.hpp" + + +namespace hailort +{ + +#define INVALID_CORE_OP_HANDLE (UINT32_MAX) +#define INVALID_DEVICE_ID (UINT32_MAX) + +using scheduler_core_op_handle_t = uint32_t; +using core_op_priority_t = uint8_t; +using scheduler_device_idx_t = uint32_t; + +class CoreOpsScheduler; +using CoreOpsSchedulerPtr = std::shared_ptr; + +// We use mostly weak pointer for the scheduler to prevent circular dependency of the pointers +using CoreOpsSchedulerWeakPtr = std::weak_ptr; + +using stream_name_t = std::string; + +class CoreOpsScheduler : public SchedulerBase +{ +public: + static Expected create_round_robin(uint32_t device_count, std::vector &devices_bdf_id, + std::vector &devices_arch); + CoreOpsScheduler(hailo_scheduling_algorithm_t algorithm, uint32_t device_count, std::vector &devices_bdf_id, + std::vector &devices_arch); + + virtual ~CoreOpsScheduler(); + CoreOpsScheduler(const CoreOpsScheduler &other) = delete; + CoreOpsScheduler &operator=(const CoreOpsScheduler &other) = delete; + CoreOpsScheduler &operator=(CoreOpsScheduler &&other) = delete; + CoreOpsScheduler(CoreOpsScheduler &&other) noexcept = delete; + + Expected add_core_op(std::shared_ptr added_core_op); + + hailo_status wait_for_write(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + const std::chrono::milliseconds &timeout, const std::function &should_cancel); + hailo_status signal_write_finish(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, bool did_write_fail); + Expected wait_for_read(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + const std::chrono::milliseconds &timeout); + hailo_status signal_read_finish(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, uint32_t device_id); + + hailo_status enable_stream(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name); + hailo_status disable_stream(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name); + + hailo_status set_timeout(const scheduler_core_op_handle_t &core_op_handle, const std::chrono::milliseconds &timeout, const std::string &network_name); + hailo_status set_threshold(const scheduler_core_op_handle_t &core_op_handle, uint32_t threshold, const std::string &network_name); + hailo_status set_priority(const scheduler_core_op_handle_t &core_op_handle, core_op_priority_t priority, const std::string &network_name); + + virtual ReadyInfo is_core_op_ready(const scheduler_core_op_handle_t &core_op_handle, bool check_threshold) override; + virtual bool has_core_op_drained_everything(const scheduler_core_op_handle_t &core_op_handle, uint32_t device_id) override; + + void notify_all(); + +protected: + bool choose_next_core_op(size_t device_id, bool check_threshold); + + std::unordered_map m_changing_current_batch_size; + std::unordered_map> m_should_core_op_stop; + +private: + hailo_status switch_core_op(const scheduler_core_op_handle_t &core_op_handle, uint32_t device_id, + bool keep_nn_config = false); + void reset_current_core_op_timestamps(uint32_t device_id); + + hailo_status send_all_pending_buffers(const scheduler_core_op_handle_t &core_op_handle, uint32_t device_id, uint32_t burst_size); + hailo_status send_pending_buffer(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, uint32_t device_id); + + void decrease_core_op_counters(const scheduler_core_op_handle_t &core_op_handle); + bool should_core_op_stop(const scheduler_core_op_handle_t &core_op_handle); + bool core_op_all_streams_aborted(const scheduler_core_op_handle_t &core_op_handle); + + std::string get_core_op_name(const scheduler_core_op_handle_t &core_op_handle); + bool is_core_op_active(const scheduler_core_op_handle_t &core_op_handle); + bool is_multi_device(); + hailo_status optimize_streaming_if_enabled(const scheduler_core_op_handle_t &network_group_handle); + uint16_t get_min_avail_buffers_count(const scheduler_core_op_handle_t &network_group_handle, uint32_t device_id); + + hailo_status start_mon(); + void time_dependent_events_cycle_calc(); + void log_monitor_device_infos(ProtoMon &mon); + void log_monitor_networks_infos(ProtoMon &mon); + void log_monitor_frames_infos(ProtoMon &mon); + void update_utilization_timers(scheduler_device_idx_t device_id, scheduler_core_op_handle_t core_op_handle); + void update_utilization_timestamp(scheduler_device_idx_t device_id); + void update_utilization_send_started(scheduler_device_idx_t device_id); + void update_device_drained_state(scheduler_device_idx_t device_id, bool state); + void update_utilization_read_buffers_finished(scheduler_device_idx_t device_id, scheduler_core_op_handle_t core_op_hanle, bool is_drained_everything); + hailo_status set_h2d_frames_counters(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + ProtoMonStreamFramesInfo &stream_frames_info); + hailo_status set_d2h_frames_counters(const scheduler_core_op_handle_t &core_op_handle, const std::string &stream_name, + ProtoMonStreamFramesInfo &stream_frames_info); +#if defined(__GNUC__) + Expected> open_temp_mon_file(); + void dump_state(); +#endif + + std::vector> m_scheduled_core_ops; + std::mutex m_before_read_write_mutex; + std::unordered_map> m_core_ops_cvs; + + // Params for the scheduler MON + std::atomic_bool m_should_monitor; + std::thread m_mon_thread; + EventPtr m_mon_shutdown_event; +#if defined(__GNUC__) + std::shared_ptr m_mon_tmp_output; +#endif + std::chrono::time_point m_last_measured_timestamp; + double m_last_measured_time_duration; + std::unordered_map m_device_utilization; + std::unordered_map m_device_has_drained_everything; + std::unordered_map> m_last_measured_utilization_timestamp; + // TODO: Consider adding Accumulator classes for more info (min, max, mean, etc..) + std::unordered_map m_core_op_utilization; + std::unordered_map m_fps_accumulator; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_NETWORK_GROUP_SCHEDULER_HPP_ */ diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_cv.hpp b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_cv.hpp new file mode 100644 index 0000000..9b6f8af --- /dev/null +++ b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_cv.hpp @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file scheduled_core_op_cv.hpp + * @brief Class declaration for scheduled core-ops conditional variables + **/ + +#ifndef _HAILO_SCHEDULED_CORE_OP_CV_HPP_ +#define _HAILO_SCHEDULED_CORE_OP_CV_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "common/utils.hpp" + +#include "vdevice/scheduler/scheduler_mon.hpp" + +#include + + +namespace hailort +{ + +class ScheduledCoreOpCV +{ +public: + static Expected> create(std::shared_ptr added_cng) + { + auto stream_infos = added_cng->get_all_stream_infos(); + CHECK_EXPECTED(stream_infos); + + std::unordered_map> cv_per_stream; + for (const auto &stream_info : stream_infos.value()) { + auto cv = make_shared_nothrow(); + CHECK_NOT_NULL_AS_EXPECTED(cv, HAILO_OUT_OF_HOST_MEMORY); + cv_per_stream[stream_info.name] = std::move(cv); + } + + auto scheduled_core_op_cv = make_shared_nothrow(cv_per_stream); + CHECK_NOT_NULL_AS_EXPECTED(scheduled_core_op_cv, HAILO_OUT_OF_HOST_MEMORY); + + return scheduled_core_op_cv; + } + + virtual ~ScheduledCoreOpCV() = default; + ScheduledCoreOpCV(const ScheduledCoreOpCV &other) = delete; + ScheduledCoreOpCV &operator=(const ScheduledCoreOpCV &other) = delete; + ScheduledCoreOpCV &operator=(ScheduledCoreOpCV &&other) = delete; + ScheduledCoreOpCV(ScheduledCoreOpCV &&other) noexcept = delete; + + void notify_one(const stream_name_t &name) + { + assert(contains(m_map, name)); + m_map[name]->notify_one(); + } + + void notify_all() + { + for (auto &cv : m_map) { + cv.second->notify_one(); + } + } + + template + bool wait_for(const stream_name_t &name, std::unique_lock& __lock, const std::chrono::duration<_Rep, _Period>& __rtime, _Predicate __p) + { + assert(contains(m_map, name)); + return m_map[name]->wait_for(__lock, __rtime, __p); + } + + ScheduledCoreOpCV(std::unordered_map> cv_map) : m_map(std::move(cv_map)) + {} + +private: + std::unordered_map> m_map; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_SCHEDULED_CORE_OP_CV_HPP_ */ diff --git a/hailort/libhailort/src/scheduled_network_group.cpp b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.cpp similarity index 54% rename from hailort/libhailort/src/scheduled_network_group.cpp rename to hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.cpp index a9a60da..037fb57 100644 --- a/hailort/libhailort/src/scheduled_network_group.cpp +++ b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.cpp @@ -3,45 +3,46 @@ * Distributed under the MIT license (https://opensource.org/licenses/MIT) **/ /** - * @file network_group_scheduler.cpp - * @brief: Network scheduler + * @file scheduled_core_op_state.cpp + * @brief: Scheduled CoreOp **/ -#include "scheduled_network_group.hpp" -#include "context_switch/network_group_internal.hpp" -#include "hef_internal.hpp" -#include "vdevice_stream_multiplexer_wrapper.hpp" -#include "scheduler_oracle.hpp" +#include "vdevice/vdevice_stream_multiplexer_wrapper.hpp" +#include "vdevice/scheduler/scheduler_oracle.hpp" +#include "vdevice/scheduler/scheduled_core_op_state.hpp" +#include "hef/hef_internal.hpp" -#include namespace hailort { -ScheduledNetworkGroup::ScheduledNetworkGroup(std::shared_ptr cng, std::chrono::milliseconds timeout, - uint16_t max_batch_size, StreamInfoVector &stream_infos, std::string network_group_name) : - m_cng(cng), +#define SINGLE_CONTEXT_BATCH_SIZE (1) + +ScheduledCoreOp::ScheduledCoreOp(std::shared_ptr core_op, std::chrono::milliseconds timeout, + uint16_t max_batch_size, StreamInfoVector &stream_infos, std::string core_op_name) : + m_core_op(core_op), m_last_run_time_stamp(std::chrono::steady_clock::now()), m_timeout(std::move(timeout)), m_frame_was_sent(false), m_max_batch_size(max_batch_size), - m_network_group_name(network_group_name), + m_priority(HAILO_SCHEDULER_PRIORITY_NORMAL), + m_last_device_index(INVALID_DEVICE_ID), + m_core_op_name(core_op_name), m_inputs_names(), m_outputs_names(), - m_is_nms(false) + m_is_nms(false), + m_ready_to_switch(false) { - // Prepare empty counters for the added cng + // Prepare empty counters for the added core-op for (const auto &stream_info : stream_infos) { m_min_threshold_per_stream[stream_info.name] = DEFAULT_SCHEDULER_MIN_THRESHOLD; if (HAILO_H2D_STREAM == stream_info.direction) { m_requested_write_frames.insert(stream_info.name); m_finished_write_frames.insert(stream_info.name); - m_h2d_requested_transferred_frames.insert(stream_info.name); m_h2d_finished_transferred_frames.insert(stream_info.name); m_inputs_names.push_back(stream_info.name); } else { m_requested_read_frames.insert(stream_info.name); - m_ongoing_read_frames.insert(stream_info.name); m_finished_read_frames.insert(stream_info.name); m_d2h_finished_transferred_frames.insert(stream_info.name); m_outputs_names.push_back(stream_info.name); @@ -53,28 +54,25 @@ ScheduledNetworkGroup::ScheduledNetworkGroup(std::shared_ptr> ScheduledNetworkGroup::create(std::shared_ptr added_cng, StreamInfoVector &stream_infos) +Expected> ScheduledCoreOp::create(std::shared_ptr added_core_op, StreamInfoVector &stream_infos) { auto timeout = DEFAULT_SCHEDULER_TIMEOUT; uint16_t max_batch_size = CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE; - auto cng_base = std::dynamic_pointer_cast(added_cng); - assert(nullptr != cng_base); - if (cng_base->get_supported_features().multi_context) { - auto batch_size = cng_base->get_stream_batch_size(stream_infos[0].name); + if (added_core_op->get_supported_features().multi_context) { + auto batch_size = added_core_op->get_stream_batch_size(stream_infos[0].name); CHECK_EXPECTED(batch_size); - if (batch_size.value() > SINGLE_CONTEXT_BATCH_SIZE) { max_batch_size = batch_size.release(); } } - return make_shared_nothrow(added_cng, timeout, max_batch_size, stream_infos, added_cng->name()); + return make_shared_nothrow(added_core_op, timeout, max_batch_size, stream_infos, added_core_op->name()); } -bool ScheduledNetworkGroup::has_enough_space_in_read_buffers(uint32_t ongoing_frames) +bool ScheduledCoreOp::has_enough_space_in_read_buffers(uint32_t ongoing_frames) { - auto output_streams = m_cng->get_output_streams(); + auto output_streams = m_core_op->get_output_streams(); for (auto &output_stream : output_streams) { OutputStreamBase &vdevice_output = static_cast(output_stream.get()); if (auto pending_frames_size = vdevice_output.get_buffer_frames_size()) { @@ -87,7 +85,20 @@ bool ScheduledNetworkGroup::has_enough_space_in_read_buffers(uint32_t ongoing_fr return true; } -bool ScheduledNetworkGroup::has_input_written_most_frames(const std::string &stream_name) +uint16_t ScheduledCoreOp::get_min_input_buffers_count(uint32_t device_count) +{ + auto input_streams = m_core_op->get_input_streams(); + uint16_t buffers_count = UINT16_MAX; + for (auto &input_stream : input_streams) { + InputStreamBase &vdevice_input = static_cast(input_stream.get()); + if (auto pending_frames_size = vdevice_input.get_buffer_frames_size()) { + buffers_count = std::min(buffers_count, static_cast(pending_frames_size.value() / device_count)); + } + } + return buffers_count; +} + +bool ScheduledCoreOp::has_input_written_most_frames(const std::string &stream_name) { auto total_writes = total_written_frames_count(); return total_writes[stream_name] == get_max_value_of_unordered_map(total_writes); @@ -95,21 +106,20 @@ bool ScheduledNetworkGroup::has_input_written_most_frames(const std::string &str // TODO: Use get_pre_transfer_h2d_frames_count + get_h2d_transferred_frames_count // TODO: Avoid returning map (malloc) -std::unordered_map ScheduledNetworkGroup::total_written_frames_count() +std::unordered_map ScheduledCoreOp::total_written_frames_count() { std::unordered_map write_sum; for (const auto &name : get_inputs_names()) { write_sum[name] = m_requested_write_frames[name] + m_finished_write_frames[name] - + m_h2d_requested_transferred_frames[name] + m_h2d_finished_transferred_frames[name]; } return write_sum; } // TODO: Use max(m_d2h_finished_transferred_frames) == 0 instead -bool ScheduledNetworkGroup::has_pending_frames() +bool ScheduledCoreOp::has_pending_frames() { - uint32_t h2d_transferred_frames_count = get_h2d_transferred_frames_count(); + auto h2d_transferred_frames_count = m_h2d_finished_transferred_frames.get_max_value(); for (const auto &name : get_outputs_names()) { if (m_finished_read_frames[name] < h2d_transferred_frames_count) { return true; @@ -118,40 +128,27 @@ bool ScheduledNetworkGroup::has_pending_frames() return false; } -uint32_t ScheduledNetworkGroup::get_h2d_transferred_frames_count() +bool ScheduledCoreOp::can_stream_read(const std::string &stream_name) { - std::unordered_map transferred_frames; - for (const auto &name : get_inputs_names()) { - transferred_frames[name] = m_h2d_requested_transferred_frames[name] + m_h2d_finished_transferred_frames[name]; - } - return get_max_value_of_unordered_map(transferred_frames); + return !m_output_streams_read_orders[stream_name].empty(); } -bool ScheduledNetworkGroup::can_stream_read(const std::string &stream_name) +bool ScheduledCoreOp::can_stream_write(const std::string &stream_name) { - return !m_output_streams_read_orders[stream_name].empty(); + auto total_written_frames = total_written_frames_count()[stream_name]; + auto min_finished_read = finished_read_frames_min_value(); + auto ongoing_frames = (min_finished_read < total_written_frames) ? (total_written_frames - min_finished_read) : 0; + return has_enough_space_in_read_buffers(ongoing_frames); } -bool ScheduledNetworkGroup::use_dynamic_batch_flow() + +bool ScheduledCoreOp::use_dynamic_batch_flow() { return (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE != m_max_batch_size); } -bool ScheduledNetworkGroup::has_ng_drained_everything(bool streaming_mode) +bool ScheduledCoreOp::has_core_op_drained_everything() { - // On streaming mode we want to check those conditions even on NMS - if (!is_nms() || streaming_mode) { - if (!m_requested_write_frames.empty()) { - return false; - } - if (!m_finished_write_frames.empty()) { - return false; - } - if (!m_h2d_requested_transferred_frames.empty()) { - return false; - } - } - uint32_t written_frames = m_h2d_finished_transferred_frames.get_max_value(); for (const auto &name : get_outputs_names()) { if ((m_finished_read_frames[name] + m_d2h_finished_transferred_frames[name]) < written_frames) { @@ -161,7 +158,7 @@ bool ScheduledNetworkGroup::has_ng_drained_everything(bool streaming_mode) return true; } -void ScheduledNetworkGroup::decrease_current_ng_counters() +void ScheduledCoreOp::decrease_current_core_op_counters() { // Decrease only if counter is 2 or bigger because reaching 0 can cause states to change if (!m_h2d_finished_transferred_frames.all_values_bigger_or_equal(2)) { @@ -179,7 +176,7 @@ void ScheduledNetworkGroup::decrease_current_ng_counters() } } -uint32_t ScheduledNetworkGroup::get_pre_transfer_h2d_frames_count() +uint32_t ScheduledCoreOp::get_pre_transfer_h2d_frames_count() { std::unordered_map write_sum; for (const auto &name : get_inputs_names()) { @@ -188,77 +185,96 @@ uint32_t ScheduledNetworkGroup::get_pre_transfer_h2d_frames_count() return get_max_value_of_unordered_map(write_sum); } -hailo_status ScheduledNetworkGroup::set_timeout(const std::chrono::milliseconds &timeout, const stream_name_t &stream_name) +hailo_status ScheduledCoreOp::set_timeout(const std::chrono::milliseconds &timeout, const stream_name_t &stream_name) { CHECK(!m_frame_was_sent, HAILO_INVALID_OPERATION, - "Setting scheduler timeout is allowed only before sending / receiving frames on the network group."); + "Setting scheduler timeout is allowed only before sending / receiving frames on the core-op."); m_timeout = timeout; - auto name = (stream_name.empty()) ? get_network_group_name() : stream_name; + auto name = (stream_name.empty()) ? get_core_op_name() : stream_name; LOGGER__INFO("Setting scheduler timeout of {} to {}ms", name, timeout.count()); return HAILO_SUCCESS; } -hailo_status ScheduledNetworkGroup::set_threshold(uint32_t threshold, const stream_name_t &stream_name) +hailo_status ScheduledCoreOp::set_threshold(uint32_t threshold, const stream_name_t &stream_name) { CHECK((CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == m_max_batch_size) || (threshold <= m_max_batch_size), HAILO_INVALID_ARGUMENT, "Threshold must be equal or lower than the maximum batch size!"); CHECK(!m_frame_was_sent, HAILO_INVALID_OPERATION, - "Setting scheduler threshold is allowed only before sending / receiving frames on the network group."); + "Setting scheduler threshold is allowed only before sending / receiving frames on the core-op."); // TODO: Support setting threshold per stream. currently stream_name is always empty and de-facto we set threshold for the whole NG for (auto &threshold_per_stream_pair : m_min_threshold_per_stream) { threshold_per_stream_pair.second = threshold; } - auto name = (stream_name.empty()) ? get_network_group_name() : stream_name; + auto name = (stream_name.empty()) ? get_core_op_name() : stream_name; LOGGER__INFO("Setting scheduler threshold of {} to {} frames", name, threshold); return HAILO_SUCCESS; } -std::string ScheduledNetworkGroup::get_network_group_name() +core_op_priority_t ScheduledCoreOp::get_priority() +{ + return m_priority; +} + +void ScheduledCoreOp::set_priority(core_op_priority_t priority) { - return m_network_group_name; + m_priority = priority; } +uint32_t ScheduledCoreOp::get_last_device_index() +{ + return m_last_device_index; +} + +void ScheduledCoreOp::set_last_device_index(uint32_t device_index) +{ + m_last_device_index = device_index; +} -std::shared_ptr ScheduledNetworkGroup::get_network_group() +std::string ScheduledCoreOp::get_core_op_name() { - return m_cng; + return m_core_op_name; } -void ScheduledNetworkGroup::mark_frame_sent() +std::shared_ptr ScheduledCoreOp::get_core_op() +{ + return m_core_op; +} + +void ScheduledCoreOp::mark_frame_sent() { m_frame_was_sent = true; } -std::chrono::time_point ScheduledNetworkGroup::get_last_run_timestamp() +std::chrono::time_point ScheduledCoreOp::get_last_run_timestamp() { return m_last_run_time_stamp; } -void ScheduledNetworkGroup::set_last_run_timestamp(const std::chrono::time_point ×tamp) +void ScheduledCoreOp::set_last_run_timestamp(const std::chrono::time_point ×tamp) { m_last_run_time_stamp = timestamp; } -Expected ScheduledNetworkGroup::get_timeout(const stream_name_t &stream_name) +Expected ScheduledCoreOp::get_timeout(const stream_name_t &stream_name) { CHECK_AS_EXPECTED(stream_name.empty(), HAILO_INVALID_OPERATION, "timeout per network is not supported"); auto timeout = m_timeout; return timeout; } -Expected ScheduledNetworkGroup::get_threshold(const stream_name_t &stream_name) +Expected ScheduledCoreOp::get_threshold(const stream_name_t &stream_name) { CHECK_AS_EXPECTED(contains(m_min_threshold_per_stream, stream_name), HAILO_NOT_FOUND); return m_min_threshold_per_stream[stream_name].load(); } -uint16_t ScheduledNetworkGroup::get_max_batch_size() +uint16_t ScheduledCoreOp::get_max_batch_size() { if (!use_dynamic_batch_flow()) { return SINGLE_CONTEXT_BATCH_SIZE; @@ -266,119 +282,94 @@ uint16_t ScheduledNetworkGroup::get_max_batch_size() return m_max_batch_size; } -Counter &ScheduledNetworkGroup::requested_write_frames() +Counter &ScheduledCoreOp::requested_write_frames() { return m_requested_write_frames; } -std::atomic_uint32_t &ScheduledNetworkGroup::requested_write_frames(const stream_name_t &stream_name) +std::atomic_uint32_t &ScheduledCoreOp::requested_write_frames(const stream_name_t &stream_name) { return m_requested_write_frames[stream_name]; } -uint32_t ScheduledNetworkGroup::requested_write_frames_max_value() -{ - return m_requested_write_frames.get_max_value(); -} - -Counter &ScheduledNetworkGroup::finished_write_frames() +Counter &ScheduledCoreOp::finished_write_frames() { return m_finished_write_frames; } -std::atomic_uint32_t &ScheduledNetworkGroup::finished_write_frames(const stream_name_t &stream_name) +std::atomic_uint32_t &ScheduledCoreOp::finished_write_frames(const stream_name_t &stream_name) { return m_finished_write_frames[stream_name]; } -uint32_t ScheduledNetworkGroup::finished_write_frames_min_value() +uint32_t ScheduledCoreOp::finished_write_frames_min_value() { return m_finished_write_frames.get_min_value(); } -Counter &ScheduledNetworkGroup::h2d_requested_transferred_frames() -{ - return m_h2d_requested_transferred_frames; -} - -std::atomic_uint32_t &ScheduledNetworkGroup::h2d_requested_transferred_frames(const stream_name_t &stream_name) -{ - return m_h2d_requested_transferred_frames[stream_name]; -} - -Counter &ScheduledNetworkGroup::h2d_finished_transferred_frames() +Counter &ScheduledCoreOp::h2d_finished_transferred_frames() { return m_h2d_finished_transferred_frames; } -std::atomic_uint32_t &ScheduledNetworkGroup::h2d_finished_transferred_frames(const stream_name_t &stream_name) +std::atomic_uint32_t &ScheduledCoreOp::h2d_finished_transferred_frames(const stream_name_t &stream_name) { return m_h2d_finished_transferred_frames[stream_name]; } -Counter &ScheduledNetworkGroup::requested_read_frames() +Counter &ScheduledCoreOp::requested_read_frames() { return m_requested_read_frames; } -std::atomic_uint32_t &ScheduledNetworkGroup::requested_read_frames(const stream_name_t &stream_name) +std::atomic_uint32_t &ScheduledCoreOp::requested_read_frames(const stream_name_t &stream_name) { return m_requested_read_frames[stream_name]; } -Counter &ScheduledNetworkGroup::ongoing_read_frames() -{ - return m_ongoing_read_frames; -} - -std::atomic_uint32_t &ScheduledNetworkGroup::ongoing_read_frames(const stream_name_t &stream_name) -{ - return m_ongoing_read_frames[stream_name]; -} - -Counter &ScheduledNetworkGroup::d2h_finished_transferred_frames() +Counter &ScheduledCoreOp::d2h_finished_transferred_frames() { return m_d2h_finished_transferred_frames; } -std::atomic_uint32_t &ScheduledNetworkGroup::d2h_finished_transferred_frames(const stream_name_t &stream_name) +std::atomic_uint32_t &ScheduledCoreOp::d2h_finished_transferred_frames(const stream_name_t &stream_name) { return m_d2h_finished_transferred_frames[stream_name]; } -Counter &ScheduledNetworkGroup::finished_read_frames() +Counter &ScheduledCoreOp::finished_read_frames() { return m_finished_read_frames; } -std::atomic_uint32_t &ScheduledNetworkGroup::finished_read_frames(const stream_name_t &stream_name) +std::atomic_uint32_t &ScheduledCoreOp::finished_read_frames(const stream_name_t &stream_name) { return m_finished_read_frames[stream_name]; } -uint32_t ScheduledNetworkGroup::finished_read_frames_min_value() +uint32_t ScheduledCoreOp::finished_read_frames_min_value() { return m_finished_read_frames.get_min_value(); } -const std::vector &ScheduledNetworkGroup::get_inputs_names() +const std::vector &ScheduledCoreOp::get_inputs_names() { return m_inputs_names; } -const std::vector &ScheduledNetworkGroup::get_outputs_names() +const std::vector &ScheduledCoreOp::get_outputs_names() { return m_outputs_names; } -void ScheduledNetworkGroup::push_device_index(uint32_t device_index) +void ScheduledCoreOp::push_device_index(uint32_t device_index) { for (auto& stream_name : get_outputs_names()) { m_output_streams_read_orders[stream_name].push(device_index); } } -uint32_t ScheduledNetworkGroup::pop_device_index(const stream_name_t &stream_name) +uint32_t ScheduledCoreOp::pop_device_index(const stream_name_t &stream_name) { assert(contains(m_output_streams_read_orders, stream_name)); assert(!m_output_streams_read_orders[stream_name].empty()); @@ -388,4 +379,19 @@ uint32_t ScheduledNetworkGroup::pop_device_index(const stream_name_t &stream_nam return device_index; } +bool ScheduledCoreOp::is_ready_to_switch() +{ + return m_ready_to_switch; +} + +void ScheduledCoreOp::mark_ready_to_switch() +{ + m_ready_to_switch = true; +} + +void ScheduledCoreOp::mark_unready_to_switch() +{ + m_ready_to_switch = false; +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/scheduled_network_group.hpp b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.hpp similarity index 73% rename from hailort/libhailort/src/scheduled_network_group.hpp rename to hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.hpp index bd0b149..29b50ae 100644 --- a/hailort/libhailort/src/scheduled_network_group.hpp +++ b/hailort/libhailort/src/vdevice/scheduler/scheduled_core_op_state.hpp @@ -4,18 +4,19 @@ **/ /** * @file network_group_scheduler.hpp - * @brief Class declaration for NetworkGroupScheduler that schedules network groups to be active depending on the scheduling algorithm. + * @brief Class declaration for CoreOpsScheduler that schedules core-ops to be active depending on the scheduling algorithm. **/ -#ifndef _HAILO_SCHEDULED_NETWORK_GROUP_HPP_ -#define _HAILO_SCHEDULED_NETWORK_GROUP_HPP_ +#ifndef _HAILO_SCHEDULED_CORE_OP_HPP_ +#define _HAILO_SCHEDULED_CORE_OP_HPP_ #include "hailo/hailort.h" #include "hailo/expected.hpp" -#include "hailo/network_group.hpp" +#include "hailo/network_rate_calculator.hpp" + #include "common/utils.hpp" -#include "common/filesystem.hpp" -#include "scheduler_mon.hpp" + +#include "core_op/core_op.hpp" #include #include @@ -26,8 +27,10 @@ namespace hailort #define DEFAULT_SCHEDULER_TIMEOUT (std::chrono::milliseconds(0)) #define DEFAULT_SCHEDULER_MIN_THRESHOLD (0) +#define INVALID_DEVICE_ID (UINT32_MAX) using stream_name_t = std::string; +using core_op_priority_t = uint8_t; #define SINGLE_CONTEXT_BATCH_SIZE (1) @@ -97,31 +100,36 @@ private: std::unordered_map m_map; }; -class ScheduledNetworkGroup +class ScheduledCoreOp { public: - static Expected> create(std::shared_ptr added_cng, StreamInfoVector &stream_infos); + static Expected> create(std::shared_ptr added_core_op, StreamInfoVector &stream_infos); - virtual ~ScheduledNetworkGroup() = default; - ScheduledNetworkGroup(const ScheduledNetworkGroup &other) = delete; - ScheduledNetworkGroup &operator=(const ScheduledNetworkGroup &other) = delete; - ScheduledNetworkGroup &operator=(ScheduledNetworkGroup &&other) = delete; - ScheduledNetworkGroup(ScheduledNetworkGroup &&other) noexcept = delete; + virtual ~ScheduledCoreOp() = default; + ScheduledCoreOp(const ScheduledCoreOp &other) = delete; + ScheduledCoreOp &operator=(const ScheduledCoreOp &other) = delete; + ScheduledCoreOp &operator=(ScheduledCoreOp &&other) = delete; + ScheduledCoreOp(ScheduledCoreOp &&other) noexcept = delete; bool has_enough_space_in_read_buffers(uint32_t ongoing_frames); + uint16_t get_min_input_buffers_count(uint32_t device_count); bool has_input_written_most_frames(const std::string &stream_name); std::unordered_map total_written_frames_count(); bool has_pending_frames(); bool can_stream_read(const std::string &stream_name); + bool can_stream_write(const std::string &stream_name); bool use_dynamic_batch_flow(); - bool has_ng_drained_everything(bool streaming_mode); - void decrease_current_ng_counters(); + bool has_core_op_drained_everything(); + void decrease_current_core_op_counters(); uint32_t get_pre_transfer_h2d_frames_count(); - std::string get_network_group_name(); - uint32_t get_h2d_transferred_frames_count(); + bool is_ready_to_switch(); + void mark_ready_to_switch(); + void mark_unready_to_switch(); - std::shared_ptr get_network_group(); + std::string get_core_op_name(); + + std::shared_ptr get_core_op(); void mark_frame_sent(); @@ -133,24 +141,25 @@ public: Expected get_threshold(const stream_name_t &stream_name); hailo_status set_threshold(uint32_t threshold, const stream_name_t &stream_name = ""); + core_op_priority_t get_priority(); + void set_priority(core_op_priority_t priority); + + uint32_t get_last_device_index(); + void set_last_device_index(uint32_t device_index); + uint16_t get_max_batch_size(); Counter &requested_write_frames(); std::atomic_uint32_t &requested_write_frames(const stream_name_t &stream_name); - uint32_t requested_write_frames_max_value(); Counter &finished_write_frames(); std::atomic_uint32_t &finished_write_frames(const stream_name_t &stream_name); uint32_t finished_write_frames_min_value(); - Counter &h2d_requested_transferred_frames(); - std::atomic_uint32_t &h2d_requested_transferred_frames(const stream_name_t &stream_name); Counter &h2d_finished_transferred_frames(); std::atomic_uint32_t &h2d_finished_transferred_frames(const stream_name_t &stream_name); Counter &requested_read_frames(); std::atomic_uint32_t &requested_read_frames(const stream_name_t &stream_name); - Counter &ongoing_read_frames(); - std::atomic_uint32_t &ongoing_read_frames(const stream_name_t &stream_name); Counter &d2h_finished_transferred_frames(); std::atomic_uint32_t &d2h_finished_transferred_frames(const stream_name_t &stream_name); @@ -169,11 +178,11 @@ public: void push_device_index(uint32_t device_index); uint32_t pop_device_index(const stream_name_t &stream_name); - ScheduledNetworkGroup(std::shared_ptr cng, std::chrono::milliseconds timeout, - uint16_t max_batch_size, StreamInfoVector &stream_infos, std::string network_group_name); + ScheduledCoreOp(std::shared_ptr core_op, std::chrono::milliseconds timeout, + uint16_t max_batch_size, StreamInfoVector &stream_infos, std::string core_op_name); private: - std::shared_ptr m_cng; + std::shared_ptr m_core_op; std::chrono::time_point m_last_run_time_stamp; std::chrono::milliseconds m_timeout; @@ -184,18 +193,20 @@ private: Counter m_requested_write_frames; // 'wait_for_write()' has been called Counter m_finished_write_frames; // 'signal_finished_write()' has been called - frame is written in buffer (writes are a-sync) - Counter m_h2d_requested_transferred_frames; // 'send_pending_buffer()' has been called Counter m_h2d_finished_transferred_frames; // Frame has been transferred to device (intrpt was raised) Counter m_requested_read_frames; // 'wait_for_read()' has been called - Counter m_ongoing_read_frames; // 'wait_for_read()' has finished, the user is blocking on read (reads are sync) Counter m_d2h_finished_transferred_frames; // Frame has been transferred from device (intrpt was raised) Counter m_finished_read_frames; // 'signal_finish_read()' has been called - user finished getting the frame std::unordered_map m_min_threshold_per_stream; - std::string m_network_group_name; + core_op_priority_t m_priority; + + std::atomic_uint32_t m_last_device_index; + + std::string m_core_op_name; std::vector m_inputs_names; std::vector m_outputs_names; @@ -203,8 +214,11 @@ private: std::unordered_map> m_output_streams_read_orders; bool m_is_nms; + + // TODO: Remove this flag when the old scheduling mode will be deprecated + std::atomic_bool m_ready_to_switch; }; } /* namespace hailort */ -#endif /* _HAILO_SCHEDULED_NETWORK_GROUP_HPP_ */ +#endif /* _HAILO_SCHEDULED_CORE_OP_HPP_ */ diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduled_stream.hpp b/hailort/libhailort/src/vdevice/scheduler/scheduled_stream.hpp new file mode 100644 index 0000000..fb89a62 --- /dev/null +++ b/hailort/libhailort/src/vdevice/scheduler/scheduled_stream.hpp @@ -0,0 +1,105 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file scheduled_stream.hpp + * @brief Internal stream implementation for scheduled streams + * + **/ + +#ifndef HAILO_SCHEDULED_STREAM_HPP_ +#define HAILO_SCHEDULED_STREAM_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "stream_common/stream_internal.hpp" +#include "vdevice/vdevice_internal.hpp" +#include "vdevice/vdevice_stream.hpp" +#include "vdma/vdma_device.hpp" + + +namespace hailort +{ + +class ScheduledInputStream : public InputVDeviceBaseStream { +public: + ScheduledInputStream( + std::vector> &&streams, + const scheduler_core_op_handle_t &core_op_handle, + EventPtr &&core_op_activated_event, + const LayerInfo &layer_info, + CoreOpsSchedulerWeakPtr core_ops_scheduler, + hailo_status &status) : + InputVDeviceBaseStream(std::move(streams), std::move(core_op_activated_event), layer_info, status), + m_core_op_handle(core_op_handle), + m_core_ops_scheduler(core_ops_scheduler) + {} + + virtual hailo_status abort() override; + virtual hailo_status clear_abort() override; + virtual bool is_scheduled() override { return true; }; + + virtual void notify_all() override + { + auto scheduler = m_core_ops_scheduler.lock(); + if (nullptr == scheduler) { + LOGGER__CRITICAL("Failed to acquire scheduler"); + return; + } + scheduler->notify_all(); + + for (auto &stream : m_streams) { + stream.get().notify_all(); + } + } + +protected: + virtual Expected sync_write_raw_buffer(const MemoryView &buffer, + const std::function &should_cancel = []() { return false; }); + + Expected sync_write_raw_buffer_impl(const MemoryView &buffer, scheduler_core_op_handle_t core_op_handle, + const std::function &should_cancel); + + scheduler_core_op_handle_t m_core_op_handle; + CoreOpsSchedulerWeakPtr m_core_ops_scheduler; + +private: + hailo_status abort_impl(scheduler_core_op_handle_t core_op_handle); + hailo_status clear_abort_impl(scheduler_core_op_handle_t core_op_handle); +}; + +class ScheduledOutputStream : public OutputVDeviceBaseStream { +public: + ScheduledOutputStream( + std::vector> &&streams, + const scheduler_core_op_handle_t &core_op_handle, + const LayerInfo &layer_info, + EventPtr &&core_op_activated_event, + CoreOpsSchedulerWeakPtr core_ops_scheduler, + hailo_status &status) : + OutputVDeviceBaseStream(std::move(streams), layer_info, std::move(core_op_activated_event), status), + m_core_op_handle(core_op_handle), + m_core_ops_scheduler(core_ops_scheduler) + {} + + virtual hailo_status abort() override; + virtual hailo_status clear_abort() override; + virtual bool is_scheduled() override { return true; }; + +protected: + virtual hailo_status read(MemoryView buffer) override; + hailo_status read_impl(MemoryView buffer, scheduler_core_op_handle_t core_op_handle); + + scheduler_core_op_handle_t m_core_op_handle; + CoreOpsSchedulerWeakPtr m_core_ops_scheduler; + +private: + hailo_status abort_impl(scheduler_core_op_handle_t core_op_handle); + hailo_status clear_abort_impl(scheduler_core_op_handle_t core_op_handle); +}; + +} /* namespace hailort */ + +#endif /* HAILO_SCHEDULED_STREAM_HPP_ */ diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduler_base.hpp b/hailort/libhailort/src/vdevice/scheduler/scheduler_base.hpp new file mode 100644 index 0000000..e9fc0a9 --- /dev/null +++ b/hailort/libhailort/src/vdevice/scheduler/scheduler_base.hpp @@ -0,0 +1,121 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file scheduler_base.hpp + * @brief Class declaration for scheduler base class. + **/ + +#ifndef _HAILO_SCHEDULER_BASE_HPP_ +#define _HAILO_SCHEDULER_BASE_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "common/utils.hpp" +#include "common/filesystem.hpp" + +#include + + +namespace hailort +{ + +#define DEFAULT_SCHEDULER_TIMEOUT (std::chrono::milliseconds(0)) +#define DEFAULT_SCHEDULER_MIN_THRESHOLD (0) + +#define INVALID_CORE_OP_HANDLE (UINT32_MAX) +#define INVALID_DEVICE_ID (UINT32_MAX) + +using scheduler_core_op_handle_t = uint32_t; +using core_op_priority_t = uint8_t; + +using stream_name_t = std::string; + +struct ActiveDeviceInfo { + ActiveDeviceInfo(uint32_t device_id, const std::string &device_bdf_id, const std::string &device_arch) : + current_core_op_handle(INVALID_CORE_OP_HANDLE), next_core_op_handle(INVALID_CORE_OP_HANDLE), is_switching_core_op(false), + current_batch_size(0), current_cycle_requested_transferred_frames_h2d(), current_cycle_finished_transferred_frames_d2h(), + current_cycle_finished_read_frames_d2h(), device_id(device_id), device_bdf_id(device_bdf_id), device_arch(device_arch) + {} + scheduler_core_op_handle_t current_core_op_handle; + scheduler_core_op_handle_t next_core_op_handle; + std::atomic_bool is_switching_core_op; + std::atomic_uint32_t current_batch_size; + std::unordered_map> current_cycle_requested_transferred_frames_h2d; + std::unordered_map> current_cycle_finished_transferred_frames_d2h; + std::unordered_map> current_cycle_finished_read_frames_d2h; + uint32_t device_id; + std::string device_bdf_id; + std::string device_arch; +}; + + +class SchedulerBase +{ +public: + hailo_scheduling_algorithm_t algorithm() + { + return m_algorithm; + } + + struct ReadyInfo { + bool threshold = false; + bool timeout = false; + bool is_ready = false; + }; + + virtual ReadyInfo is_core_op_ready(const scheduler_core_op_handle_t &core_op_handle, bool check_threshold) = 0; + virtual bool has_core_op_drained_everything(const scheduler_core_op_handle_t &core_op_handle, uint32_t device_id) = 0; + + virtual uint32_t get_device_count() const + { + return static_cast(m_devices.size()); + } + + virtual std::shared_ptr get_devices_info(uint32_t device_id) + { + return m_devices[device_id]; + } + + virtual std::map> get_core_op_priority_map() + { + return m_core_op_priority; + } + + virtual scheduler_core_op_handle_t get_last_choosen_core_op(core_op_priority_t priority) + { + return m_last_choosen_core_op[priority]; + } + + virtual void set_last_choosen_core_op(const core_op_priority_t priority, const scheduler_core_op_handle_t &core_op_handle) + { + m_last_choosen_core_op[priority] = core_op_handle; + } + +protected: + SchedulerBase(hailo_scheduling_algorithm_t algorithm, uint32_t device_count, std::vector &devices_bdf_id, + std::vector &devices_arch) : m_algorithm(algorithm) + { + for (uint32_t i = 0; i < device_count; i++) { + m_devices.push_back(make_shared_nothrow(i, devices_bdf_id[i], devices_arch[i])); + } + }; + + virtual ~SchedulerBase() = default; + SchedulerBase(const SchedulerBase &other) = delete; + SchedulerBase &operator=(const SchedulerBase &other) = delete; + SchedulerBase &operator=(SchedulerBase &&other) = delete; + SchedulerBase(SchedulerBase &&other) noexcept = delete; + + std::vector> m_devices; + std::map> m_core_op_priority; + + hailo_scheduling_algorithm_t m_algorithm; + std::unordered_map m_last_choosen_core_op; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_SCHEDULER_BASE_HPP_ */ diff --git a/hailort/libhailort/src/scheduler_mon.hpp b/hailort/libhailort/src/vdevice/scheduler/scheduler_mon.hpp similarity index 93% rename from hailort/libhailort/src/scheduler_mon.hpp rename to hailort/libhailort/src/vdevice/scheduler/scheduler_mon.hpp index 158064b..64fa99b 100644 --- a/hailort/libhailort/src/scheduler_mon.hpp +++ b/hailort/libhailort/src/vdevice/scheduler/scheduler_mon.hpp @@ -10,7 +10,8 @@ #ifndef _HAILO_SCHEDULER_MON_HPP_ #define _HAILO_SCHEDULER_MON_HPP_ -#include "hailo/hailort.hpp" +#include "hailo/hailort.h" + #include "common/filesystem.hpp" #if defined(_MSC_VER) @@ -30,11 +31,12 @@ #include #include + namespace hailort { #define SCHEDULER_MON_TMP_DIR ("/tmp/hmon_files/") -#define SCHEDULER_MON_ENV_VAR ("SCHEDULER_MONITOR") +#define SCHEDULER_MON_ENV_VAR ("HAILO_MONITOR") #define DEFAULT_SCHEDULER_MON_INTERVAL (std::chrono::seconds(1)) #define SCHEDULER_MON_NAN_VAL (-1) diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduler_oracle.cpp b/hailort/libhailort/src/vdevice/scheduler/scheduler_oracle.cpp new file mode 100644 index 0000000..b39ea3d --- /dev/null +++ b/hailort/libhailort/src/vdevice/scheduler/scheduler_oracle.cpp @@ -0,0 +1,109 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file scheduler_oracle.cpp + * @brief: + **/ + +#include "vdevice/scheduler/scheduler_oracle.hpp" +#include "utils/profiler/tracer_macros.hpp" + + +namespace hailort +{ + +bool CoreOpsSchedulerOracle::choose_next_model(SchedulerBase &scheduler, uint32_t device_id, bool check_threshold) +{ + auto device_info = scheduler.get_devices_info(device_id); + auto priority_map = scheduler.get_core_op_priority_map(); + for (auto iter = priority_map.rbegin(); iter != priority_map.rend(); ++iter) { + auto priority_group_size = iter->second.size(); + + for (uint32_t i = 0; i < priority_group_size; i++) { + uint32_t index = scheduler.get_last_choosen_core_op(iter->first) + i + 1; + index %= static_cast(priority_group_size); + auto core_op_handle = iter->second[index]; + if (!is_core_op_active(scheduler, core_op_handle)) { + auto ready_info = scheduler.is_core_op_ready(core_op_handle, check_threshold); + if (ready_info.is_ready) { + TRACE(ChooseCoreOpTrace, "", core_op_handle, ready_info.threshold, ready_info.timeout, iter->first); + device_info->is_switching_core_op = true; + device_info->next_core_op_handle = core_op_handle; + scheduler.set_last_choosen_core_op(iter->first, index); + + return true; + } + } + } + } + + return false; +} + +// TODO: return device handle instead index +uint32_t CoreOpsSchedulerOracle::get_avail_device(SchedulerBase &scheduler, scheduler_core_op_handle_t core_op_handle) +{ + const bool check_threshold = false; + auto device_count = scheduler.get_device_count(); + + // Check if should be next + /* Checking (INVALID_CORE_OP_HANDLE == m_current_core_op) for activating the first time the scheduler is running. + In this case we don't want to check threshold. */ + for (uint32_t device_index = 0; device_index < device_count; device_index++) { + auto active_device_info = scheduler.get_devices_info(device_index); + if (active_device_info->is_switching_core_op && scheduler.has_core_op_drained_everything(active_device_info->current_core_op_handle, active_device_info->device_id) && + (((INVALID_CORE_OP_HANDLE == active_device_info->current_core_op_handle) && + scheduler.is_core_op_ready(core_op_handle, check_threshold).is_ready) || + (active_device_info->next_core_op_handle == core_op_handle))) { + return active_device_info->device_id; + } + } + + // Check if device Idle + // We dont need to check if the core op is ready, because the device is idle and if we arrive here frame is already sent and as a space in the output buffer. + for (uint32_t device_index = 0; device_index < device_count; device_index++) { + auto active_device_info = scheduler.get_devices_info(device_index); + if (!active_device_info->is_switching_core_op && scheduler.has_core_op_drained_everything(active_device_info->current_core_op_handle, active_device_info->device_id)) { + return active_device_info->device_id; + } + } + + return INVALID_DEVICE_ID; +} + +bool CoreOpsSchedulerOracle::should_stop_streaming(SchedulerBase &scheduler, core_op_priority_t core_op_priority) +{ + auto priority_map = scheduler.get_core_op_priority_map(); + for (auto iter = priority_map.rbegin(); (iter != priority_map.rend()) && (iter->first >= core_op_priority); ++iter) { + auto priority_group_size = iter->second.size(); + + for (uint32_t i = 0; i < priority_group_size; i++) { + uint32_t index = scheduler.get_last_choosen_core_op(iter->first) + i + 1; + index %= static_cast(priority_group_size); + auto core_op_handle = iter->second[index]; + // We dont want to stay with the same network group if there is a other qualified network group + if ((!is_core_op_active(scheduler, core_op_handle)) && scheduler.is_core_op_ready(core_op_handle, true).is_ready) { + return true; + } + } + } + + return false; +} + +bool CoreOpsSchedulerOracle::is_core_op_active(SchedulerBase &scheduler, scheduler_core_op_handle_t core_op_handle) +{ + auto device_count = scheduler.get_device_count(); + for (uint32_t device_index = 0; device_index < device_count; device_index++) { + auto active_device_info = scheduler.get_devices_info(device_index); + if (core_op_handle == active_device_info->current_core_op_handle) { + return true; + } + } + + return false; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdevice/scheduler/scheduler_oracle.hpp b/hailort/libhailort/src/vdevice/scheduler/scheduler_oracle.hpp new file mode 100644 index 0000000..766bf45 --- /dev/null +++ b/hailort/libhailort/src/vdevice/scheduler/scheduler_oracle.hpp @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file scheduler_oracle.hpp + * @brief + **/ + +#ifndef _HAILO_SCHEDULER_ORACLE_HPP_ +#define _HAILO_SCHEDULER_ORACLE_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "common/utils.hpp" + +#include "vdevice/scheduler/scheduler_base.hpp" + + +namespace hailort +{ + +class CoreOpsSchedulerOracle +{ +public: + static bool choose_next_model(SchedulerBase &scheduler, uint32_t device_id, bool check_threshold); + static uint32_t get_avail_device(SchedulerBase &scheduler, scheduler_core_op_handle_t core_op_handle); + static bool should_stop_streaming(SchedulerBase &scheduler, core_op_priority_t core_op_priority); + +private: + CoreOpsSchedulerOracle() {} + // TODO: Consider returning a vector of devices (we can use this function in other places) + static bool is_core_op_active(SchedulerBase &scheduler, scheduler_core_op_handle_t core_op_handle); +}; + +} /* namespace hailort */ + +#endif /* _HAILO_SCHEDULER_ORACLE_HPP_ */ diff --git a/hailort/libhailort/src/vdevice.cpp b/hailort/libhailort/src/vdevice/vdevice.cpp similarity index 58% rename from hailort/libhailort/src/vdevice.cpp rename to hailort/libhailort/src/vdevice/vdevice.cpp index 9045718..7030013 100644 --- a/hailort/libhailort/src/vdevice.cpp +++ b/hailort/libhailort/src/vdevice/vdevice.cpp @@ -11,19 +11,23 @@ #include "hailo/hailort.h" #include "hailo/vdevice.hpp" -#include "vdevice_internal.hpp" -#include "pcie_device.hpp" -#include "core_device.hpp" -#include "hailort_defaults.hpp" -#include "shared_resource_manager.hpp" -#include "context_switch/network_group_internal.hpp" -#include "context_switch/vdevice_network_group.hpp" +#include "hailo/hailort_defaults.hpp" + +#include "vdevice/vdevice_internal.hpp" +#include "vdevice/vdevice_core_op.hpp" + +#include "vdma/pcie/pcie_device.hpp" +#include "vdma/integrated/integrated_device.hpp" +#include "utils/shared_resource_manager.hpp" +#include "network_group/network_group_internal.hpp" +#include "core_op/core_op.hpp" #ifdef HAILO_SUPPORT_MULTI_PROCESS -#include "rpc_client_utils.hpp" +#include "service/rpc_client_utils.hpp" #include "rpc/rpc_definitions.hpp" #endif // HAILO_SUPPORT_MULTI_PROCESS + namespace hailort { @@ -72,6 +76,22 @@ void release_resource_if(bool condition, uint32_t key) { } } +Expected VDevice::create_configure_params(Hef &hef) const +{ + auto stream_interface = get_default_streams_interface(); + CHECK_EXPECTED(stream_interface, "Failed to get default streams interface"); + + return hef.create_configure_params(stream_interface.release()); +} + +Expected VDevice::create_configure_params(Hef &hef, const std::string &network_group_name) const +{ + auto stream_interface = get_default_streams_interface(); + CHECK_EXPECTED(stream_interface, "Failed to get default streams interface"); + + return hef.create_configure_params(stream_interface.release(), network_group_name); +} + VDeviceHandle::VDeviceHandle(uint32_t handle) : m_handle(handle) {} @@ -82,9 +102,8 @@ VDeviceHandle::~VDeviceHandle() Expected> VDeviceHandle::create(const hailo_vdevice_params_t ¶ms) { - CHECK_AS_EXPECTED((HAILO_SCHEDULING_ALGORITHM_NONE == params.scheduling_algorithm) - || (1 == params.device_count) || (VDeviceBase::enable_multi_device_schedeulr()), HAILO_NOT_SUPPORTED, - "Multiple devices scheduler feature is preview. To enable it, set env variable 'HAILO_ENABLE_MULTI_DEVICE_SCHEDULER' to 1"); + auto status = VDeviceBase::validate_params(params); + CHECK_SUCCESS_AS_EXPECTED(status); auto &manager = SharedResourceManager::get_instance(); auto create = [¶ms]() { @@ -152,24 +171,66 @@ VDeviceClient::VDeviceClient(std::unique_ptr client, uint32_t VDeviceClient::~VDeviceClient() { + // Note: We clear m_network_groups to prevent double destruction on ConfiguredNetworkGroupBase. + // Explanation: When the VDeviceClient is destructed, it's members are destructed last. + // That would cause the m_network_groups (vector of ConfiguredNetworkGroupClient) to be destructed after the vdevice in the service. + // The vdevice in the service will destruct the ConfiguredNetworkGroupBase, + // and then the ConfiguredNetworkGroupClient destructor will be called - causing double destruction on ConfiguredNetworkGroupBase. + m_network_groups.clear(); auto reply = m_client->VDevice_release(m_handle); if (reply != HAILO_SUCCESS) { LOGGER__CRITICAL("VDevice_release failed!"); } } +hailo_status VDeviceClient::before_fork() +{ + HailoRtRpcClientUtils::get_instance().before_fork(); + m_client.reset(); + return HAILO_SUCCESS; +} + +hailo_status VDeviceClient::create_client() +{ + grpc::ChannelArguments ch_args; + ch_args.SetMaxReceiveMessageSize(-1); + auto channel = grpc::CreateCustomChannel(HAILORT_SERVICE_DEFAULT_ADDR, grpc::InsecureChannelCredentials(), ch_args); + CHECK_NOT_NULL(channel, HAILO_INTERNAL_FAILURE); + auto client = make_unique_nothrow(channel); + CHECK_NOT_NULL(client, HAILO_INTERNAL_FAILURE); + m_client = std::move(client); + return HAILO_SUCCESS; +} + +hailo_status VDeviceClient::after_fork_in_parent() +{ + HailoRtRpcClientUtils::get_instance().after_fork_in_parent(); + return create_client(); +} + +hailo_status VDeviceClient::after_fork_in_child() +{ + HailoRtRpcClientUtils::get_instance().after_fork_in_child(); + auto status = create_client(); + CHECK_SUCCESS(status); + auto expected_dup_handle = m_client->VDevice_dup_handle(OsUtils::get_curr_pid(), m_handle); + CHECK_EXPECTED_AS_STATUS(expected_dup_handle); + m_handle = expected_dup_handle.value(); + return HAILO_SUCCESS; +} + Expected> VDeviceClient::create(const hailo_vdevice_params_t ¶ms) { grpc::ChannelArguments ch_args; ch_args.SetMaxReceiveMessageSize(-1); - auto channel = grpc::CreateCustomChannel(HAILO_DEFAULT_UDS_ADDR, grpc::InsecureChannelCredentials(), ch_args); + auto channel = grpc::CreateCustomChannel(HAILORT_SERVICE_DEFAULT_ADDR, grpc::InsecureChannelCredentials(), ch_args); CHECK_AS_EXPECTED(channel != nullptr, HAILO_INTERNAL_FAILURE); - auto client = std::unique_ptr(new HailoRtRpcClient(channel)); + auto client = make_unique_nothrow(channel); CHECK_AS_EXPECTED(client != nullptr, HAILO_OUT_OF_HOST_MEMORY); auto init_status = HailoRtRpcClientUtils::get_instance().init_client_service_communication(); CHECK_SUCCESS_AS_EXPECTED(init_status); - auto reply = client->VDevice_create(params, getpid()); + auto reply = client->VDevice_create(params, OsUtils::get_curr_pid()); CHECK_EXPECTED(reply); auto client_vdevice = std::unique_ptr(new VDeviceClient(std::move(client), reply.value())); @@ -181,17 +242,21 @@ Expected> VDeviceClient::create(const hailo_vdevice_par Expected VDeviceClient::configure(Hef &hef, const NetworkGroupsParamsMap &configure_params) { - auto networks_handles = m_client->VDevice_configure(m_handle, hef, getpid(), configure_params); + auto networks_handles = m_client->VDevice_configure(m_handle, hef, OsUtils::get_curr_pid(), configure_params); CHECK_EXPECTED(networks_handles); ConfiguredNetworkGroupVector networks; networks.reserve(networks_handles->size()); for (auto &handle : networks_handles.value()) { - auto channel = grpc::CreateChannel(HAILO_DEFAULT_UDS_ADDR, grpc::InsecureChannelCredentials()); - CHECK_AS_EXPECTED(channel != nullptr, HAILO_INTERNAL_FAILURE); + auto expected_client = HailoRtRpcClientUtils::create_client(); + CHECK_EXPECTED(expected_client); - auto client = std::unique_ptr(new HailoRtRpcClient(channel)); - networks.emplace_back(make_shared_nothrow(std::move(client), handle)); + auto client = expected_client.release(); + auto network_group = make_shared_nothrow(std::move(client), handle); + CHECK_NOT_NULL_AS_EXPECTED(network_group, HAILO_OUT_OF_HOST_MEMORY); + + networks.emplace_back(network_group); + m_network_groups.push_back(network_group); } return networks; } @@ -217,9 +282,6 @@ Expected VDeviceClient::get_default_streams_interface( Expected> VDevice::create(const hailo_vdevice_params_t ¶ms) { - CHECK_AS_EXPECTED(0 != params.device_count, HAILO_INVALID_ARGUMENT, - "VDevice creation failed. invalid device_count ({}).", params.device_count); - std::unique_ptr vdevice; if (params.multi_process_service) { #ifdef HAILO_SUPPORT_MULTI_PROCESS @@ -259,31 +321,60 @@ Expected> VDevice::create(const std::vector> VDeviceBase::create(const hailo_vdevice_params_t ¶ms) +hailo_status VDeviceBase::validate_params(const hailo_vdevice_params_t ¶ms) { - NetworkGroupSchedulerPtr scheduler_ptr; - if (HAILO_SCHEDULING_ALGORITHM_NONE != params.scheduling_algorithm) { - if (HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN == params.scheduling_algorithm) { - auto network_group_scheduler = NetworkGroupScheduler::create_round_robin(params.device_count); - CHECK_EXPECTED(network_group_scheduler); - scheduler_ptr = network_group_scheduler.release(); - } else { - LOGGER__ERROR("Unsupported scheduling algorithm"); - return make_unexpected(HAILO_INVALID_ARGUMENT); + CHECK(0 != params.device_count, HAILO_INVALID_ARGUMENT, + "VDevice creation failed. invalid device_count ({}).", params.device_count); + + if (params.device_ids != nullptr) { + for (uint32_t i = 0; i < params.device_count; i++) { + auto dev_type = Device::get_device_type(params.device_ids[i].id); + CHECK_EXPECTED_AS_STATUS(dev_type); + CHECK((Device::Type::ETH != dev_type.value() || (1 == params.device_count)), HAILO_INVALID_ARGUMENT, + "VDevice over ETH is supported for 1 device. Passed device_count: {}", params.device_count); + CHECK((Device::Type::ETH != dev_type.value() || (HAILO_SCHEDULING_ALGORITHM_NONE == params.scheduling_algorithm)), HAILO_INVALID_ARGUMENT, + "VDevice over ETH is not supported when scheduler is enabled."); } } + return HAILO_SUCCESS; +} +Expected> VDeviceBase::create(const hailo_vdevice_params_t ¶ms) +{ auto devices_expected = create_devices(params); CHECK_EXPECTED(devices_expected); auto devices = devices_expected.release(); + std::vector device_ids; + device_ids.reserve(params.device_count); + std::vector device_archs; + device_archs.reserve(params.device_count); + std::string vdevice_ids = "VDevice Infos:"; for (const auto &device : devices) { - auto info_str = device->get_dev_id(); - vdevice_ids += " " + std::string(info_str); + auto id_info_str = device->get_dev_id(); + device_ids.emplace_back(id_info_str); + auto device_arch = device->get_architecture(); + CHECK_EXPECTED(device_arch); + auto device_arch_str = HailoRTCommon::get_device_arch_str(device_arch.value()); + device_archs.emplace_back(device_arch_str); + vdevice_ids += " " + std::string(id_info_str); } LOGGER__INFO("{}", vdevice_ids); + + CoreOpsSchedulerPtr scheduler_ptr; + if (HAILO_SCHEDULING_ALGORITHM_NONE != params.scheduling_algorithm) { + if (HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN == params.scheduling_algorithm) { + auto core_ops_scheduler = CoreOpsScheduler::create_round_robin(params.device_count, device_ids, device_archs); + CHECK_EXPECTED(core_ops_scheduler); + scheduler_ptr = core_ops_scheduler.release(); + } else { + LOGGER__ERROR("Unsupported scheduling algorithm"); + return make_unexpected(HAILO_INVALID_ARGUMENT); + } + } + auto vdevice = std::unique_ptr(new (std::nothrow) VDeviceBase(std::move(devices), scheduler_ptr)); CHECK_AS_EXPECTED(nullptr != vdevice, HAILO_OUT_OF_HOST_MEMORY); @@ -296,91 +387,47 @@ Expected VDeviceBase::configure(Hef &hef, std::unique_lock lock(m_mutex); auto start_time = std::chrono::steady_clock::now(); - for (auto &device : m_devices) { - auto status = device->check_hef_is_compatible(hef); - CHECK_SUCCESS_AS_EXPECTED(status); - } - - auto local_config_params = configure_params; - if (local_config_params.empty()) { - // All stream iface should be the same - auto stream_interface = m_devices[0]->get_default_streams_interface(); - CHECK_EXPECTED(stream_interface); - auto config_params_exp = hef.create_configure_params(stream_interface.value()); - CHECK_EXPECTED(config_params_exp); - local_config_params = config_params_exp.release(); - } - - /* Validate batch size is identical for all networks in case scheduler is enabled */ - if (m_network_group_scheduler) { - uint16_t ref_batch_size = UINT16_MAX; - for (const auto &ng_params_pair : local_config_params) { - for (const auto &network_params_pair : ng_params_pair.second.network_params_by_name) { - if (UINT16_MAX == ref_batch_size) { - ref_batch_size = network_params_pair.second.batch_size; - } - CHECK_AS_EXPECTED(ref_batch_size == network_params_pair.second.batch_size, HAILO_INVALID_OPERATION, - "When scheduler is enabled, all networks should have the same batch_size. configure_params contains {} and {}. " - "To disable scheduler, set HAILO_SCHEDULING_ALGORITHM_NONE in VDevice creation.", ref_batch_size, network_params_pair.second.batch_size); - } - } - } + auto local_config_params = create_local_config_params(hef, configure_params); + CHECK_EXPECTED(local_config_params); ConfiguredNetworkGroupVector added_network_groups; added_network_groups.reserve(configure_params.size()); - for (const auto &network_params_pair : local_config_params) { - std::shared_ptr identical_ng = nullptr; - if (m_network_group_scheduler && PipelineMultiplexer::should_use_multiplexer()) { - for (auto &network_group : m_network_groups) { - if ((network_group->equals(hef, network_params_pair.first)) && (1 == network_group->get_input_streams().size())) { + for (const auto &network_params_pair : local_config_params.value()) { + std::vector> core_ops; + std::shared_ptr identical_core_op = nullptr; + if (m_core_ops_scheduler && PipelineMultiplexer::should_use_multiplexer()) { + for (auto &network_group : m_vdevice_core_ops) { + if ((network_group->equals(hef, network_params_pair)) && (1 == network_group->get_input_streams().size())) { // TODO (HRT-8634): Support multi-inputs NGs (multi networks) - identical_ng = network_group; + identical_core_op = network_group; break; } } } - std::shared_ptr vdevice_netwrok_group = nullptr; - if (identical_ng) { - auto vdevice_netwrok_group_exp = VDeviceNetworkGroup::duplicate(identical_ng); + std::shared_ptr vdevice_netwrok_group = nullptr; + if (identical_core_op) { + auto vdevice_netwrok_group_exp = VDeviceCoreOp::duplicate(identical_core_op); CHECK_EXPECTED(vdevice_netwrok_group_exp); vdevice_netwrok_group = vdevice_netwrok_group_exp.release(); - - vdevice_netwrok_group->set_network_group_handle(identical_ng->network_group_handle()); - vdevice_netwrok_group->create_vdevice_streams_from_duplicate(identical_ng); - + vdevice_netwrok_group->set_core_op_handle(identical_core_op->core_op_handle()); + vdevice_netwrok_group->create_vdevice_streams_from_duplicate(identical_core_op); } else { - ConfiguredNetworkGroupVector network_group_bundle; // bundle of the same NGs for all devices - network_group_bundle.reserve(m_devices.size()); - - for (auto &device : m_devices) { - auto ng_vector = device->configure(hef, { std::make_pair(network_params_pair.first, network_params_pair.second) }); - CHECK_EXPECTED(ng_vector); - - assert(1 == ng_vector->size()); - network_group_bundle.push_back(ng_vector.release()[0]); - } - - auto vdevice_netwrok_group_exp = VDeviceNetworkGroup::create(network_group_bundle, m_network_group_scheduler); - CHECK_EXPECTED(vdevice_netwrok_group_exp); - - vdevice_netwrok_group = vdevice_netwrok_group_exp.release(); - - auto ng_handle = INVALID_NETWORK_GROUP_HANDLE; - if (m_network_group_scheduler) { - auto network_group_handle_exp = m_network_group_scheduler->add_network_group(vdevice_netwrok_group); - CHECK_EXPECTED(network_group_handle_exp); - ng_handle = network_group_handle_exp.release(); - } - vdevice_netwrok_group->set_network_group_handle(ng_handle); - auto status = vdevice_netwrok_group->create_vdevice_streams_from_config_params(make_shared_nothrow(), ng_handle); - CHECK_SUCCESS_AS_EXPECTED(status); - - m_network_groups.push_back(vdevice_netwrok_group); + auto vdevice_netwrok_group_expected = create_vdevice_network_group(hef, network_params_pair); + CHECK_EXPECTED(vdevice_netwrok_group_expected); + vdevice_netwrok_group = vdevice_netwrok_group_expected.release(); + m_vdevice_core_ops.push_back(vdevice_netwrok_group); } - added_network_groups.push_back(vdevice_netwrok_group); + core_ops.push_back(vdevice_netwrok_group); + auto net_flow_ops = hef.pimpl->post_process_ops(vdevice_netwrok_group->name()); + auto net_group_expected = ConfiguredNetworkGroupBase::create(network_params_pair.second, std::move(core_ops), std::move(net_flow_ops)); + CHECK_EXPECTED(net_group_expected); + auto network_group_ptr = net_group_expected.release(); + + added_network_groups.push_back(network_group_ptr); + m_network_groups.push_back(network_group_ptr); } auto elapsed_time_ms = std::chrono::duration(std::chrono::steady_clock::now() - start_time).count(); @@ -402,9 +449,9 @@ Expected VDeviceBase::get_default_streams_interface() return stream_interface.release(); } -Expected>> VDeviceBase::create_devices(const hailo_vdevice_params_t ¶ms) +Expected>> VDeviceBase::create_devices(const hailo_vdevice_params_t ¶ms) { - std::vector> devices; + std::vector> devices; devices.reserve(params.device_count); const bool user_specific_devices = (params.device_ids != nullptr); @@ -416,7 +463,7 @@ Expected>> VDeviceBase::create_devices(c if (devices.size() == params.device_count) { break; } - auto device = VdmaDevice::create(device_id); + auto device = Device::create(device_id); CHECK_EXPECTED(device); // Validate That if (device_count != 1), device arch is not H8L. May be changed in SDK-28729 @@ -427,12 +474,16 @@ Expected>> VDeviceBase::create_devices(c "VDevice with multiple devices is not supported on HAILO_ARCH_HAILO8L. device {} is HAILO_ARCH_HAILO8L", device_id); } - auto status = device.value()->mark_as_used(); - if (!user_specific_devices && (HAILO_DEVICE_IN_USE == status)) { - // Continue only if the user didn't ask for specific devices - continue; + auto dev_type = Device::get_device_type(device_id); + CHECK_EXPECTED(dev_type); + if ((Device::Type::INTEGRATED == dev_type.value()) || (Device::Type::PCIE == dev_type.value())) { + auto status = dynamic_cast(*device.value()).mark_as_used(); + if (!user_specific_devices && (HAILO_DEVICE_IN_USE == status)) { + // Continue only if the user didn't ask for specific devices + continue; + } + CHECK_SUCCESS_AS_EXPECTED(status); } - CHECK_SUCCESS_AS_EXPECTED(status); devices.emplace_back(device.release()); } CHECK_AS_EXPECTED(params.device_count == devices.size(), HAILO_OUT_OF_PHYSICAL_DEVICES, @@ -460,5 +511,72 @@ Expected> VDeviceBase::get_device_ids(const hailo_vdevi } } +Expected VDeviceBase::create_local_config_params(Hef &hef, const NetworkGroupsParamsMap &configure_params) +{ + for (auto &device : m_devices) { + auto status = dynamic_cast(*device).check_hef_is_compatible(hef); + CHECK_SUCCESS_AS_EXPECTED(status); + } + + auto local_config_params = configure_params; + if (local_config_params.empty()) { + // All stream iface should be the same + auto config_params_exp = m_devices[0]->create_configure_params(hef); + CHECK_EXPECTED(config_params_exp); + local_config_params = config_params_exp.release(); + } + + /* Validate batch size is identical for all networks in case scheduler is enabled */ + if (m_core_ops_scheduler) { + uint16_t ref_batch_size = UINT16_MAX; + for (const auto &ng_params_pair : local_config_params) { + for (const auto &network_params_pair : ng_params_pair.second.network_params_by_name) { + if (UINT16_MAX == ref_batch_size) { + ref_batch_size = network_params_pair.second.batch_size; + } + CHECK_AS_EXPECTED(ref_batch_size == network_params_pair.second.batch_size, HAILO_INVALID_OPERATION, + "When scheduler is enabled, all networks should have the same batch_size. configure_params contains {} and {}. " + "To disable scheduler, set HAILO_SCHEDULING_ALGORITHM_NONE in VDevice creation.", ref_batch_size, network_params_pair.second.batch_size); + } + } + } + + return local_config_params; +} + +Expected> VDeviceBase::create_vdevice_network_group(Hef &hef, const std::pair ¶ms) +{ + std::vector> core_ops_bundle; // bundle of the same CoreOps for all devices + core_ops_bundle.reserve(m_devices.size()); + + // configure all the devices to this ng and then push the core ops to bundle vector + for (auto &device : m_devices) { + auto ng_vector = device->configure(hef, { std::make_pair(params.first, params.second) }); + CHECK_EXPECTED(ng_vector); + + assert(1 == ng_vector->size()); + auto network_group_base = std::dynamic_pointer_cast(ng_vector.value()[0]); + auto ng_core_ops = network_group_base->get_core_ops(); + + core_ops_bundle.insert(core_ops_bundle.begin(), ng_core_ops.begin(), ng_core_ops.end()); + } + + auto vdevice_netwrok_group_exp = VDeviceCoreOp::create(core_ops_bundle, m_core_ops_scheduler, hef.hash()); + CHECK_EXPECTED(vdevice_netwrok_group_exp); + auto vdevice_netwrok_group = vdevice_netwrok_group_exp.release(); + + auto ng_handle = INVALID_CORE_OP_HANDLE; + if (m_core_ops_scheduler) { + auto core_op_handle_exp = m_core_ops_scheduler->add_core_op(vdevice_netwrok_group); + CHECK_EXPECTED(core_op_handle_exp); + ng_handle = core_op_handle_exp.release(); + } + vdevice_netwrok_group->set_core_op_handle(ng_handle); + auto status = vdevice_netwrok_group->create_vdevice_streams_from_config_params(make_shared_nothrow(), ng_handle); + CHECK_SUCCESS_AS_EXPECTED(status); + + return vdevice_netwrok_group; +} + } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdevice/vdevice_core_op.cpp b/hailort/libhailort/src/vdevice/vdevice_core_op.cpp new file mode 100644 index 0000000..a37c01d --- /dev/null +++ b/hailort/libhailort/src/vdevice/vdevice_core_op.cpp @@ -0,0 +1,410 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdevice_core_op.cpp + * @brief: VDeviceCoreOp implementation + **/ + +#include "vdevice/vdevice_core_op.hpp" +#include "vdevice/vdevice_stream.hpp" +#include "vdevice/vdevice_stream_multiplexer_wrapper.hpp" +#include "net_flow/pipeline/vstream_internal.hpp" +#include "utils/profiler/tracer_macros.hpp" + + +namespace hailort +{ + +Expected> VDeviceActivatedCoreOp::create( + std::vector> &core_ops, + std::map> &input_streams, + std::map> &output_streams, + const hailo_activate_network_group_params_t &network_group_params, + EventPtr core_op_activated_event, uint16_t dynamic_batch_size, + AccumulatorPtr deactivation_time_accumulator, + bool resume_pending_stream_transfers) +{ + auto status = HAILO_UNINITIALIZED; + std::vector> activated_network_groups; + activated_network_groups.reserve(core_ops.size()); + for (auto core_op : core_ops) { + auto ang = core_op->create_activated_network_group(network_group_params, dynamic_batch_size, + resume_pending_stream_transfers); + CHECK_EXPECTED(ang); + activated_network_groups.emplace_back(ang.release()); + } + auto ang = VDeviceActivatedCoreOp(std::move(activated_network_groups), input_streams, output_streams, + network_group_params, core_op_activated_event, deactivation_time_accumulator, status); + + CHECK_SUCCESS_AS_EXPECTED(status); + std::unique_ptr activated_net_group_ptr = + make_unique_nothrow(std::move(ang)); + CHECK_AS_EXPECTED(nullptr != activated_net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); + + status = core_op_activated_event->signal(); + CHECK_SUCCESS_AS_EXPECTED(status, "Failed to signal network activation event"); + + return activated_net_group_ptr; +} + +const std::string &VDeviceActivatedCoreOp::get_network_group_name() const +{ + // network_group_name is same across all NGs + return m_activated_network_groups[0]->get_network_group_name(); +} + +Expected VDeviceActivatedCoreOp::get_intermediate_buffer(const IntermediateBufferKey &key) +{ + CHECK_AS_EXPECTED(1 == m_activated_network_groups.size(), HAILO_INVALID_OPERATION, "getting intermediate buffer is supported only over single device"); + return m_activated_network_groups[0]->get_intermediate_buffer(key); +} + +hailo_status VDeviceActivatedCoreOp::set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) +{ + for (auto &activated_network_group : m_activated_network_groups) { + auto status = activated_network_group->set_keep_nn_config_during_reset(keep_nn_config_during_reset); + CHECK_SUCCESS(status); + } + return HAILO_SUCCESS; +} + +VDeviceActivatedCoreOp::VDeviceActivatedCoreOp(std::vector> &&activated_network_groups, + std::map> &input_streams, std::map> &output_streams, + const hailo_activate_network_group_params_t &network_group_params, EventPtr core_op_activated_event, AccumulatorPtr deactivation_time_accumulator, hailo_status &status) + : ActivatedCoreOp(network_group_params, input_streams, output_streams, std::move(core_op_activated_event), status), + m_activated_network_groups(std::move(activated_network_groups)), m_should_reset_core_op(true), m_deactivation_time_accumulator(deactivation_time_accumulator) +{ +} + +VDeviceActivatedCoreOp::VDeviceActivatedCoreOp(VDeviceActivatedCoreOp &&other) noexcept : + ActivatedCoreOp(std::move(other)), + m_activated_network_groups(std::move(other.m_activated_network_groups)), + m_should_reset_core_op(std::exchange(other.m_should_reset_core_op, false)), + m_deactivation_time_accumulator(std::move(other.m_deactivation_time_accumulator)) +{ +} + + +Expected> VDeviceCoreOp::create(std::vector> core_ops, + CoreOpsSchedulerWeakPtr core_ops_scheduler, const std::string &hef_hash) +{ + auto status = HAILO_UNINITIALIZED; + + VDeviceCoreOp object(std::move(core_ops), core_ops_scheduler, hef_hash, status); + CHECK_SUCCESS_AS_EXPECTED(status); + + auto obj_ptr = make_shared_nothrow(std::move(object)); + CHECK_NOT_NULL_AS_EXPECTED(obj_ptr, HAILO_OUT_OF_HOST_MEMORY); + + return obj_ptr; +} + +Expected> VDeviceCoreOp::duplicate(std::shared_ptr other) +{ + auto status = HAILO_UNINITIALIZED; + auto copy = other->m_core_ops; + + VDeviceCoreOp object(std::move(copy), other->m_core_ops_scheduler, other->m_hef_hash, status); + CHECK_SUCCESS_AS_EXPECTED(status); + + auto obj_ptr = make_shared_nothrow(std::move(object)); + CHECK_NOT_NULL_AS_EXPECTED(obj_ptr, HAILO_OUT_OF_HOST_MEMORY); + + return obj_ptr; +} + + +VDeviceCoreOp::VDeviceCoreOp(std::vector> core_ops, + CoreOpsSchedulerWeakPtr core_ops_scheduler, const std::string &hef_hash, hailo_status &status) : + CoreOp(core_ops[0]->m_config_params, core_ops[0]->m_metadata, status), + m_core_ops(std::move(core_ops)), + m_core_ops_scheduler(core_ops_scheduler), + m_scheduler_handle(INVALID_CORE_OP_HANDLE), + m_multiplexer_handle(0), + m_multiplexer(), + m_hef_hash(hef_hash) +{} + +Expected VDeviceCoreOp::get_default_streams_interface() +{ + auto first_streams_interface = m_core_ops[0]->get_default_streams_interface(); + CHECK_EXPECTED(first_streams_interface); +#ifndef NDEBUG + // Check that all physical devices has the same interface + for (auto &core_op : m_core_ops) { + auto iface = core_op->get_default_streams_interface(); + CHECK_EXPECTED(iface); + CHECK_AS_EXPECTED(iface.value() == first_streams_interface.value(), HAILO_INTERNAL_FAILURE, + "Not all default stream interfaces are the same"); + } +#endif + return first_streams_interface; +} + +hailo_status VDeviceCoreOp::create_vdevice_streams_from_config_params(std::shared_ptr multiplexer, scheduler_core_op_handle_t scheduler_handle) +{ + // TODO - HRT-6931 - raise error on this case + if (((m_config_params.latency & HAILO_LATENCY_MEASURE) == HAILO_LATENCY_MEASURE) && (1 < m_core_ops.size())) { + LOGGER__WARNING("Latency measurement is not supported on more than 1 physical device."); + } + + m_multiplexer = multiplexer; + + for (const auto &stream_parameters_pair : m_config_params.stream_params_by_name) { + switch (stream_parameters_pair.second.direction) { + case HAILO_H2D_STREAM: + { + auto status = create_input_vdevice_stream_from_config_params(stream_parameters_pair.second, + stream_parameters_pair.first, multiplexer, scheduler_handle); + CHECK_SUCCESS(status); + } + break; + case HAILO_D2H_STREAM: + { + auto status = create_output_vdevice_stream_from_config_params(stream_parameters_pair.second, + stream_parameters_pair.first, multiplexer, scheduler_handle); + CHECK_SUCCESS(status); + } + break; + default: + LOGGER__ERROR("stream name {} direction is invalid.", stream_parameters_pair.first); + return HAILO_INVALID_ARGUMENT; + } + } + + for (const auto &input_stream : m_input_streams) { + if (HAILO_STREAM_INTERFACE_ETH == static_cast(*input_stream.second).get_interface()) { + continue; + } + auto expected_queue_size = static_cast(*input_stream.second).get_buffer_frames_size(); + CHECK_EXPECTED_AS_STATUS(expected_queue_size); + TRACE(CreateCoreOpInputStreamsTrace, "", name(), input_stream.first, (uint32_t)expected_queue_size.value()); + } + for (const auto &output_stream : m_output_streams) { + if ((hailo_format_order_t::HAILO_FORMAT_ORDER_HAILO_NMS == (static_cast(*output_stream.second).get_layer_info().format.order)) || + (HAILO_STREAM_INTERFACE_ETH == static_cast(*output_stream.second).get_interface())) { + continue; + } + auto expected_queue_size = static_cast(*output_stream.second).get_buffer_frames_size(); + CHECK_EXPECTED_AS_STATUS(expected_queue_size); + TRACE(CreateCoreOpOutputStreamsTrace, "", name(), output_stream.first, (uint32_t)expected_queue_size.value()); + } + + auto status = m_multiplexer->add_core_op_instance(m_multiplexer_handle, *this); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +hailo_status VDeviceCoreOp::create_input_vdevice_stream_from_config_params(const hailo_stream_parameters_t &stream_params, + const std::string &stream_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t scheduler_handle) +{ + auto edge_layer = get_layer_info(stream_name); + CHECK_EXPECTED_AS_STATUS(edge_layer); + + if (HailoRTCommon::is_vdma_stream_interface(stream_params.stream_interface)){ + std::vector> low_level_streams; + low_level_streams.reserve(m_core_ops.size()); + for (auto &core_op : m_core_ops) { + auto stream = core_op->get_input_stream_by_name(stream_name); + CHECK(stream, HAILO_INTERNAL_FAILURE); + low_level_streams.emplace_back(dynamic_cast(stream.release().get())); + } + auto input_stream = InputVDeviceBaseStream::create(std::move(low_level_streams), edge_layer.value(), + scheduler_handle, m_core_op_activated_event, m_core_ops_scheduler); + CHECK_EXPECTED_AS_STATUS(input_stream); + auto input_stream_wrapper = VDeviceInputStreamMultiplexerWrapper::create(input_stream.release(), edge_layer->network_name, multiplexer, scheduler_handle); + CHECK_EXPECTED_AS_STATUS(input_stream_wrapper); + m_input_streams.insert(make_pair(stream_name, input_stream_wrapper.release())); + } else { + assert(1 == m_core_ops.size()); + auto stream = m_core_ops[0]->get_input_stream_by_name(stream_name); + CHECK(stream, HAILO_INTERNAL_FAILURE); + assert(1 == m_core_ops.size()); + assert(contains(m_core_ops[0]->m_input_streams, stream_name)); + m_input_streams.insert(make_pair(stream_name, m_core_ops[0]->m_input_streams.at(stream_name))); + } + + return HAILO_SUCCESS; +} + +hailo_status VDeviceCoreOp::create_output_vdevice_stream_from_config_params(const hailo_stream_parameters_t &stream_params, + const std::string &stream_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t scheduler_handle) +{ + auto edge_layer = get_layer_info(stream_name); + CHECK_EXPECTED_AS_STATUS(edge_layer); + + if (HailoRTCommon::is_vdma_stream_interface(stream_params.stream_interface)) { + std::vector> low_level_streams; + low_level_streams.reserve(m_core_ops.size()); + for (auto &core_op : m_core_ops) { + auto stream = core_op->get_output_stream_by_name(stream_name); + CHECK(stream, HAILO_INTERNAL_FAILURE); + low_level_streams.emplace_back(dynamic_cast(stream.release().get())); + } + auto output_stream = OutputVDeviceBaseStream::create(std::move(low_level_streams), edge_layer.value(), + scheduler_handle, m_core_op_activated_event, m_core_ops_scheduler); + CHECK_EXPECTED_AS_STATUS(output_stream); + auto output_stream_wrapper = VDeviceOutputStreamMultiplexerWrapper::create(output_stream.release(), edge_layer->network_name, multiplexer, scheduler_handle); + CHECK_EXPECTED_AS_STATUS(output_stream_wrapper); + m_output_streams.insert(make_pair(stream_name, output_stream_wrapper.release())); + } else { + assert(1 == m_core_ops.size()); + assert(contains(m_core_ops[0]->m_output_streams, stream_name)); + m_output_streams.insert(make_pair(stream_name, m_core_ops[0]->m_output_streams.at(stream_name))); + } + + return HAILO_SUCCESS; +} + +hailo_status VDeviceCoreOp::create_vdevice_streams_from_duplicate(std::shared_ptr other) +{ + // TODO - HRT-6931 - raise error on this case + if (((m_config_params.latency & HAILO_LATENCY_MEASURE) == HAILO_LATENCY_MEASURE) && (1 < m_core_ops.size())) { + LOGGER__WARNING("Latency measurement is not supported on more than 1 physical device."); + } + + m_multiplexer = other->m_multiplexer; + m_multiplexer_handle = other->multiplexer_duplicates_count() + 1; + + for (auto &name_stream_pair : other->m_input_streams) { + auto input_stream = static_cast(name_stream_pair.second.get()); + auto copy = input_stream->clone(m_multiplexer_handle); + CHECK_EXPECTED_AS_STATUS(copy); + + m_input_streams.insert(make_pair(name_stream_pair.first, copy.release())); + } + + for (auto &name_stream_pair : other->m_output_streams) { + auto output_stream = static_cast(name_stream_pair.second.get()); + auto copy = output_stream->clone(m_multiplexer_handle); + CHECK_EXPECTED_AS_STATUS(copy); + + m_output_streams.insert(make_pair(name_stream_pair.first, copy.release())); + } + + auto status = other->m_multiplexer->add_core_op_instance(m_multiplexer_handle, *this); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +void VDeviceCoreOp::set_core_op_handle(scheduler_core_op_handle_t handle) +{ + m_scheduler_handle = handle; +} + +scheduler_core_op_handle_t VDeviceCoreOp::core_op_handle() const +{ + return m_scheduler_handle; +} + +bool VDeviceCoreOp::is_scheduled() const +{ + return !m_core_ops_scheduler.expired(); +}; + +hailo_status VDeviceCoreOp::set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) +{ + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INVALID_OPERATION, + "Cannot set scheduler timeout for core-op {}, as it is configured on a vdevice which does not have scheduling enabled", name()); + if (network_name != HailoRTDefaults::get_network_name(name())) { + CHECK(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Setting scheduler timeout for a specific network is currently not supported"); + } + auto status = core_ops_scheduler->set_timeout(m_scheduler_handle, timeout, network_name); + CHECK_SUCCESS(status); + return HAILO_SUCCESS; +} + +hailo_status VDeviceCoreOp::set_scheduler_threshold(uint32_t threshold, const std::string &network_name) +{ + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INVALID_OPERATION, + "Cannot set scheduler threshold for core-op {}, as it is configured on a vdevice which does not have scheduling enabled", name()); + if (network_name != HailoRTDefaults::get_network_name(name())) { + CHECK(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Setting scheduler threshold for a specific network is currently not supported"); + } + auto status = core_ops_scheduler->set_threshold(m_scheduler_handle, threshold, network_name); + CHECK_SUCCESS(status); + return HAILO_SUCCESS; +} + +hailo_status VDeviceCoreOp::set_scheduler_priority(uint8_t priority, const std::string &network_name) +{ + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INVALID_OPERATION, + "Cannot set scheduler priority for core-op {}, as it is configured on a vdevice which does not have scheduling enabled", name()); + if (network_name != HailoRTDefaults::get_network_name(name())) { + CHECK(network_name.empty(), HAILO_NOT_IMPLEMENTED, "Setting scheduler priority for a specific network is currently not supported"); + } + auto status = core_ops_scheduler->set_priority(m_scheduler_handle, priority, network_name); + CHECK_SUCCESS(status); + return HAILO_SUCCESS; +} + +Expected> VDeviceCoreOp::get_latency_meters() +{ + return m_core_ops[0]->get_latency_meters(); +} + +Expected VDeviceCoreOp::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) +{ + CHECK_AS_EXPECTED(1 == m_core_ops.size(), HAILO_INVALID_OPERATION, + "get_boundary_vdma_channel_by_stream_name function is not supported on more than 1 physical device."); + + return m_core_ops[0]->get_boundary_vdma_channel_by_stream_name(stream_name); +} + +void VDeviceCoreOp::set_vstreams_multiplexer_callbacks(std::vector &output_vstreams) +{ + if (nullptr == m_multiplexer) { + return; + } + + m_multiplexer->set_output_vstreams_names(m_multiplexer_handle, output_vstreams); + + for (auto &vstream : output_vstreams) { + static_cast(*vstream.m_vstream).set_on_vstream_cant_read_callback([this, name = vstream.name()] () { + m_multiplexer->set_can_output_vstream_read(m_multiplexer_handle, name, false); + }); + static_cast(*vstream.m_vstream).set_on_vstream_can_read_callback([this, name = vstream.name()] () { + m_multiplexer->set_can_output_vstream_read(m_multiplexer_handle, name, true); + }); + } +} + +Expected> VDeviceCoreOp::get_core_op_by_device_index(uint32_t device_index) +{ + CHECK_AS_EXPECTED(device_index < m_core_ops.size(), HAILO_INVALID_ARGUMENT); + auto core_op = std::dynamic_pointer_cast(m_core_ops[device_index]); + CHECK_NOT_NULL_AS_EXPECTED(core_op, HAILO_INTERNAL_FAILURE); + return core_op; +} + +Expected> VDeviceCoreOp::create_activated_network_group( + const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size, + bool resume_pending_stream_transfers) +{ + auto start_time = std::chrono::steady_clock::now(); + + CHECK_AS_EXPECTED(!m_core_ops_scheduler.lock(), HAILO_INVALID_OPERATION, + "Manually activating a core-op is not allowed when the core-op scheduler is active!"); + + auto res = VDeviceActivatedCoreOp::create(m_core_ops, m_input_streams, m_output_streams, + network_group_params, m_core_op_activated_event, dynamic_batch_size, m_deactivation_time_accumulator, + resume_pending_stream_transfers); + const auto elapsed_time_ms = std::chrono::duration( + std::chrono::steady_clock::now() - start_time).count(); + CHECK_EXPECTED(res); + + LOGGER__INFO("Activating {} on VDevice took {} milliseconds. Note that the function is asynchronous and" + " thus the network is not fully activated yet.", name(), elapsed_time_ms); + m_activation_time_accumulator->add_data_point(elapsed_time_ms); + + return res; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdevice/vdevice_core_op.hpp b/hailort/libhailort/src/vdevice/vdevice_core_op.hpp new file mode 100644 index 0000000..6fb1837 --- /dev/null +++ b/hailort/libhailort/src/vdevice/vdevice_core_op.hpp @@ -0,0 +1,174 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdevice_core_op.hpp + * @brief Class declaration for VDeviceCoreOp, which is used to support multiple CoreOps objects, + * that encapsulate the same actual CoreOp. + **/ + +#ifndef _HAILO_VDEVICE_CORE_OP_HPP_ +#define _HAILO_VDEVICE_CORE_OP_HPP_ + +#include "hailo/hailort.h" +#include "common/utils.hpp" +#include "hailo/network_group.hpp" +#include "hailo/vstream.hpp" + +#include "vdevice/scheduler/network_group_scheduler.hpp" +#include "vdevice/pipeline_multiplexer.hpp" + +#include + + +namespace hailort +{ + +class VDeviceActivatedCoreOp : public ActivatedCoreOp +{ +public: + static Expected> create(std::vector> &core_ops, + std::map> &input_streams, + std::map> &output_streams, + const hailo_activate_network_group_params_t &network_group_params, EventPtr core_op_activated_event, + uint16_t dynamic_batch_size, AccumulatorPtr deactivation_time_accumulator, + bool resume_pending_stream_transfers); + + virtual ~VDeviceActivatedCoreOp() + { + if (!m_should_reset_core_op) { + return; + } + const auto start_time = std::chrono::steady_clock::now(); + + m_core_op_activated_event->reset(); + m_activated_network_groups.clear(); + + const auto elapsed_time_ms = std::chrono::duration( + std::chrono::steady_clock::now() - start_time).count(); + LOGGER__INFO("Deactivating took {} ms", elapsed_time_ms); + m_deactivation_time_accumulator->add_data_point(elapsed_time_ms); + } + + VDeviceActivatedCoreOp(const VDeviceActivatedCoreOp &other) = delete; + VDeviceActivatedCoreOp &operator=(const VDeviceActivatedCoreOp &other) = delete; + VDeviceActivatedCoreOp &operator=(VDeviceActivatedCoreOp &&other) = delete; + VDeviceActivatedCoreOp(VDeviceActivatedCoreOp &&other) noexcept; + + virtual const std::string &get_network_group_name() const override; + virtual Expected get_intermediate_buffer(const IntermediateBufferKey &key) override; + virtual hailo_status set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) override; + +private: + VDeviceActivatedCoreOp( + std::vector> &&activated_network_groups, + std::map> &input_streams, + std::map> &output_streams, + const hailo_activate_network_group_params_t &network_group_params, EventPtr core_op_activated_event, + AccumulatorPtr deactivation_time_accumulator, hailo_status &status); + + std::vector> m_activated_network_groups; + bool m_should_reset_core_op; + AccumulatorPtr m_deactivation_time_accumulator; +}; + +class VDeviceCoreOp : public CoreOp +{ +public: + static Expected> create(std::vector> core_ops, + CoreOpsSchedulerWeakPtr core_ops_scheduler, const std::string &hef_hash); + + static Expected> duplicate(std::shared_ptr other); + + virtual ~VDeviceCoreOp() = default; + VDeviceCoreOp(const VDeviceCoreOp &other) = delete; + VDeviceCoreOp &operator=(const VDeviceCoreOp &other) = delete; + VDeviceCoreOp &operator=(VDeviceCoreOp &&other) = delete; + VDeviceCoreOp(VDeviceCoreOp &&other) = default; + + hailo_status create_vdevice_streams_from_config_params(std::shared_ptr multiplexer, + scheduler_core_op_handle_t scheduler_handle); + hailo_status create_input_vdevice_stream_from_config_params( + const hailo_stream_parameters_t &stream_params, const std::string &stream_name, + std::shared_ptr multiplexer, scheduler_core_op_handle_t scheduler_handle); + hailo_status create_output_vdevice_stream_from_config_params( + const hailo_stream_parameters_t &stream_params, const std::string &stream_name, + std::shared_ptr multiplexer, scheduler_core_op_handle_t scheduler_handle); + + hailo_status create_vdevice_streams_from_duplicate(std::shared_ptr other); + + bool equals(const Hef &hef, const std::pair ¶ms_pair) + { + if ((params_pair.first == name()) && (hef.hash() == m_hef_hash)) { + if ((params_pair.second.batch_size == m_config_params.batch_size) && + (params_pair.second.power_mode == m_config_params.power_mode)) { + return true; + } + LOGGER__INFO("The network group: {} was already configured to the device with different params." + " To use the Stream Multiplexer configure the network with the same params.", name()); + } + + return false; + } + + uint32_t multiplexer_duplicates_count() + { + assert(m_multiplexer->instances_count() > 0); + return static_cast(m_multiplexer->instances_count() - 1); + } + + virtual Expected get_default_streams_interface() override; + + virtual Expected> get_latency_meters() override; + virtual Expected get_boundary_vdma_channel_by_stream_name( + const std::string &stream_name) override; + + void set_core_op_handle(scheduler_core_op_handle_t handle); + scheduler_core_op_handle_t core_op_handle() const; + virtual bool is_scheduled() const override; + virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) override; + virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) override; + virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name) override; + + void set_vstreams_multiplexer_callbacks(std::vector &output_vstreams) override; + + virtual hailo_status wait_for_activation(const std::chrono::milliseconds &timeout) override + { + CHECK(!m_core_ops_scheduler.lock(), HAILO_INVALID_OPERATION, + "Waiting for core-op activation is not allowed when the core-ops scheduler is active!"); + + return m_core_op_activated_event->wait(timeout); + } + + virtual hailo_status activate_impl(uint16_t /*dynamic_batch_size*/, bool /* resume_pending_stream_transfers */) override + { + return HAILO_INTERNAL_FAILURE; + } + + virtual hailo_status deactivate_impl(bool /* keep_nn_config_during_reset */) override + { + return HAILO_INTERNAL_FAILURE; + } + + virtual Expected> create_activated_network_group( + const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size, + bool resume_pending_stream_transfers) override; + + Expected> get_core_op_by_device_index(uint32_t device_index); + +private: + VDeviceCoreOp(std::vector> core_ops, CoreOpsSchedulerWeakPtr core_ops_scheduler, + const std::string &hef_hash, hailo_status &status); + + std::vector> m_core_ops; + CoreOpsSchedulerWeakPtr m_core_ops_scheduler; + scheduler_core_op_handle_t m_scheduler_handle; + multiplexer_core_op_handle_t m_multiplexer_handle; + std::shared_ptr m_multiplexer; + std::string m_hef_hash; +}; + +} + +#endif /* _HAILO_VDEVICE_CORE_OP_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdevice_internal.hpp b/hailort/libhailort/src/vdevice/vdevice_internal.hpp similarity index 74% rename from hailort/libhailort/src/vdevice_internal.hpp rename to hailort/libhailort/src/vdevice/vdevice_internal.hpp index 42c20f4..22c2948 100644 --- a/hailort/libhailort/src/vdevice_internal.hpp +++ b/hailort/libhailort/src/vdevice/vdevice_internal.hpp @@ -23,15 +23,17 @@ #include "hailo/hailort.h" #include "hailo/vdevice.hpp" -#include "vdma_device.hpp" -#include "context_switch/multi_context/vdma_config_manager.hpp" -#include "context_switch/vdevice_network_group.hpp" -#include "network_group_scheduler.hpp" + +#include "vdma/vdma_device.hpp" +#include "vdma/vdma_config_manager.hpp" +#include "vdevice/vdevice_core_op.hpp" +#include "vdevice/scheduler/network_group_scheduler.hpp" #ifdef HAILO_SUPPORT_MULTI_PROCESS -#include "hailort_rpc_client.hpp" +#include "service/hailort_rpc_client.hpp" #endif // HAILO_SUPPORT_MULTI_PROCESS + namespace hailort { @@ -69,33 +71,30 @@ public: return device_ids; } - const NetworkGroupSchedulerPtr &network_group_scheduler() + const CoreOpsSchedulerPtr &core_ops_scheduler() { - return m_network_group_scheduler; + return m_core_ops_scheduler; } // Currently only homogeneous vDevice is allow (= all devices are from the same type) virtual Expected get_default_streams_interface() const override; - // TODO: Remove when feature becomes 'released' - static bool enable_multi_device_schedeulr() - { - auto enable_multi_device_schedeulr_env = std::getenv(HAILO_ENABLE_MULTI_DEVICE_SCHEDULER); - return ((nullptr != enable_multi_device_schedeulr_env) && - (strnlen(enable_multi_device_schedeulr_env, 2) == 1) && (strncmp(enable_multi_device_schedeulr_env, "1", 1) == 0)); - } + static hailo_status validate_params(const hailo_vdevice_params_t ¶ms); private: - VDeviceBase(std::vector> &&devices, NetworkGroupSchedulerPtr network_group_scheduler) : - m_devices(std::move(devices)), m_network_group_scheduler(network_group_scheduler), m_network_groups({}) + VDeviceBase(std::vector> &&devices, CoreOpsSchedulerPtr core_ops_scheduler) : + m_devices(std::move(devices)), m_core_ops_scheduler(core_ops_scheduler) {} - static Expected>> create_devices(const hailo_vdevice_params_t ¶ms); + static Expected>> create_devices(const hailo_vdevice_params_t ¶ms); static Expected> get_device_ids(const hailo_vdevice_params_t ¶ms); + Expected create_local_config_params(Hef &hef, const NetworkGroupsParamsMap &configure_params); + Expected> create_vdevice_network_group(Hef &hef, const std::pair ¶ms); - std::vector> m_devices; - NetworkGroupSchedulerPtr m_network_group_scheduler; - std::vector> m_network_groups; + std::vector> m_devices; + CoreOpsSchedulerPtr m_core_ops_scheduler; + std::vector> m_vdevice_core_ops; + std::vector> m_network_groups; // TODO: HRT-9547 - Remove when ConfiguredNetworkGroup will be kept in global context std::mutex m_mutex; }; @@ -120,11 +119,18 @@ public: Expected> get_physical_devices_ids() const override; Expected get_default_streams_interface() const override; + virtual hailo_status before_fork() override; + virtual hailo_status after_fork_in_parent() override; + virtual hailo_status after_fork_in_child() override; + private: VDeviceClient(std::unique_ptr client, uint32_t handle); + hailo_status create_client(); + std::unique_ptr m_client; uint32_t m_handle; + std::vector> m_network_groups; }; #endif // HAILO_SUPPORT_MULTI_PROCESS diff --git a/hailort/libhailort/src/vdevice_native_stream.hpp b/hailort/libhailort/src/vdevice/vdevice_native_stream.hpp similarity index 74% rename from hailort/libhailort/src/vdevice_native_stream.hpp rename to hailort/libhailort/src/vdevice/vdevice_native_stream.hpp index f1b9ec7..5ddfe8c 100644 --- a/hailort/libhailort/src/vdevice_native_stream.hpp +++ b/hailort/libhailort/src/vdevice/vdevice_native_stream.hpp @@ -11,26 +11,24 @@ #ifndef HAILO_VDEVICE_NATIVE_STREAM_HPP_ #define HAILO_VDEVICE_NATIVE_STREAM_HPP_ -#include "stream_internal.hpp" #include "hailo/hailort.h" -#include "vdevice_stream.hpp" #include "hailo/expected.hpp" +#include "stream_common/stream_internal.hpp" +#include "vdevice_stream.hpp" + + namespace hailort { class InputVDeviceNativeStream : public InputVDeviceBaseStream { public: - InputVDeviceNativeStream(InputVDeviceNativeStream &&other) : - InputVDeviceBaseStream(std::move(other)) - {} - - explicit InputVDeviceNativeStream( + InputVDeviceNativeStream( std::vector> &&streams, - EventPtr &&network_group_activated_event, + EventPtr &&core_op_activated_event, const LayerInfo &layer_info, hailo_status &status) : - InputVDeviceBaseStream(std::move(streams), std::move(network_group_activated_event), layer_info, status) + InputVDeviceBaseStream(std::move(streams), std::move(core_op_activated_event), layer_info, status) {} virtual hailo_status abort() override; @@ -44,16 +42,12 @@ protected: class OutputVDeviceNativeStream : public OutputVDeviceBaseStream { public: - OutputVDeviceNativeStream(OutputVDeviceNativeStream &&other) : - OutputVDeviceBaseStream(std::move(other)) - {} - - explicit OutputVDeviceNativeStream( + OutputVDeviceNativeStream( std::vector> &&streams, const LayerInfo &layer_info, - EventPtr &&network_group_activated_event, + EventPtr &&core_op_activated_event, hailo_status &status) : - OutputVDeviceBaseStream(std::move(streams), layer_info, std::move(network_group_activated_event), status) + OutputVDeviceBaseStream(std::move(streams), layer_info, std::move(core_op_activated_event), status) {} virtual hailo_status abort() override; diff --git a/hailort/libhailort/src/vdevice_stream.cpp b/hailort/libhailort/src/vdevice/vdevice_stream.cpp similarity index 77% rename from hailort/libhailort/src/vdevice_stream.cpp rename to hailort/libhailort/src/vdevice/vdevice_stream.cpp index 7b9a792..f50ec24 100644 --- a/hailort/libhailort/src/vdevice_stream.cpp +++ b/hailort/libhailort/src/vdevice/vdevice_stream.cpp @@ -9,19 +9,22 @@ * TODO: doc **/ -#include - #include "hailo/hailort.h" -#include "common/utils.hpp" #include "hailo/stream.hpp" #include "hailo/hef.hpp" #include "hailo/hailort_common.hpp" -#include "tracer_macros.hpp" -#include "vdevice_stream.hpp" -#include "scheduled_stream.hpp" -#include "vdevice_native_stream.hpp" -#include "context_switch/multi_context/resource_manager.hpp" -#include "multi_device_scheduled_stream.hpp" + +#include "common/utils.hpp" + +#include "utils/profiler/tracer_macros.hpp" +#include "vdevice/vdevice_stream.hpp" +#include "vdevice/vdevice_native_stream.hpp" +#include "vdevice/scheduler/multi_device_scheduled_stream.hpp" +#include "vdevice/scheduler/scheduled_stream.hpp" +#include "core_op/resource_manager/resource_manager.hpp" + +#include + namespace hailort { @@ -50,10 +53,10 @@ InputVDeviceBaseStream::~InputVDeviceBaseStream() } } -hailo_status InputVDeviceBaseStream::activate_stream(uint16_t dynamic_batch_size) +hailo_status InputVDeviceBaseStream::activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) { for (auto &stream : m_streams) { - auto status = stream.get().activate_stream(dynamic_batch_size); + auto status = stream.get().activate_stream(dynamic_batch_size, resume_pending_stream_transfers); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to activate input stream. (device: {})", stream.get().get_dev_id()); deactivate_stream(); @@ -104,32 +107,33 @@ Expected InputVDeviceBaseStream::get_pending_frames_count() const } Expected> InputVDeviceBaseStream::create(std::vector> &&low_level_streams, - const LayerInfo &edge_layer, const scheduler_ng_handle_t &network_group_handle, - EventPtr network_group_activated_event, NetworkGroupSchedulerWeakPtr network_group_scheduler) + const LayerInfo &edge_layer, const scheduler_core_op_handle_t &core_op_handle, + EventPtr core_op_activated_event, CoreOpsSchedulerWeakPtr core_ops_scheduler) { assert(0 < low_level_streams.size()); auto status = HAILO_UNINITIALIZED; std::unique_ptr local_vdevice_stream; - if (network_group_scheduler.lock()) { + if (core_ops_scheduler.lock()) { if (1 < low_level_streams.size()) { - const auto batch_size = low_level_streams[0].get().get_dynamic_batch_size(); + auto buffer_frame_size = low_level_streams[0].get().get_buffer_frames_size(); + CHECK_EXPECTED(buffer_frame_size); auto frame_size = low_level_streams[0].get().get_frame_size(); - auto buffers_queue_ptr = BuffersQueue::create_unique(frame_size, (low_level_streams.size() * batch_size)); + auto buffers_queue_ptr = BuffersQueue::create_unique(frame_size, (low_level_streams.size() * buffer_frame_size.value())); CHECK_EXPECTED(buffers_queue_ptr); local_vdevice_stream = make_unique_nothrow(std::move(low_level_streams), - network_group_handle, std::move(network_group_activated_event), edge_layer, - network_group_scheduler, buffers_queue_ptr.release(), status); + core_op_handle, std::move(core_op_activated_event), edge_layer, + core_ops_scheduler, buffers_queue_ptr.release(), status); } else { local_vdevice_stream = make_unique_nothrow(std::move(low_level_streams), - network_group_handle, std::move(network_group_activated_event), edge_layer, - network_group_scheduler, status); + core_op_handle, std::move(core_op_activated_event), edge_layer, + core_ops_scheduler, status); } } else { local_vdevice_stream = make_unique_nothrow(std::move(low_level_streams), - std::move(network_group_activated_event), edge_layer,status); + std::move(core_op_activated_event), edge_layer,status); } CHECK_AS_EXPECTED((nullptr != local_vdevice_stream), HAILO_OUT_OF_HOST_MEMORY); @@ -174,7 +178,7 @@ hailo_status InputVDeviceBaseStream::flush() Expected ScheduledInputStream::sync_write_raw_buffer(const MemoryView &buffer, const std::function &should_cancel) { - return sync_write_raw_buffer_impl(buffer, m_network_group_handle, should_cancel); + return sync_write_raw_buffer_impl(buffer, m_core_op_handle, should_cancel); } Expected InputVDeviceNativeStream::sync_write_raw_buffer(const MemoryView &buffer, const std::function &should_cancel) @@ -198,43 +202,42 @@ Expected InputVDeviceNativeStream::sync_write_raw_buffer(const MemoryVie return written_bytes; } -Expected ScheduledInputStream::sync_write_raw_buffer_impl(const MemoryView &buffer, scheduler_ng_handle_t network_group_handle, +Expected ScheduledInputStream::sync_write_raw_buffer_impl(const MemoryView &buffer, scheduler_core_op_handle_t core_op_handle, const std::function &should_cancel) { - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK_AS_EXPECTED(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK_AS_EXPECTED(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto status = network_group_scheduler->wait_for_write(network_group_handle, name(), get_timeout(), should_cancel); + auto status = core_ops_scheduler->wait_for_write(core_op_handle, name(), get_timeout(), should_cancel); if (HAILO_STREAM_ABORTED_BY_USER == status) { LOGGER__INFO("Write to stream was aborted."); return make_unexpected(status); } CHECK_SUCCESS_AS_EXPECTED(status); - TRACE(WriteFrameTrace, "", network_group_handle, m_stream_info.name); + TRACE(WriteFrameTrace, "", core_op_handle, m_stream_info.name); assert(1 == m_streams.size()); status = m_streams[0].get().write_buffer_only(buffer, should_cancel); + + auto write_finish_status = core_ops_scheduler->signal_write_finish(core_op_handle, name(), status != HAILO_SUCCESS); if (HAILO_SUCCESS != status) { LOGGER__INFO("Write to stream has failed! status = {}", status); - network_group_scheduler->mark_failed_write(network_group_handle, name()); return make_unexpected(status); } - status = network_group_scheduler->signal_write_finish(network_group_handle, name()); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - return make_unexpected(status); + if (HAILO_STREAM_ABORTED_BY_USER == write_finish_status) { + return make_unexpected(write_finish_status); } - CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_SUCCESS_AS_EXPECTED(write_finish_status); auto written_bytes = buffer.size(); - return written_bytes; } hailo_status ScheduledInputStream::abort() { - return abort_impl(m_network_group_handle); + return abort_impl(m_core_op_handle); } hailo_status InputVDeviceNativeStream::abort() @@ -251,7 +254,7 @@ hailo_status InputVDeviceNativeStream::abort() return status; } -hailo_status ScheduledInputStream::abort_impl(scheduler_ng_handle_t network_group_handle) +hailo_status ScheduledInputStream::abort_impl(scheduler_core_op_handle_t core_op_handle) { auto status = HAILO_SUCCESS; // Best effort assert(1 == m_streams.size()); @@ -261,12 +264,12 @@ hailo_status ScheduledInputStream::abort_impl(scheduler_ng_handle_t network_grou status = abort_status; } - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto disable_status = network_group_scheduler->disable_stream(network_group_handle, name()); + auto disable_status = core_ops_scheduler->disable_stream(core_op_handle, name()); if (HAILO_SUCCESS != disable_status) { - LOGGER__ERROR("Failed to disable stream in the network group scheduler. (status: {})", disable_status); + LOGGER__ERROR("Failed to disable stream in the core-op scheduler. (status: {})", disable_status); status = disable_status; } @@ -275,7 +278,7 @@ hailo_status ScheduledInputStream::abort_impl(scheduler_ng_handle_t network_grou hailo_status ScheduledInputStream::clear_abort() { - return clear_abort_impl(m_network_group_handle); + return clear_abort_impl(m_core_op_handle); } hailo_status InputVDeviceNativeStream::clear_abort() @@ -292,7 +295,7 @@ hailo_status InputVDeviceNativeStream::clear_abort() return status; } -hailo_status ScheduledInputStream::clear_abort_impl(scheduler_ng_handle_t network_group_handle) +hailo_status ScheduledInputStream::clear_abort_impl(scheduler_core_op_handle_t core_op_handle) { auto status = HAILO_SUCCESS; // Best effort assert(1 == m_streams.size()); @@ -302,12 +305,12 @@ hailo_status ScheduledInputStream::clear_abort_impl(scheduler_ng_handle_t networ status = clear_abort_status; } - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto enable_status = network_group_scheduler->enable_stream(network_group_handle, name()); + auto enable_status = core_ops_scheduler->enable_stream(core_op_handle, name()); if (HAILO_SUCCESS != enable_status) { - LOGGER__ERROR("Failed to enable stream in the network group scheduler. (status: {})", enable_status); + LOGGER__ERROR("Failed to enable stream in the core-op scheduler. (status: {})", enable_status); status = enable_status; } @@ -338,10 +341,10 @@ OutputVDeviceBaseStream::~OutputVDeviceBaseStream() } } -hailo_status OutputVDeviceBaseStream::activate_stream(uint16_t dynamic_batch_size) +hailo_status OutputVDeviceBaseStream::activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) { for (auto &stream : m_streams) { - auto status = stream.get().activate_stream(dynamic_batch_size); + auto status = stream.get().activate_stream(dynamic_batch_size, resume_pending_stream_transfers); if (HAILO_SUCCESS != status) { LOGGER__ERROR("Failed to activate output stream. (device: {})", stream.get().get_dev_id()); deactivate_stream(); @@ -366,7 +369,7 @@ Expected OutputVDeviceBaseStream::sync_read_raw_buffer(MemoryView &/*buf hailo_status ScheduledOutputStream::read(MemoryView buffer) { - return read_impl(buffer, m_network_group_handle); + return read_impl(buffer, m_core_op_handle); } hailo_status OutputVDeviceNativeStream::read(MemoryView buffer) @@ -386,26 +389,26 @@ hailo_status OutputVDeviceNativeStream::read(MemoryView buffer) return HAILO_SUCCESS; } -hailo_status ScheduledOutputStream::read_impl(MemoryView buffer, scheduler_ng_handle_t network_group_handle) +hailo_status ScheduledOutputStream::read_impl(MemoryView buffer, scheduler_core_op_handle_t core_op_handle) { - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto device_id = network_group_scheduler->wait_for_read(network_group_handle, name(), get_timeout()); + auto device_id = core_ops_scheduler->wait_for_read(core_op_handle, name(), get_timeout()); if (HAILO_STREAM_ABORTED_BY_USER == device_id.status()) { LOGGER__INFO("Read from stream was aborted."); return device_id.status(); } CHECK_EXPECTED_AS_STATUS(device_id); - TRACE(ReadFrameTrace, "", network_group_handle, m_stream_info.name); + TRACE(ReadFrameTrace, "", core_op_handle, m_stream_info.name); auto status = m_streams[device_id.value()].get().read(buffer); if (HAILO_SUCCESS != status) { LOGGER__INFO("Read from stream has failed! status = {}", status); return status; } - status = network_group_scheduler->signal_read_finish(network_group_handle, name(), device_id.value()); + status = core_ops_scheduler->signal_read_finish(core_op_handle, name(), device_id.value()); if (HAILO_STREAM_ABORTED_BY_USER == status) { return status; } @@ -415,19 +418,19 @@ hailo_status ScheduledOutputStream::read_impl(MemoryView buffer, scheduler_ng_ha } Expected> OutputVDeviceBaseStream::create(std::vector> &&low_level_streams, - const LayerInfo &edge_layer, const scheduler_ng_handle_t &network_group_handle, EventPtr network_group_activated_event, - NetworkGroupSchedulerWeakPtr network_group_scheduler) + const LayerInfo &edge_layer, const scheduler_core_op_handle_t &core_op_handle, EventPtr core_op_activated_event, + CoreOpsSchedulerWeakPtr core_ops_scheduler) { assert(0 < low_level_streams.size()); auto status = HAILO_UNINITIALIZED; std::unique_ptr local_vdevice_stream; - if (network_group_scheduler.lock()) { - local_vdevice_stream = make_unique_nothrow(std::move(low_level_streams), network_group_handle, - edge_layer, std::move(network_group_activated_event), network_group_scheduler, status); + if (core_ops_scheduler.lock()) { + local_vdevice_stream = make_unique_nothrow(std::move(low_level_streams), core_op_handle, + edge_layer, std::move(core_op_activated_event), core_ops_scheduler, status); } else { local_vdevice_stream = make_unique_nothrow(std::move(low_level_streams), edge_layer, - std::move(network_group_activated_event), status); + std::move(core_op_activated_event), status); } CHECK_AS_EXPECTED((nullptr != local_vdevice_stream), HAILO_OUT_OF_HOST_MEMORY); @@ -459,7 +462,7 @@ hailo_stream_interface_t OutputVDeviceBaseStream::get_interface() const hailo_status ScheduledOutputStream::abort() { - return abort_impl(m_network_group_handle); + return abort_impl(m_core_op_handle); } hailo_status OutputVDeviceNativeStream::abort() @@ -476,7 +479,7 @@ hailo_status OutputVDeviceNativeStream::abort() return status; } -hailo_status ScheduledOutputStream::abort_impl(scheduler_ng_handle_t network_group_handle) +hailo_status ScheduledOutputStream::abort_impl(scheduler_core_op_handle_t core_op_handle) { auto status = HAILO_SUCCESS; // Best effort for (auto& stream : m_streams) { @@ -487,12 +490,12 @@ hailo_status ScheduledOutputStream::abort_impl(scheduler_ng_handle_t network_gro } } - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto disable_status = network_group_scheduler->disable_stream(network_group_handle, name()); + auto disable_status = core_ops_scheduler->disable_stream(core_op_handle, name()); if (HAILO_SUCCESS != disable_status) { - LOGGER__ERROR("Failed to disable stream in the network group scheduler. (status: {})", disable_status); + LOGGER__ERROR("Failed to disable stream in the core-op scheduler. (status: {})", disable_status); status = disable_status; } @@ -501,7 +504,7 @@ hailo_status ScheduledOutputStream::abort_impl(scheduler_ng_handle_t network_gro hailo_status ScheduledOutputStream::clear_abort() { - return clear_abort_impl(m_network_group_handle); + return clear_abort_impl(m_core_op_handle); } hailo_status OutputVDeviceNativeStream::clear_abort() @@ -518,7 +521,7 @@ hailo_status OutputVDeviceNativeStream::clear_abort() return status; } -hailo_status ScheduledOutputStream::clear_abort_impl(scheduler_ng_handle_t network_group_handle) +hailo_status ScheduledOutputStream::clear_abort_impl(scheduler_core_op_handle_t core_op_handle) { auto status = HAILO_SUCCESS; // Best effort for (auto& stream : m_streams) { @@ -529,12 +532,12 @@ hailo_status ScheduledOutputStream::clear_abort_impl(scheduler_ng_handle_t netwo } } - auto network_group_scheduler = m_network_group_scheduler.lock(); - CHECK(network_group_scheduler, HAILO_INTERNAL_FAILURE); + auto core_ops_scheduler = m_core_ops_scheduler.lock(); + CHECK(core_ops_scheduler, HAILO_INTERNAL_FAILURE); - auto enable_status = network_group_scheduler->enable_stream(network_group_handle, name()); + auto enable_status = core_ops_scheduler->enable_stream(core_op_handle, name()); if (HAILO_SUCCESS != enable_status) { - LOGGER__ERROR("Failed to enable stream in the network group scheduler. (status: {})", enable_status); + LOGGER__ERROR("Failed to enable stream in the core-op scheduler. (status: {})", enable_status); status = enable_status; } diff --git a/hailort/libhailort/src/vdevice_stream.hpp b/hailort/libhailort/src/vdevice/vdevice_stream.hpp similarity index 74% rename from hailort/libhailort/src/vdevice_stream.hpp rename to hailort/libhailort/src/vdevice/vdevice_stream.hpp index 2d113e3..e1aa294 100644 --- a/hailort/libhailort/src/vdevice_stream.hpp +++ b/hailort/libhailort/src/vdevice/vdevice_stream.hpp @@ -22,13 +22,15 @@ #ifndef HAILO_VDEVICE_STREAM_HPP_ #define HAILO_VDEVICE_STREAM_HPP_ -#include "stream_internal.hpp" #include "hailo/hailort.h" -#include "vdevice_internal.hpp" -#include "vdma_device.hpp" -#include "vdma_stream.hpp" #include "hailo/expected.hpp" +#include "vdevice/vdevice_internal.hpp" +#include "vdma/vdma_device.hpp" +#include "vdma/vdma_stream.hpp" +#include "stream_common/stream_internal.hpp" + + namespace hailort { @@ -36,20 +38,12 @@ class InputVDeviceBaseStream : public InputStreamBase { public: static Expected> create(std::vector> &&low_level_streams, - const LayerInfo &edge_layer, const scheduler_ng_handle_t &network_group_handle, - EventPtr network_group_activated_event, NetworkGroupSchedulerWeakPtr network_group_scheduler); - - InputVDeviceBaseStream(InputVDeviceBaseStream &&other) : - InputStreamBase(std::move(other)), - m_streams(std::move(other.m_streams)), - m_is_stream_activated(std::exchange(other.m_is_stream_activated, false)), - m_next_transfer_stream_index(other.m_next_transfer_stream_index), - m_acc_frames(other.m_acc_frames) - {} + const LayerInfo &edge_layer, const scheduler_core_op_handle_t &core_op_handle, + EventPtr core_op_activated_event, CoreOpsSchedulerWeakPtr core_ops_scheduler); virtual ~InputVDeviceBaseStream(); - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; virtual hailo_status deactivate_stream() override; virtual hailo_stream_interface_t get_interface() const override; virtual std::chrono::milliseconds get_timeout() const override; @@ -62,6 +56,15 @@ public: virtual hailo_status abort() override = 0; virtual hailo_status clear_abort() override = 0; + virtual hailo_status register_interrupt_callback(const vdma::ProcessingCompleteCallback &callback) override + { + for (auto &stream : m_streams) { + auto status = stream.get().register_interrupt_callback(callback); + CHECK_SUCCESS(status); + } + return HAILO_SUCCESS; + } + virtual void notify_all() { // Overriden in scheduled_stream @@ -78,10 +81,10 @@ protected: explicit InputVDeviceBaseStream( std::vector> &&streams, - EventPtr &&network_group_activated_event, + EventPtr &&core_op_activated_event, const LayerInfo &layer_info, hailo_status &status) : - InputStreamBase(layer_info, streams[0].get().get_interface(), std::move(network_group_activated_event), status), + InputStreamBase(layer_info, streams[0].get().get_interface(), std::move(core_op_activated_event), status), m_streams(std::move(streams)), m_is_stream_activated(false), m_next_transfer_stream_index(0), @@ -101,21 +104,13 @@ private: class OutputVDeviceBaseStream : public OutputStreamBase { public: - OutputVDeviceBaseStream(OutputVDeviceBaseStream &&other) : - OutputStreamBase(std::move(other)), - m_streams(std::move(other.m_streams)), - m_is_stream_activated(std::exchange(other.m_is_stream_activated, false)), - m_next_transfer_stream_index(other.m_next_transfer_stream_index), - m_acc_frames(other.m_acc_frames) - {} - virtual ~OutputVDeviceBaseStream(); static Expected> create(std::vector> &&low_level_streams, - const LayerInfo &edge_layer, const scheduler_ng_handle_t &network_group_handle, - EventPtr network_group_activated_event, NetworkGroupSchedulerWeakPtr network_group_scheduler); + const LayerInfo &edge_layer, const scheduler_core_op_handle_t &core_op_handle, + EventPtr core_op_activated_event, CoreOpsSchedulerWeakPtr core_ops_scheduler); - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; virtual hailo_status deactivate_stream() override; virtual hailo_stream_interface_t get_interface() const override; virtual std::chrono::milliseconds get_timeout() const override; @@ -126,10 +121,10 @@ public: virtual hailo_status clear_abort() override = 0; virtual bool is_scheduled() override = 0; - virtual hailo_status register_for_d2h_interrupts(const std::function &callback) override + virtual hailo_status register_interrupt_callback(const vdma::ProcessingCompleteCallback &callback) override { for (auto &stream : m_streams) { - auto status = stream.get().register_for_d2h_interrupts(callback); + auto status = stream.get().register_interrupt_callback(callback); CHECK_SUCCESS(status); } return HAILO_SUCCESS; @@ -141,9 +136,9 @@ protected: explicit OutputVDeviceBaseStream( std::vector> &&streams, const LayerInfo &layer_info, - EventPtr &&network_group_activated_event, + EventPtr &&core_op_activated_event, hailo_status &status) : - OutputStreamBase(layer_info, std::move(network_group_activated_event), status), + OutputStreamBase(layer_info, std::move(core_op_activated_event), status), m_streams(std::move(streams)), m_is_stream_activated(false), m_next_transfer_stream_index(0), diff --git a/hailort/libhailort/src/vdevice_stream_multiplexer_wrapper.cpp b/hailort/libhailort/src/vdevice/vdevice_stream_multiplexer_wrapper.cpp similarity index 78% rename from hailort/libhailort/src/vdevice_stream_multiplexer_wrapper.cpp rename to hailort/libhailort/src/vdevice/vdevice_stream_multiplexer_wrapper.cpp index 365d7af..b9d9b00 100644 --- a/hailort/libhailort/src/vdevice_stream_multiplexer_wrapper.cpp +++ b/hailort/libhailort/src/vdevice/vdevice_stream_multiplexer_wrapper.cpp @@ -1,4 +1,4 @@ -#include "vdevice_stream_multiplexer_wrapper.hpp" +#include "vdevice/vdevice_stream_multiplexer_wrapper.hpp" namespace hailort { @@ -13,9 +13,9 @@ const CONTROL_PROTOCOL__nn_stream_config_t &VDeviceInputStreamMultiplexerWrapper return m_vdevice_input_stream->get_nn_stream_config(); } -hailo_status VDeviceInputStreamMultiplexerWrapper::activate_stream(uint16_t dynamic_batch_size) +hailo_status VDeviceInputStreamMultiplexerWrapper::activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) { - return m_vdevice_input_stream->activate_stream(dynamic_batch_size); + return m_vdevice_input_stream->activate_stream(dynamic_batch_size, resume_pending_stream_transfers); } hailo_status VDeviceInputStreamMultiplexerWrapper::deactivate_stream() @@ -35,15 +35,18 @@ std::chrono::milliseconds VDeviceInputStreamMultiplexerWrapper::get_timeout() co hailo_status VDeviceInputStreamMultiplexerWrapper::abort() { + if (*m_is_aborted) { + return HAILO_SUCCESS; + } + *m_is_aborted = true; + if (is_scheduled()) { - auto status = m_multiplexer->disable_network_group(m_network_group_multiplexer_handle); + auto status = m_multiplexer->disable_stream(m_core_op_multiplexer_handle, name()); CHECK_SUCCESS(status); - *m_is_aborted = true; m_vdevice_input_stream->notify_all(); - // TODO: HRT-7638 - status = m_multiplexer->run_once_for_stream(name(), INPUT_RUN_ONCE_HANDLE__ABORT, m_network_group_multiplexer_handle); + status = m_multiplexer->run_once_for_stream(name(), INPUT_RUN_ONCE_HANDLE__ABORT, m_core_op_multiplexer_handle); CHECK_SUCCESS(status); return HAILO_SUCCESS; @@ -57,13 +60,16 @@ hailo_status VDeviceInputStreamMultiplexerWrapper::abort() hailo_status VDeviceInputStreamMultiplexerWrapper::clear_abort() { + if (!(*m_is_aborted)) { + return HAILO_SUCCESS; + } + *m_is_aborted = false; + if (is_scheduled()) { - auto status = m_multiplexer->enable_network_group(m_network_group_multiplexer_handle); + auto status = m_multiplexer->enable_stream(m_core_op_multiplexer_handle, name()); CHECK_SUCCESS(status); - *m_is_aborted = false; - - status = m_multiplexer->run_once_for_stream(name(), INPUT_RUN_ONCE_HANDLE__CLEAR_ABORT, m_network_group_multiplexer_handle); + status = m_multiplexer->run_once_for_stream(name(), INPUT_RUN_ONCE_HANDLE__CLEAR_ABORT, m_core_op_multiplexer_handle); CHECK_SUCCESS(status); m_vdevice_input_stream->notify_all(); @@ -100,7 +106,7 @@ Expected VDeviceInputStreamMultiplexerWrapper::get_pending_frames_count( Expected VDeviceInputStreamMultiplexerWrapper::sync_write_raw_buffer(const MemoryView &buffer) { if (is_scheduled()) { - auto status = m_multiplexer->wait_for_write(m_network_group_multiplexer_handle); + auto status = m_multiplexer->wait_for_write(m_core_op_multiplexer_handle); if (HAILO_STREAM_ABORTED_BY_USER == status) { return make_unexpected(status); } @@ -108,16 +114,15 @@ Expected VDeviceInputStreamMultiplexerWrapper::sync_write_raw_buffer(con } auto exp = m_vdevice_input_stream->sync_write_raw_buffer(buffer, [this]() { return m_is_aborted->load(); }); + if (is_scheduled()) { + auto status = m_multiplexer->signal_write_finish(m_core_op_multiplexer_handle, exp.status() != HAILO_SUCCESS); + CHECK_SUCCESS_AS_EXPECTED(status); + } if (HAILO_STREAM_ABORTED_BY_USER == exp.status()) { return make_unexpected(exp.status()); } CHECK_EXPECTED(exp); - if (is_scheduled()) { - auto status = m_multiplexer->signal_write_finish(m_network_group_multiplexer_handle); - CHECK_SUCCESS_AS_EXPECTED(status); - } - return exp; } @@ -136,7 +141,7 @@ hailo_status VDeviceInputStreamMultiplexerWrapper::set_timeout(std::chrono::mill hailo_status VDeviceInputStreamMultiplexerWrapper::flush() { if (is_scheduled()) { - auto status = m_multiplexer->run_once_for_stream(name(), INPUT_RUN_ONCE_HANDLE__FLUSH, m_network_group_multiplexer_handle); + auto status = m_multiplexer->run_once_for_stream(name(), INPUT_RUN_ONCE_HANDLE__FLUSH, m_core_op_multiplexer_handle); CHECK_SUCCESS(status); return HAILO_SUCCESS; @@ -146,35 +151,35 @@ hailo_status VDeviceInputStreamMultiplexerWrapper::flush() } Expected> VDeviceInputStreamMultiplexerWrapper::create(std::shared_ptr vdevice_input_stream, - std::string network_name, std::shared_ptr multiplexer, scheduler_ng_handle_t network_group_scheduler_handle, - multiplexer_ng_handle_t network_group_multiplexer_handle) + std::string network_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t core_ops_scheduler_handle, + multiplexer_core_op_handle_t core_op_multiplexer_handle) { hailo_status status = HAILO_UNINITIALIZED; std::unique_ptr wrapper(new (std::nothrow) VDeviceInputStreamMultiplexerWrapper(vdevice_input_stream, network_name, multiplexer, - network_group_scheduler_handle, network_group_multiplexer_handle, status)); + core_ops_scheduler_handle, core_op_multiplexer_handle, status)); CHECK_NOT_NULL_AS_EXPECTED(wrapper, HAILO_OUT_OF_HOST_MEMORY); CHECK_SUCCESS_AS_EXPECTED(status); return wrapper; } -Expected> VDeviceInputStreamMultiplexerWrapper::clone(multiplexer_ng_handle_t network_group_multiplexer_handle) +Expected> VDeviceInputStreamMultiplexerWrapper::clone(multiplexer_core_op_handle_t core_op_multiplexer_handle) { - auto wrapper = create(m_vdevice_input_stream, m_network_name, m_multiplexer, m_network_group_scheduler_handle, network_group_multiplexer_handle); + auto wrapper = create(m_vdevice_input_stream, m_network_name, m_multiplexer, m_core_ops_scheduler_handle, core_op_multiplexer_handle); CHECK_EXPECTED(wrapper); return wrapper; } VDeviceInputStreamMultiplexerWrapper::VDeviceInputStreamMultiplexerWrapper(std::shared_ptr &vdevice_input_stream, - std::string network_name, std::shared_ptr multiplexer, scheduler_ng_handle_t network_group_scheduler_handle, - multiplexer_ng_handle_t network_group_multiplexer_handle, hailo_status &status) : + std::string network_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t core_ops_scheduler_handle, + multiplexer_core_op_handle_t core_op_multiplexer_handle, hailo_status &status) : InputStreamBase(vdevice_input_stream->get_info(), - vdevice_input_stream->m_nn_stream_config, vdevice_input_stream->get_network_group_activated_event()), + vdevice_input_stream->m_nn_stream_config, vdevice_input_stream->get_core_op_activated_event()), m_vdevice_input_stream(vdevice_input_stream), m_multiplexer(multiplexer), - m_network_group_scheduler_handle(network_group_scheduler_handle), - m_network_group_multiplexer_handle(network_group_multiplexer_handle), + m_core_ops_scheduler_handle(core_ops_scheduler_handle), + m_core_op_multiplexer_handle(core_op_multiplexer_handle), m_network_name(network_name), m_is_aborted() { @@ -222,9 +227,9 @@ const CONTROL_PROTOCOL__nn_stream_config_t &VDeviceOutputStreamMultiplexerWrappe return m_vdevice_output_stream->get_nn_stream_config(); } -hailo_status VDeviceOutputStreamMultiplexerWrapper::activate_stream(uint16_t dynamic_batch_size) +hailo_status VDeviceOutputStreamMultiplexerWrapper::activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) { - return m_vdevice_output_stream->activate_stream(dynamic_batch_size); + return m_vdevice_output_stream->activate_stream(dynamic_batch_size, resume_pending_stream_transfers); } hailo_status VDeviceOutputStreamMultiplexerWrapper::deactivate_stream() @@ -244,12 +249,16 @@ std::chrono::milliseconds VDeviceOutputStreamMultiplexerWrapper::get_timeout() c hailo_status VDeviceOutputStreamMultiplexerWrapper::abort() { + if (*m_is_aborted) { + return HAILO_SUCCESS; + } + *m_is_aborted = true; + if (is_scheduled()) { - auto status = m_multiplexer->disable_network_group(m_network_group_multiplexer_handle); + auto status = m_multiplexer->disable_stream(m_core_op_multiplexer_handle, name()); CHECK_SUCCESS(status); - // TODO: HRT-7638 - status = m_multiplexer->run_once_for_stream(name(), OUTPUT_RUN_ONCE_HANDLE__ABORT, m_network_group_multiplexer_handle); + status = m_multiplexer->run_once_for_stream(name(), OUTPUT_RUN_ONCE_HANDLE__ABORT, m_core_op_multiplexer_handle); CHECK_SUCCESS(status); return HAILO_SUCCESS; @@ -263,11 +272,16 @@ hailo_status VDeviceOutputStreamMultiplexerWrapper::abort() hailo_status VDeviceOutputStreamMultiplexerWrapper::clear_abort() { + if (!(*m_is_aborted)) { + return HAILO_SUCCESS; + } + *m_is_aborted = false; + if (is_scheduled()) { - auto status = m_multiplexer->enable_network_group(m_network_group_multiplexer_handle); + auto status = m_multiplexer->enable_stream(m_core_op_multiplexer_handle, name()); CHECK_SUCCESS(status); - status = m_multiplexer->run_once_for_stream(name(), OUTPUT_RUN_ONCE_HANDLE__CLEAR_ABORT, m_network_group_multiplexer_handle); + status = m_multiplexer->run_once_for_stream(name(), OUTPUT_RUN_ONCE_HANDLE__CLEAR_ABORT, m_core_op_multiplexer_handle); CHECK_SUCCESS(status); return HAILO_SUCCESS; @@ -307,7 +321,7 @@ hailo_status VDeviceOutputStreamMultiplexerWrapper::read(MemoryView buffer) { uint32_t frames_to_drain_count = 0; if (is_scheduled()) { - auto expected_drain_count = m_multiplexer->wait_for_read(m_network_group_multiplexer_handle, name(), + auto expected_drain_count = m_multiplexer->wait_for_read(m_core_op_multiplexer_handle, name(), m_vdevice_output_stream->get_timeout()); if (HAILO_STREAM_ABORTED_BY_USER == expected_drain_count.status()) { return expected_drain_count.status(); @@ -332,10 +346,7 @@ hailo_status VDeviceOutputStreamMultiplexerWrapper::read(MemoryView buffer) CHECK_SUCCESS(status); if (is_scheduled()) { - status = m_multiplexer->signal_read_finish(m_network_group_multiplexer_handle); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - return status; - } + status = m_multiplexer->signal_read_finish(); CHECK_SUCCESS(status); } @@ -348,36 +359,44 @@ hailo_status VDeviceOutputStreamMultiplexerWrapper::set_timeout(std::chrono::mil } Expected> VDeviceOutputStreamMultiplexerWrapper::create(std::shared_ptr vdevice_output_stream, - std::string network_name, std::shared_ptr multiplexer, scheduler_ng_handle_t network_group_scheduler_handle, - multiplexer_ng_handle_t network_group_multiplexer_handle) + std::string network_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t core_ops_scheduler_handle, + multiplexer_core_op_handle_t core_op_multiplexer_handle) { hailo_status status = HAILO_UNINITIALIZED; std::unique_ptr wrapper(new (std::nothrow) VDeviceOutputStreamMultiplexerWrapper(vdevice_output_stream, network_name, multiplexer, - network_group_scheduler_handle, network_group_multiplexer_handle, status)); + core_ops_scheduler_handle, core_op_multiplexer_handle, status)); CHECK_NOT_NULL_AS_EXPECTED(wrapper, HAILO_OUT_OF_HOST_MEMORY); return wrapper; } -Expected> VDeviceOutputStreamMultiplexerWrapper::clone(scheduler_ng_handle_t network_group_multiplexer_handle) +Expected> VDeviceOutputStreamMultiplexerWrapper::clone(scheduler_core_op_handle_t core_op_multiplexer_handle) { - auto wrapper = create(m_vdevice_output_stream, m_network_name, m_multiplexer, m_network_group_scheduler_handle, network_group_multiplexer_handle); + auto wrapper = create(m_vdevice_output_stream, m_network_name, m_multiplexer, m_core_ops_scheduler_handle, core_op_multiplexer_handle); CHECK_EXPECTED(wrapper); return wrapper; } VDeviceOutputStreamMultiplexerWrapper::VDeviceOutputStreamMultiplexerWrapper(std::shared_ptr &vdevice_output_stream, - std::string network_name, std::shared_ptr multiplexer, scheduler_ng_handle_t network_group_scheduler_handle, - multiplexer_ng_handle_t network_group_multiplexer_handle, hailo_status &status) : + std::string network_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t core_ops_scheduler_handle, + multiplexer_core_op_handle_t core_op_multiplexer_handle, hailo_status &status) : OutputStreamBase(vdevice_output_stream->get_layer_info(), vdevice_output_stream->get_info(), - vdevice_output_stream->m_nn_stream_config, vdevice_output_stream->get_network_group_activated_event()), + vdevice_output_stream->m_nn_stream_config, vdevice_output_stream->get_core_op_activated_event()), m_vdevice_output_stream(vdevice_output_stream), m_multiplexer(multiplexer), - m_network_group_scheduler_handle(network_group_scheduler_handle), - m_network_group_multiplexer_handle(network_group_multiplexer_handle), - m_network_name(network_name) + m_core_ops_scheduler_handle(core_ops_scheduler_handle), + m_core_op_multiplexer_handle(core_op_multiplexer_handle), + m_network_name(network_name), + m_is_aborted() { + m_is_aborted = make_unique_nothrow(false); + if (nullptr == m_is_aborted) { + status = HAILO_OUT_OF_HOST_MEMORY; + LOGGER__ERROR("Failed to allocate memory! status = {}", status); + return; + } + status = multiplexer->register_run_once_for_stream(vdevice_output_stream->name(), OUTPUT_RUN_ONCE_HANDLE__ABORT, [this] { return m_vdevice_output_stream->abort(); diff --git a/hailort/libhailort/src/vdevice_stream_multiplexer_wrapper.hpp b/hailort/libhailort/src/vdevice/vdevice_stream_multiplexer_wrapper.hpp similarity index 68% rename from hailort/libhailort/src/vdevice_stream_multiplexer_wrapper.hpp rename to hailort/libhailort/src/vdevice/vdevice_stream_multiplexer_wrapper.hpp index ddb9cba..0876d29 100644 --- a/hailort/libhailort/src/vdevice_stream_multiplexer_wrapper.hpp +++ b/hailort/libhailort/src/vdevice/vdevice_stream_multiplexer_wrapper.hpp @@ -10,10 +10,12 @@ #ifndef HAILO_VDEVICE_STREAM_MULTIPLEXER_WRAPPER_HPP_ #define HAILO_VDEVICE_STREAM_MULTIPLEXER_WRAPPER_HPP_ -#include "vdevice_stream.hpp" -#include "stream_internal.hpp" #include "hailo/expected.hpp" -#include "pipeline_multiplexer.hpp" + +#include "stream_common/stream_internal.hpp" +#include "vdevice/vdevice_stream.hpp" +#include "vdevice/pipeline_multiplexer.hpp" + namespace hailort { @@ -32,19 +34,14 @@ enum output_run_once_handle_t { class VDeviceInputStreamMultiplexerWrapper : public InputStreamBase { public: virtual ~VDeviceInputStreamMultiplexerWrapper() = default; - VDeviceInputStreamMultiplexerWrapper(const VDeviceInputStreamMultiplexerWrapper &other) = delete; - VDeviceInputStreamMultiplexerWrapper &operator=(const VDeviceInputStreamMultiplexerWrapper &other) = delete; - VDeviceInputStreamMultiplexerWrapper &operator=(VDeviceInputStreamMultiplexerWrapper &&other) = delete; - VDeviceInputStreamMultiplexerWrapper(VDeviceInputStreamMultiplexerWrapper &&other) = default; - static Expected> create(std::shared_ptr vdevice_input_stream, - std::string network_name, std::shared_ptr multiplexer, scheduler_ng_handle_t network_group_scheduler_handle, - multiplexer_ng_handle_t network_group_multiplexer_handle = 0); - Expected> clone(multiplexer_ng_handle_t network_group_multiplexer_handle); + std::string network_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t core_ops_scheduler_handle, + multiplexer_core_op_handle_t core_op_multiplexer_handle = 0); + Expected> clone(multiplexer_core_op_handle_t core_op_multiplexer_handle); virtual const hailo_stream_info_t &get_info() const override; virtual const CONTROL_PROTOCOL__nn_stream_config_t &get_nn_stream_config() override; - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; virtual hailo_status deactivate_stream() override; virtual hailo_stream_interface_t get_interface() const override; virtual std::chrono::milliseconds get_timeout() const override; @@ -56,22 +53,27 @@ public: virtual Expected get_buffer_frames_size() const override; virtual Expected get_pending_frames_count() const override; + virtual hailo_status register_interrupt_callback(const vdma::ProcessingCompleteCallback &callback) override + { + return m_vdevice_input_stream->register_interrupt_callback(callback); + } + protected: virtual Expected sync_write_raw_buffer(const MemoryView &buffer) override; virtual hailo_status sync_write_all_raw_buffer_no_transform_impl(void *buffer, size_t offset, size_t size) override; private: VDeviceInputStreamMultiplexerWrapper(std::shared_ptr &vdevice_input_stream, - std::string network_name, std::shared_ptr multiplexer, scheduler_ng_handle_t network_group_scheduler_handle, - multiplexer_ng_handle_t network_group_multiplexer_handle, hailo_status &status); + std::string network_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t core_ops_scheduler_handle, + multiplexer_core_op_handle_t core_op_multiplexer_handle, hailo_status &status); virtual hailo_status set_timeout(std::chrono::milliseconds timeout) override; virtual hailo_status flush() override; std::shared_ptr m_vdevice_input_stream; std::shared_ptr m_multiplexer; - scheduler_ng_handle_t m_network_group_scheduler_handle; - multiplexer_ng_handle_t m_network_group_multiplexer_handle; + scheduler_core_op_handle_t m_core_ops_scheduler_handle; + multiplexer_core_op_handle_t m_core_op_multiplexer_handle; std::string m_network_name; std::unique_ptr m_is_aborted; @@ -80,19 +82,15 @@ private: class VDeviceOutputStreamMultiplexerWrapper : public OutputStreamBase { public: virtual ~VDeviceOutputStreamMultiplexerWrapper() noexcept = default; - VDeviceOutputStreamMultiplexerWrapper(const VDeviceOutputStreamMultiplexerWrapper &other) = delete; - VDeviceOutputStreamMultiplexerWrapper &operator=(const VDeviceOutputStreamMultiplexerWrapper &other) = delete; - VDeviceOutputStreamMultiplexerWrapper &operator=(VDeviceOutputStreamMultiplexerWrapper &&other) = delete; - VDeviceOutputStreamMultiplexerWrapper(VDeviceOutputStreamMultiplexerWrapper &&other) = default; static Expected> create(std::shared_ptr vdevice_output_stream, - std::string network_name, std::shared_ptr multiplexer, scheduler_ng_handle_t network_group_scheduler_handle, - multiplexer_ng_handle_t network_group_multiplexer_handle = 0); - Expected> clone(multiplexer_ng_handle_t network_group_multiplexer_handle); + std::string network_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t core_ops_scheduler_handle, + multiplexer_core_op_handle_t core_op_multiplexer_handle = 0); + Expected> clone(multiplexer_core_op_handle_t core_op_multiplexer_handle); virtual const hailo_stream_info_t &get_info() const override; virtual const CONTROL_PROTOCOL__nn_stream_config_t &get_nn_stream_config() override; - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; virtual hailo_status deactivate_stream() override; virtual hailo_stream_interface_t get_interface() const override; virtual std::chrono::milliseconds get_timeout() const override; @@ -102,9 +100,9 @@ public: virtual Expected get_buffer_frames_size() const override; virtual Expected get_pending_frames_count() const override; - virtual hailo_status register_for_d2h_interrupts(const std::function &callback) override + virtual hailo_status register_interrupt_callback(const vdma::ProcessingCompleteCallback &callback) override { - return m_vdevice_output_stream->register_for_d2h_interrupts(callback); + return m_vdevice_output_stream->register_interrupt_callback(callback); } protected: @@ -112,8 +110,8 @@ protected: private: VDeviceOutputStreamMultiplexerWrapper(std::shared_ptr &vdevice_output_stream, - std::string network_name, std::shared_ptr multiplexer, scheduler_ng_handle_t network_group_scheduler_handle, - multiplexer_ng_handle_t network_group_multiplexer_handle, hailo_status &status); + std::string network_name, std::shared_ptr multiplexer, scheduler_core_op_handle_t core_ops_scheduler_handle, + multiplexer_core_op_handle_t core_op_multiplexer_handle, hailo_status &status); virtual hailo_status set_timeout(std::chrono::milliseconds timeout) override; virtual hailo_status read_all(MemoryView &buffer) override; @@ -121,10 +119,12 @@ private: std::shared_ptr m_vdevice_output_stream; std::shared_ptr m_multiplexer; - scheduler_ng_handle_t m_network_group_scheduler_handle; - multiplexer_ng_handle_t m_network_group_multiplexer_handle; + scheduler_core_op_handle_t m_core_ops_scheduler_handle; + multiplexer_core_op_handle_t m_core_op_multiplexer_handle; std::string m_network_name; EventPtr m_read_event; + + std::unique_ptr m_is_aborted; }; } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/CMakeLists.txt b/hailort/libhailort/src/vdma/CMakeLists.txt new file mode 100644 index 0000000..4111464 --- /dev/null +++ b/hailort/libhailort/src/vdma/CMakeLists.txt @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.0.0) + +set(SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/vdma_device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdma_config_core_op.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdma_config_activated_core_op.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdma_config_manager.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdma_stream.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdma_stream_base.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/vdma_async_stream.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/pcie/pcie_device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/integrated/integrated_device.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/channel/channel_state.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/channel/channel_base.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/channel/buffered_channel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/channel/boundary_channel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/channel/async_channel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/channel/interrupts_dispatcher.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/memory/descriptor_list.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory/vdma_buffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory/dma_mapped_buffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory/mapped_buffer_impl.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory/mapped_buffer_factory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory/sg_buffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory/continuous_buffer.cpp +) + +set(HAILORT_CPP_SOURCES ${HAILORT_CPP_SOURCES} ${SRC_FILES} PARENT_SCOPE) diff --git a/hailort/libhailort/src/vdma/channel/async_channel.cpp b/hailort/libhailort/src/vdma/channel/async_channel.cpp new file mode 100644 index 0000000..890390e --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/async_channel.cpp @@ -0,0 +1,209 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file async_channel.cpp + * @brief Implementation of the AsyncChannel class + **/ + +#include "async_channel.hpp" +#include "hailo/hailort.h" +#include "hailo/hailort_common.hpp" + +namespace hailort +{ +namespace vdma +{ + +Expected AsyncChannel::create(vdma::ChannelId channel_id, Direction direction, + HailoRTDriver &driver, uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name, + LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr) +{ + hailo_status status = HAILO_UNINITIALIZED; + auto channel_ptr = make_shared_nothrow(channel_id, direction, driver, descs_count, + desc_page_size, stream_name, latency_meter, transfers_per_axi_intr, status); + CHECK_NOT_NULL_AS_EXPECTED(channel_ptr, HAILO_OUT_OF_HOST_MEMORY); + CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating AsyncChannel"); + return channel_ptr; +} + +AsyncChannel::AsyncChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, + uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name, + LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr, + hailo_status &status) : + BoundaryChannel(BoundaryChannel::Type::ASYNC, channel_id, direction, driver, descs_count, desc_page_size, + stream_name, latency_meter, transfers_per_axi_intr, status) +{ + // Check that base constructor was successful + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed building Vdma Channel base class"); + return; + } + + status = HAILO_SUCCESS; +} + +hailo_status AsyncChannel::transfer(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque) +{ + CHECK_NOT_NULL(buffer, HAILO_INVALID_ARGUMENT); + CHECK(0 != buffer->size(), HAILO_INVALID_ARGUMENT); + + std::lock_guard state_guard(m_state->mutex()); + if (m_state->m_is_aborted) { + LOGGER__INFO("Tried to write to aborted channel {}", m_channel_id); + return HAILO_STREAM_ABORTED_BY_USER; + } + + hailo_status status = HAILO_UNINITIALIZED; + if (Direction::H2D == m_direction) { + status = transfer_h2d(buffer, user_callback, opaque); + } else { + status = transfer_d2h(buffer, user_callback, opaque); + } + + if (HAILO_STREAM_NOT_ACTIVATED == status) { + LOGGER__INFO("Transfer failed because Channel {} is not activated", m_channel_id); + return HAILO_STREAM_NOT_ACTIVATED; + } + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Transfer failed for channel {} with status {}", m_channel_id, status); + return status; + } + + return HAILO_SUCCESS; +} + +hailo_status AsyncChannel::cancel_pending_transfers() +{ + std::lock_guard state_guard(m_state->mutex()); + for (auto &pending_buffer_info : m_state->m_pending_buffers) { + if (pending_buffer_info.on_transfer_done) { + pending_buffer_info.on_transfer_done(pending_buffer_info.buffer, + hailo_async_transfer_completion_info_t{HAILO_STREAM_NOT_ACTIVATED}, + pending_buffer_info.opaque); + // Release our references to user buffer, callback and opaque + pending_buffer_info = PendingBuffer{}; + } else { + LOGGER__WARNING("No transfer done callback found for transfer (channel {}); skipping", m_channel_id); + } + } + + return HAILO_SUCCESS; +} + +hailo_status AsyncChannel::complete_channel_activation(uint32_t /* transfer_size */, bool /* resume_pending_transfers */) +{ + return HAILO_SUCCESS; +} + +hailo_status AsyncChannel::complete_channel_deactivation() +{ + // Note: We don't reset channel counters here as the resource manager will signal pending transfers + // (i.e. transfers in m_pending_buffers) via cancel_pending_async_transfers. + // The counters are reset in the channel activation + return HAILO_SUCCESS; +} + +hailo_status AsyncChannel::transfer(void */* buf */, size_t /* count */) +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status AsyncChannel::write_buffer(const MemoryView &/* buffer */, std::chrono::milliseconds /* timeout */, + const std::function &/* should_cancel */) +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status AsyncChannel::send_pending_buffer() +{ + return HAILO_NOT_IMPLEMENTED; +} + +void AsyncChannel::notify_all() +{} + +Expected AsyncChannel::get_buffer_state() +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +Expected AsyncChannel::get_h2d_pending_frames_count() +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +Expected AsyncChannel::get_d2h_pending_descs_count() +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +hailo_status AsyncChannel::transfer_d2h(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque) +{ + InterruptsDomain first_desc_interrupts_domain = InterruptsDomain::NONE; + // Provide FW interrupt only in the end of the last transfer in the batch + InterruptsDomain last_desc_interrupts_domain = (m_state->m_accumulated_transfers + 1 == m_transfers_per_axi_intr) ? + InterruptsDomain::BOTH : InterruptsDomain::HOST; + + const auto status = prepare_descriptors(buffer, user_callback, opaque, first_desc_interrupts_domain, last_desc_interrupts_domain); + CHECK_SUCCESS(status); + + m_state->m_accumulated_transfers = (m_state->m_accumulated_transfers + 1) % m_transfers_per_axi_intr; + + return HAILO_SUCCESS; +} + +hailo_status AsyncChannel::transfer_h2d(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque) +{ + // For h2d, only the host need to get transfer done interrupts + InterruptsDomain last_desc_interrupts_domain = InterruptsDomain::HOST; + // If we measure latency, we need interrupt on the first descriptor + InterruptsDomain first_desc_interrupts_domain = (m_latency_meter != nullptr) ? + InterruptsDomain::HOST : InterruptsDomain::NONE; + + return prepare_descriptors(buffer, user_callback, opaque, first_desc_interrupts_domain, last_desc_interrupts_domain); +} + +hailo_status AsyncChannel::prepare_descriptors(std::shared_ptr buffer, const TransferDoneCallback &user_callback, + void *opaque, InterruptsDomain first_desc_interrupts_domain, InterruptsDomain last_desc_interrupts_domain) +{ + const auto desired_desc_num = m_desc_list->descriptors_in_buffer(buffer->size()); + CHECK(desired_desc_num <= MAX_DESCS_COUNT, HAILO_INTERNAL_FAILURE); + const uint16_t desc_num = static_cast(desired_desc_num); + + int num_available = get_num_available(); + int num_processed = CB_TAIL(m_state->m_descs); + int num_free = CB_AVAIL(m_state->m_descs, num_available, num_processed); + if (num_free < desc_num) { + // TODO: do we want to block here? + return HAILO_OUT_OF_DESCRIPTORS; + } + + const auto status = m_desc_list->configure_to_use_buffer(*buffer, m_channel_id, num_available); + CHECK_SUCCESS(status); + if (nullptr != m_latency_meter) { + // Program first descriptor + m_desc_list->program_single_descriptor((*m_desc_list)[num_available], m_desc_list->desc_page_size(), + first_desc_interrupts_domain); + } + auto actual_desc_count = m_desc_list->program_last_descriptor(buffer->size(), last_desc_interrupts_domain, + num_available, true); + CHECK_EXPECTED_AS_STATUS(actual_desc_count, "Failed to program desc_list for channel {}", m_channel_id); + assert (actual_desc_count.value() == desc_num); + int last_desc_avail = ((num_available + desc_num - 1) & m_state->m_descs.size_mask); + + const auto callback = [this, user_callback](std::shared_ptr buffer, const hailo_async_transfer_completion_info_t &status, void *opaque) { + user_callback(buffer, status, opaque); + + // opaque is only for the user callback + static constexpr void *NO_CONTEXT = nullptr; + m_transfer_done_callback(buffer, status, NO_CONTEXT); + }; + + m_state->add_pending_buffer(num_available, last_desc_avail, m_direction, callback, buffer, opaque); + return inc_num_available(desc_num); +} + +} /* namespace vdma */ +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/channel/async_channel.hpp b/hailort/libhailort/src/vdma/channel/async_channel.hpp new file mode 100644 index 0000000..a161ced --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/async_channel.hpp @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file async_channel.hpp + * @brief AsyncChannel - Implements the BoundaryChannel interface, allowing for asyc send/recv and zero copy io + **/ + +#ifndef _HAILO_ASYNC_CHANNEL_HPP_ +#define _HAILO_ASYNC_CHANNEL_HPP_ + +#include "hailo/hailort.h" + +#include "vdma/channel/boundary_channel.hpp" +#include "vdma/channel/channel_state.hpp" + +#include + + +namespace hailort +{ +namespace vdma +{ + +class AsyncChannel; +using AsyncChannelPtr = std::shared_ptr; + +class AsyncChannel : public BoundaryChannel +{ +public: + static Expected create(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, + uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name = "", LatencyMeterPtr latency_meter = nullptr, + uint16_t transfers_per_axi_intr = 1); + + AsyncChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, uint32_t descs_count, + uint16_t desc_page_size, const std::string &stream_name, LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr, + hailo_status &status); + AsyncChannel(AsyncChannel &&) = delete; + AsyncChannel(const AsyncChannel &) = delete; + AsyncChannel &operator=(AsyncChannel &&) = delete; + AsyncChannel &operator=(const AsyncChannel &) = delete; + virtual ~AsyncChannel() = default; + + virtual hailo_status complete_channel_activation(uint32_t transfer_size, bool resume_pending_transfers) override; + virtual hailo_status complete_channel_deactivation() override; + + virtual hailo_status transfer(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque) override; + virtual hailo_status cancel_pending_transfers() override; + + virtual hailo_status transfer(void *buf, size_t count) override; + // TODO: don't want + virtual hailo_status write_buffer(const MemoryView &buffer, std::chrono::milliseconds timeout, + const std::function &should_cancel) override; + // TODO: don't want + virtual hailo_status send_pending_buffer() override; + // TODO: don't want + virtual void notify_all() override; + + // TODO: don't want + virtual Expected get_buffer_state() override; + // TODO: don't want + virtual Expected get_h2d_pending_frames_count() override; + // TODO: don't want + virtual Expected get_d2h_pending_descs_count() override; + +private: + hailo_status transfer_d2h(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque); + hailo_status transfer_h2d(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque); + hailo_status prepare_descriptors(std::shared_ptr buffer, const TransferDoneCallback &user_callback, + void *opaque, InterruptsDomain first_desc_interrupts_domain, InterruptsDomain last_desc_interrupts_domain); +}; + +} /* namespace vdma */ +} /* namespace hailort */ + +#endif /* _HAILO_ASYNC_CHANNEL_HPP_ */ diff --git a/hailort/libhailort/src/vdma/channel/boundary_channel.cpp b/hailort/libhailort/src/vdma/channel/boundary_channel.cpp new file mode 100644 index 0000000..c5652a8 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/boundary_channel.cpp @@ -0,0 +1,361 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file boundary_channel.cpp + * @brief BoundaryChannel - Base class functionality + **/ + +#include "hailo/hailort_common.hpp" + +#include "common/os_utils.hpp" + +#include "vdma/channel/boundary_channel.hpp" +#include "vdma/channel/buffered_channel.hpp" +#include "vdma/channel/async_channel.hpp" + +#include +#include +#include +#include + + +namespace hailort { +namespace vdma { + + +Expected BoundaryChannel::create(vdma::ChannelId channel_id, Direction direction, + HailoRTDriver &driver, uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name, + LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr, Type type) +{ + switch (type) + { + case Type::BUFFERED: + { + auto buffered_channel = BufferedChannel::create(channel_id, direction, driver, descs_count, desc_page_size, + stream_name, latency_meter, transfers_per_axi_intr); + CHECK_EXPECTED(buffered_channel); + + // Upcasting + return std::static_pointer_cast(buffered_channel.value()); + } + + case Type::ASYNC: + { + auto async_channel = AsyncChannel::create(channel_id, direction, driver, descs_count, desc_page_size, + stream_name, latency_meter, transfers_per_axi_intr); + CHECK_EXPECTED(async_channel); + + // Upcasting + return std::static_pointer_cast(async_channel.value()); + } + } + + // Shouldn't get here + return make_unexpected(HAILO_INVALID_ARGUMENT); +} + +BoundaryChannel::BoundaryChannel(Type type, vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, + uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name, + LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr, hailo_status &status) : + ChannelBase(channel_id, direction, driver, descs_count, desc_page_size, stream_name, latency_meter, status), + m_type(type), + m_user_interrupt_callback(ignore_processing_complete), + m_transfers_per_axi_intr(transfers_per_axi_intr) +{ + // Check that base constructor was successful + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed building vdma channel base class"); + return; + } + + if (Direction::BOTH == direction) { + LOGGER__ERROR("Boundary channels must be unidirectional"); + status = HAILO_INVALID_ARGUMENT; + return; + } + + if (m_transfers_per_axi_intr == 0) { + LOGGER__ERROR("Invalid transfers per axi interrupt"); + status = HAILO_INVALID_ARGUMENT; + return; + } + + m_transfer_done_callback = [this](std::shared_ptr, const hailo_async_transfer_completion_info_t &, void *) { + m_user_interrupt_callback(1); + }; +} + +void BoundaryChannel::clear_pending_buffers_descriptors() +{ + for (const auto &pending_buffer : m_state->m_pending_buffers) { + const auto last_desc_index = pending_buffer.last_desc; + + // Clear relevant descriptors from previous transfer + if (nullptr != m_latency_meter) { + const auto latency_desc_index = pending_buffer.latency_measure_desc; + m_desc_list->clear_descriptor(latency_desc_index); + } + m_desc_list->clear_descriptor(last_desc_index); + } +} + +hailo_status BoundaryChannel::trigger_channel_completion(uint16_t hw_num_processed) +{ + size_t processed_no = 0; + + { + // NOTE: right now, we can retake the 'completion' descriptor for a new transfer before handling the interrupt. + // we should have our own pointers indicating whats free instead of reading from HW. + // TODO: consider calculating the last descriptor using the src_desc_avail and src_desc_proc instead of using + // status? + // TODO: we might free a pending buffer which we didn't get an interrupt for yet. we should still handle this + // situation correctly. + + std::lock_guard state_guard(m_state->mutex()); + + // Although the hw_num_processed should be a number between 0 and m_descs.size-1, if m_desc.size < 0x10000 + // (the maximum desc size), the actual hw_num_processed is a number between 1 and m_descs.size. Therefore the + // value can be m_descs.size, in this case we change it to zero. + hw_num_processed = static_cast(hw_num_processed & m_state->m_descs.size_mask); + + if (m_state->m_is_aborted) { + return HAILO_STREAM_ABORTED_BY_USER; + } + + if (!m_state->m_is_channel_activated) { + return HAILO_STREAM_NOT_ACTIVATED; + } + + if (m_latency_meter != nullptr) { + // The latency meter gets an updated hw_num_processed via a call to vdma_interrupts_read_timestamps + // (the desc index of the last measured timestamp returned from that ioctl). Since update_latency_meter + // processed m_pending_buffers based on this hw_num_processed, and this function (i.e. + // trigger_channel_completion) also processes m_pending_buffers based on the value of hw_num_processed, + // we want the two to be the same. Hence, we'll use the more up to date num_processed returned by + // update_latency_meter. + // TODO: fix update_latency_meter flow (HRT-10284) + auto latency_meter_hw_num_processed = update_latency_meter(); + CHECK_EXPECTED_AS_STATUS(latency_meter_hw_num_processed); + hw_num_processed = latency_meter_hw_num_processed.value(); + } + + const auto last_num_processed = static_cast(CB_TAIL(m_state->m_descs)); + + // Calculate pending_buffers_count before iteration, because the iteration removes done transfers + const auto pending_buffers_count = m_state->m_pending_buffers.size(); + for (size_t i = 0; i < pending_buffers_count; i++) { + auto &last_pending_buffer_info = m_state->m_pending_buffers.front(); + const auto last_desc_index = static_cast(last_pending_buffer_info.last_desc); + // Transfer is complete if its last descriptor is in [last_num_processed, hw_num_processed) or + // the the buffer is empty (hw_num_processed == get_num_available()) + const bool is_complete = is_desc_between(last_num_processed, hw_num_processed, last_desc_index) || + (hw_num_processed == get_num_available()); + + #ifndef NDEBUG + static constexpr auto STATUS_MASK = 0xFF; + static constexpr auto ERROR_BIT = 1; + const auto status = (*m_desc_list)[last_desc_index].RemainingPageSize_Status & STATUS_MASK; + CHECK(!is_bit_set(status, ERROR_BIT), HAILO_INTERNAL_FAILURE, + "Error while processing descriptor {} of DMA {} on board {}.", + last_desc_index, m_channel_id, m_driver.dev_path()); + + // status is read after hw_num_processed, so we want is_complete -> (status == 1). + assert(!is_complete || ((status & 0x1) == 1)); + #endif + + if (!is_complete) { + break; + } + + // Clear relevant descriptors from previous transfer + if (nullptr != m_latency_meter) { + const auto latency_desc_index = last_pending_buffer_info.latency_measure_desc; + m_desc_list->clear_descriptor(latency_desc_index); + } + m_desc_list->clear_descriptor(last_desc_index); + + _CB_SET(m_state->m_descs.tail, (last_pending_buffer_info.last_desc + 1) & m_state->m_descs.size_mask); + last_pending_buffer_info.on_transfer_done(last_pending_buffer_info.buffer, + hailo_async_transfer_completion_info_t{HAILO_SUCCESS}, last_pending_buffer_info.opaque); + processed_no++; + m_state->m_pending_buffers.pop_front(); + } + } + + if (0 < processed_no) { + m_state->transfer_buffer_cv().notify_all(); + } + + return HAILO_SUCCESS; +} + +hailo_status BoundaryChannel::register_interrupt_callback(const ProcessingCompleteCallback &callback) +{ + std::lock_guard state_guard(m_state->mutex()); + m_user_interrupt_callback = callback; + return HAILO_SUCCESS; +} + +CONTROL_PROTOCOL__host_buffer_info_t BoundaryChannel::get_boundary_buffer_info(uint32_t transfer_size) +{ + // Boundary channels always have scatter gather buffers + return VdmaBuffer::get_host_buffer_info(VdmaBuffer::Type::SCATTER_GATHER, m_desc_list->dma_address(), + m_desc_list->desc_page_size(), m_desc_list->count(), transfer_size); +} + +hailo_status BoundaryChannel::abort() +{ + { + std::lock_guard state_guard(m_state->mutex()); + m_state->m_is_aborted = true; + } + + m_state->transfer_buffer_cv().notify_all(); + + return HAILO_SUCCESS; +} + +hailo_status BoundaryChannel::clear_abort() +{ + std::lock_guard state_guard(m_state->mutex()); + m_state->m_is_aborted = false; + + return HAILO_SUCCESS; +} + +hailo_status BoundaryChannel::activate(uint32_t transfer_size, bool resume_pending_transfers) +{ + std::lock_guard state_guard(m_state->mutex()); + + CHECK(!m_state->m_is_channel_activated, HAILO_INTERNAL_FAILURE, + "Vdma channel {} is already activated", m_channel_id); + m_state->m_is_channel_activated = true; + clear_pending_buffers_descriptors(); + m_state->reset_counters(); + + auto status = complete_channel_activation(transfer_size, resume_pending_transfers); + if (HAILO_SUCCESS != status) { + m_state->m_is_channel_activated = false; + return status; + } + + return HAILO_SUCCESS; +} + +hailo_status BoundaryChannel::deactivate() +{ + std::unique_lock state_guard(m_state->mutex()); + + CHECK(m_state->m_is_channel_activated, HAILO_INTERNAL_FAILURE, + "Vdma channel {} is not activated", m_channel_id); + m_state->m_is_channel_activated = false; + + // Reset the user callback, so as not to keep objects provided by the user alive (they may lead to a chain of refs + // back to this channel causing it to be leaked). + // Note: PendingBuffers held by m_pending_buffers may still hold copies of the current m_transfer_done_callback, + // which in turn holds a reference to *this. Since we stop the m_wait_interrupts_thread there's no risk that + // these callbacks will be called and we don't need to reset this callback. + m_user_interrupt_callback = ignore_processing_complete; + + auto status = complete_channel_deactivation(); + CHECK_SUCCESS(status); + + return HAILO_SUCCESS; +} + +BoundaryChannel::Type BoundaryChannel::type() const +{ + return m_type; +} + +hailo_status BoundaryChannel::flush(const std::chrono::milliseconds &timeout) +{ + if (Direction::D2H == m_direction) { + // We are not buffering user data + return HAILO_SUCCESS; + } + + std::unique_lock state_guard(m_state->mutex()); + hailo_status status = HAILO_SUCCESS; // Best effort + bool was_successful = m_state->transfer_buffer_cv().wait_for(state_guard, timeout, [this, &status] () { + if (m_state->m_is_aborted) { + status = HAILO_STREAM_ABORTED_BY_USER; + return true; // return true so that the wait will finish + } + return m_state->m_pending_buffers.empty(); + }); + CHECK(was_successful, HAILO_TIMEOUT, "Got HAILO_TIMEOUT while waiting for channel {} interrupts on flush", m_channel_id); + return status; +} + +bool BoundaryChannel::is_ready_for_transfer_h2d(size_t buffer_size) +{ + return has_room_in_desc_list(buffer_size); +} + +bool BoundaryChannel::is_ready_for_transfer_d2h(size_t buffer_size) +{ + return has_room_in_desc_list(buffer_size); +} + +bool BoundaryChannel::has_room_in_desc_list(size_t buffer_size) +{ + size_t desired_desc_num = m_desc_list->descriptors_in_buffer(buffer_size); + assert(desired_desc_num <= MAX_DESCS_COUNT); + int desc_num = static_cast(desired_desc_num); + + if (m_state->m_pending_buffers.full()) { + return false; + } + + int num_available = get_num_available(); + int num_processed = CB_TAIL(m_state->m_descs); + + if (desc_num == m_state->m_descs.size) { + // Special case when the checking if the buffer is empty + return num_available == num_processed; + } + + int num_free = CB_AVAIL(m_state->m_descs, num_available, num_processed); + if (num_free < desc_num) { + return false; + } + + return true; +} + +hailo_status BoundaryChannel::wait(size_t buffer_size, std::chrono::milliseconds timeout) +{ + const auto max_transfer_size = m_desc_list->desc_page_size() * m_desc_list->count(); + CHECK(buffer_size < max_transfer_size, HAILO_INVALID_ARGUMENT, + "Requested transfer size ({}) must be smaller than ({})", buffer_size, max_transfer_size); + + auto is_ready_for_transfer = (Direction::H2D == m_direction) ? + std::bind(&BoundaryChannel::is_ready_for_transfer_h2d, this, buffer_size) : + std::bind(&BoundaryChannel::is_ready_for_transfer_d2h, this, buffer_size); + + std::unique_lock state_guard(m_state->mutex()); + hailo_status status = HAILO_SUCCESS; // Best effort + bool was_successful = m_state->transfer_buffer_cv().wait_for(state_guard, timeout, [this, is_ready_for_transfer, &status] () { + if (m_state->m_is_aborted) { + status = HAILO_STREAM_ABORTED_BY_USER; + return true; // return true so that the wait will finish + } + + return is_ready_for_transfer(); + }); + CHECK(was_successful, HAILO_TIMEOUT, "Got HAILO_TIMEOUT while waiting for channel {} interrupts", m_channel_id); + return status; +} + +hailo_status BoundaryChannel::set_transfers_per_axi_intr(uint16_t transfers_per_axi_intr) +{ + CHECK(0 != transfers_per_axi_intr, HAILO_INVALID_ARGUMENT, "Invalid transfers per axi interrupt"); + m_transfers_per_axi_intr = transfers_per_axi_intr; + return HAILO_SUCCESS; +} + +} /* namespace vdma */ +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/channel/boundary_channel.hpp b/hailort/libhailort/src/vdma/channel/boundary_channel.hpp new file mode 100644 index 0000000..d578a24 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/boundary_channel.hpp @@ -0,0 +1,141 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file boundary_channel.hpp + * @brief BoundaryChannel - vdma boundary channel interface + * The hierarchy is as follows: + * ------------------------------------------------------------------------- + * | ChannelBase | (Base class - includes state) | + * | | | | + * | BoundaryChannel | (Boundary interface) | + * | / \ | | + * | AsyncChannel BufferedChannel | (Impls) | + * ------------------------------------------------------------------------- + **/ + +#ifndef _HAILO_VDMA_BOUNDARY_CHANNEL_HPP_ +#define _HAILO_VDMA_BOUNDARY_CHANNEL_HPP_ + +#include "hailo/hailort.h" +#include "hailo/stream.hpp" + +#include "vdma/channel/channel_base.hpp" + +#include + + +namespace hailort { +namespace vdma { + +class BoundaryChannel; +using BoundaryChannelPtr = std::shared_ptr; + +using ProcessingCompleteCallback = std::function; + +class BoundaryChannel : public ChannelBase +{ +public: + enum class Type + { + BUFFERED = 0, + ASYNC + }; + + static Expected create(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, + uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name = "", LatencyMeterPtr latency_meter = nullptr, + uint16_t transfers_per_axi_intr = 1, Type type = Type::BUFFERED); + + BoundaryChannel(Type type, vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, uint32_t descs_count, + uint16_t desc_page_size, const std::string &stream_name, LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr, + hailo_status &status); + BoundaryChannel(const BoundaryChannel &other) = delete; + BoundaryChannel &operator=(const BoundaryChannel &other) = delete; + BoundaryChannel(BoundaryChannel &&other) = delete; + BoundaryChannel &operator=(BoundaryChannel &&other) = delete; + virtual ~BoundaryChannel() = default; + + // Called after the FW activated the channel. + hailo_status activate(uint32_t transfer_size, bool resume_pending_transfers); + + // Called before the FW deactivated the channel. + hailo_status deactivate(); + + Type type() const; + + void clear_pending_buffers_descriptors(); + hailo_status trigger_channel_completion(uint16_t hw_num_processed); + virtual hailo_status register_interrupt_callback(const ProcessingCompleteCallback &callback); + CONTROL_PROTOCOL__host_buffer_info_t get_boundary_buffer_info(uint32_t transfer_size); + virtual hailo_status abort(); + virtual hailo_status clear_abort(); + + // For D2H channels, we don't buffer data + // Hence there's nothing to be "flushed" and the function will return with HAILO_SUCCESS + virtual hailo_status flush(const std::chrono::milliseconds &timeout); + virtual hailo_status wait(size_t buffer_size, std::chrono::milliseconds timeout); + hailo_status set_transfers_per_axi_intr(uint16_t transfers_per_axi_intr); + + virtual hailo_status transfer(void *buf, size_t count) = 0; + // TODO: can write_buffer + send_pending_buffer move to BufferedChannel? (HRT-9105) + // Either write_buffer + send_pending_buffer or transfer (h2d) should be used on a given channel, not both + virtual hailo_status write_buffer(const MemoryView &buffer, std::chrono::milliseconds timeout, + const std::function &should_cancel) = 0; + virtual hailo_status send_pending_buffer() = 0; + + // TODO: move buffer? + // TODO: If the same callback is used for different buffers we need a way to tell the transfers appart + // - Passing buffer to callback could do the trick. However, what will happen if the same buffer has been transferred twice? + // - Maybe add a unique transfer_id? At least unique in the context of the maximum number of ongoing transfers + // TODO: What if there's no more room in desc list so the transfer can't be programmed? Should the function block + // - Maybe define that if more than max_concurrent_transfers() (based on a param passed to create) the function will return a failure? + // When the transfer is complete (i.e. data is written to/from buffer with a D2H/H2D channel) callback is called + // buffer can't be freed until callback is called + virtual hailo_status transfer(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque) = 0; + + // Calls all pending transfer callbacks (if they exist), marking them as canceled by passing hailo_async_transfer_completion_info_t{HAILO_STREAM_NOT_ACTIVATED}. + // Note: This function is to be called on a deactivated channel object. Calling on an active channel will lead to unexpected results + virtual hailo_status cancel_pending_transfers() = 0; + + virtual void notify_all() = 0; + + class BufferState { + public: + std::vector> desc_buffer_pairing; + uint16_t num_avail; + uint16_t num_processed; + uint16_t hw_num_avail; + uint16_t hw_num_processed; + }; + + // Assumes that the channel is idle; doesn't block changes to the channel + // To be used for debugging purposes + // TODO: these will move to BufferedChannel (HRT-9105) + virtual Expected get_buffer_state() = 0; + virtual Expected get_h2d_pending_frames_count() = 0; + virtual Expected get_d2h_pending_descs_count() = 0; + +protected: + static void ignore_processing_complete(uint32_t) {} + void stop_interrupts_thread(std::unique_lock &lock); + virtual bool is_ready_for_transfer_h2d(size_t buffer_size); + virtual bool is_ready_for_transfer_d2h(size_t buffer_size); + + // Called after activate/deactivate with the state mutex held + virtual hailo_status complete_channel_activation(uint32_t transfer_size, bool resume_pending_transfers) = 0; + virtual hailo_status complete_channel_deactivation() = 0; + + const Type m_type; + TransferDoneCallback m_transfer_done_callback; + ProcessingCompleteCallback m_user_interrupt_callback; + uint16_t m_transfers_per_axi_intr; + +private: + bool has_room_in_desc_list(size_t buffer_size); +}; + +} /* namespace vdma */ +} /* namespace hailort */ + +#endif // _HAILO_VDMA_BOUNDARY_CHANNEL_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/channel/buffered_channel.cpp b/hailort/libhailort/src/vdma/channel/buffered_channel.cpp new file mode 100644 index 0000000..d1176ee --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/buffered_channel.cpp @@ -0,0 +1,573 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file buffered_channel.cpp + * @brief Implementation of the BufferedChannel class + **/ + +#include "hailo/hailort_common.hpp" + +#include "common/logger_macros.hpp" + +#include "vdma/channel/buffered_channel.hpp" +#include "vdma/memory/mapped_buffer_factory.hpp" +#include "vdma/memory/mapped_buffer_impl.hpp" +#include "hw_consts.hpp" + +#include +#include +#include +#include + + +namespace hailort { +namespace vdma { + +Expected BufferedChannel::create(vdma::ChannelId channel_id, Direction direction, + HailoRTDriver &driver, uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name, + LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr) +{ + hailo_status status = HAILO_UNINITIALIZED; + auto channel_ptr = make_shared_nothrow(channel_id, direction, driver, descs_count, + desc_page_size, stream_name, latency_meter, transfers_per_axi_intr, status); + CHECK_NOT_NULL_AS_EXPECTED(channel_ptr, HAILO_OUT_OF_HOST_MEMORY); + CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating BufferedChannel"); + + return channel_ptr; +} + +BufferedChannel::BufferedChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, + uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name, + LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr, hailo_status &status) : + BoundaryChannel(BoundaryChannel::Type::BUFFERED, channel_id, direction, driver, descs_count, desc_page_size, + stream_name, latency_meter, transfers_per_axi_intr, status), + m_channel_buffer(nullptr), + m_pending_buffers_sizes(0), + m_pending_num_avail_offset(0) +{ + // Check that base constructor was successful + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed building boundary channel base class"); + return; + } + + auto mapped_buffer = create_mapped_buffer(descs_count, desc_page_size, direction, driver); + if (!mapped_buffer) { + LOGGER__ERROR("Failed building mapped vdma buffer"); + status = mapped_buffer.status(); + return; + } + m_channel_buffer = mapped_buffer.release(); + + status = m_desc_list->configure_to_use_buffer(*m_channel_buffer, channel_id, 0); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed binding vdma buffer to desc list"); + return; + } + + m_pending_buffers_sizes = CircularArray(descs_count); + + status = HAILO_SUCCESS; +} + +Expected> BufferedChannel::create_mapped_buffer(uint32_t descs_count, uint16_t desc_page_size, + Direction direction, HailoRTDriver &driver) +{ + auto desc_page_size_value = driver.calc_desc_page_size(desc_page_size); + CHECK_AS_EXPECTED(is_powerof2(desc_page_size_value), HAILO_INVALID_ARGUMENT, "Descriptor page_size must be a power of two."); + + auto mapped_buffer_exp = MappedBufferFactory::create_mapped_buffer(descs_count * desc_page_size_value, direction, driver); + CHECK_EXPECTED(mapped_buffer_exp); + + auto mapped_buffer = make_shared_nothrow(mapped_buffer_exp.release()); + CHECK_NOT_NULL_AS_EXPECTED(mapped_buffer, HAILO_OUT_OF_HOST_MEMORY); + + return mapped_buffer; +} + +hailo_status BufferedChannel::complete_channel_deactivation() +{ + const auto status = store_channel_buffer_state(); + CHECK_SUCCESS(status); + + if (Direction::H2D == m_direction) { + clear_pending_buffers_descriptors(); + // For H2D channels we reset counters as we want to allow writes to the start of the buffer while the channel is stopped + m_state->reset_counters(); + } + + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::store_channel_buffer_state() +{ + // TODO: If a D2H channel is deactivated before all of it's pending frames have recv'd ints + // we'll store a tail value that won't be up to date when the channel is activated again. + // Potentially, we might overwrite frames in that situation. Note that we can't flush() in the case + // of D2H channels (as we can with H2D channels), because num_avail may be greater than the number of frames + // that will be recv'd on a given channel. E.g., upon channel activation for the first time we call + // prepare_d2h_pending_descriptors with the maximum number of descs possible for this channel, which will + // accommodate X frames. If the usert only sends Y < X frames on the input channel, only Y output frames will + // be recv'd (assuming one output frame per input frame). Hence, flush() won't return (we won't dequeue all + // pending buffers). This needs to be handled by the sched that uses this feature. (HRT-9456) + auto tail = get_hw_num_processed(); + CHECK_EXPECTED_AS_STATUS(tail); + + const auto temp = m_state->m_previous_tail; + m_state->m_previous_tail = (tail.value() + m_state->m_previous_tail) & m_state->m_descs.size_mask; + m_state->m_desc_list_delta = temp - m_state->m_previous_tail; + + return HAILO_SUCCESS; +} + +Expected BufferedChannel::get_h2d_pending_frames_count() +{ + return m_pending_buffers_sizes.size(); +} + +Expected BufferedChannel::get_d2h_pending_descs_count() +{ + std::lock_guard state_guard(m_state->mutex()); + + int num_proc = CB_TAIL(m_state->m_descs); + int desc_num_ready = CB_PROG(m_state->m_descs, num_proc, m_state->m_d2h_read_desc_index); + + return desc_num_ready; +} + +hailo_status BufferedChannel::prepare_d2h_pending_descriptors(uint32_t transfer_size, uint32_t transfers_count) +{ + // on D2H no need for interrupt of first descriptor + const auto first_desc_interrupts_domain = InterruptsDomain::NONE; + for (uint32_t i = 0; i < transfers_count; i++) { + // Provide FW interrupt only in the end of the last transfer in the batch + auto last_desc_interrutps_domain = + (static_cast(m_transfers_per_axi_intr - 1) == (i % m_transfers_per_axi_intr)) ? + InterruptsDomain::BOTH : InterruptsDomain::HOST; + auto status = prepare_descriptors(transfer_size, first_desc_interrupts_domain, last_desc_interrutps_domain); + if (HAILO_STREAM_NOT_ACTIVATED == status) { + LOGGER__INFO("preparing descriptors failed because channel is not activated"); + return status; + } + CHECK_SUCCESS(status, "Failed prepare desc status={}", status); + } + + // We assume each output transfer is in the same size + m_state->m_accumulated_transfers += ((m_state->m_accumulated_transfers + transfers_count) % m_transfers_per_axi_intr); + + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::complete_channel_activation(uint32_t transfer_size, bool resume_pending_transfers) +{ + auto status = HAILO_UNINITIALIZED; + + // We should have no active transfers now + if (resume_pending_transfers) { + // We want the first descriptor (at index zero) to point to where the descriptor at index + // m_state->m_previous_tail currently points to: + // * In the case of a D2H channel, m_state->m_previous_tail is the index of the desc where the hw would next + // write to (num_proc). Hence, the hw will now write exactly where it left off. Previously unread frames from + // the user (pointed to by m_state->m_d2h_read_desc_index) can still be read (the hw won't overwrite them). + // * In the case of a H2D channel, m_state->m_previous_tail is the index of the desc where the hw would next + // read from (num_proc). Hence, the hw will now read exactly where it left off. Previously written frames + // from the user (that appear before m_state->m_previous_tail), will not be re-written. + const uint32_t starting_desc_offset = (m_desc_list->count() - m_state->m_previous_tail) % m_desc_list->count(); + status = m_desc_list->configure_to_use_buffer(*m_channel_buffer, m_channel_id, + starting_desc_offset); + CHECK_SUCCESS(status); + + if (Direction::D2H == m_direction) { + // m_d2h_read_desc_index, which is relative to the first desc, needs to shift by m_desc_list_delta + m_state->m_d2h_read_desc_index = (m_state->m_d2h_read_desc_index + m_state->m_desc_list_delta) & m_state->m_descs.size_mask; + } + } else { + // We're not resuming pending transfers - clear relevant pointers. + m_state->reset_previous_state_counters(); + } + + if ((Direction::D2H == m_direction) && (transfer_size != 0)) { + const auto transfers_in_buffer = get_transfers_count_in_buffer(transfer_size); + const auto pending_descs = get_d2h_pending_descs_count(); + const auto descs_in_transfer = m_desc_list->descriptors_in_buffer(transfer_size); + const auto pending_transfers = pending_descs.value() / descs_in_transfer; + // We prepare descs in advance for D2H channels: + // (1) The channel's buffer can store up to 'transfers_in_buffer' frames of size transfer_size + // (2) There are 'pending_transfers' frames from the previous channel activation (we assume that the same + // 'transfer_size' was used) + // (3) Hence, we have room for 'transfers_in_buffer - pending_transfers' frames in the buffer currently. + // (4) However, we can allow at most 'm_state->m_pending_buffers.capacity()' transfers. We can't store more than + // that in the pending buffers circular array. + // (5) Hence, we'll take the minimum between (3) and (4). + const auto transfers_count = std::min(transfers_in_buffer - pending_transfers, + m_state->m_pending_buffers.capacity()); + status = prepare_d2h_pending_descriptors(transfer_size, static_cast(transfers_count)); + CHECK_SUCCESS(status); + } + + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::transfer(void *buf, size_t count) +{ + CHECK_NOT_NULL(buf, HAILO_INVALID_ARGUMENT); + CHECK(0 != count, HAILO_INVALID_ARGUMENT); + + std::lock_guard state_guard(m_state->mutex()); + if (m_state->m_is_aborted) { + LOGGER__INFO("Tried to write to aborted channel {}", m_channel_id); + return HAILO_STREAM_ABORTED_BY_USER; + } + + hailo_status status = HAILO_UNINITIALIZED; + if (Direction::H2D == m_direction) { + status = transfer_h2d(buf, count); + } else { + status = transfer_d2h(buf, count); + } + + if (HAILO_STREAM_NOT_ACTIVATED == status) { + LOGGER__INFO("Transfer failed because Channel {} is not activated", m_channel_id); + return HAILO_STREAM_NOT_ACTIVATED; + } + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Transfer failed for channel {} with status {}", m_channel_id, status); + return status; + } + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::write_buffer_impl(const MemoryView &buffer) +{ + const uint32_t desired_desc_num = m_desc_list->descriptors_in_buffer(buffer.size()); + const uint32_t desc_avail = (get_num_available() + m_pending_num_avail_offset) & m_state->m_descs.size_mask; + assert(desired_desc_num <= MAX_DESCS_COUNT); + assert(CB_AVAIL(m_state->m_descs, desc_avail, CB_TAIL(m_state->m_descs)) >= desired_desc_num); + + const size_t buffer_write_offset = ((desc_avail + m_state->m_previous_tail) & m_state->m_descs.size_mask) * m_desc_list->desc_page_size(); + const auto status = write_to_channel_buffer_cyclic(buffer, buffer_write_offset); + CHECK_SUCCESS(status); + + m_pending_num_avail_offset = static_cast(m_pending_num_avail_offset + desired_desc_num); + + CHECK(!m_pending_buffers_sizes.full(), HAILO_INVALID_OPERATION, "Cannot add more pending buffers!"); + m_pending_buffers_sizes.push_back(buffer.size()); + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::write_to_channel_buffer_cyclic(const MemoryView &buffer, size_t channel_buffer_write_offset) +{ + CHECK(buffer.size() <= m_channel_buffer->size(), HAILO_INSUFFICIENT_BUFFER, + "Can't write {} bytes to channel buffer (channel buffer size {})", + buffer.size(), m_channel_buffer->size()); + + const auto size_to_end = m_channel_buffer->size() - channel_buffer_write_offset; + const auto first_chunk_size = std::min(size_to_end, buffer.size()); + const auto first_chunk_addr = static_cast(m_channel_buffer->user_address()) + channel_buffer_write_offset; + + // Copy from buffer to m_channel_buffer and then synchronize + std::memcpy(first_chunk_addr, buffer.data(), first_chunk_size); + auto status = m_channel_buffer->pimpl->synchronize(channel_buffer_write_offset, first_chunk_size); + CHECK_SUCCESS(status); + + const auto remaining_size = buffer.size() - first_chunk_size; + if (remaining_size > 0) { + // Copy the remainder from buffer to m_channel_buffer and then synchronize + std::memcpy(m_channel_buffer->user_address(), buffer.data() + first_chunk_size, remaining_size); + status = m_channel_buffer->pimpl->synchronize(0, remaining_size); + CHECK_SUCCESS(status); + } + + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::read_from_channel_buffer_cyclic(uint8_t *dest_buffer, size_t read_size, size_t channel_buffer_read_offset) +{ + CHECK(read_size <= m_channel_buffer->size(), HAILO_INSUFFICIENT_BUFFER, + "Can't read {} bytes from channel buffer (channel buffer size {})", + read_size, m_channel_buffer->size()); + + const auto size_to_end = m_channel_buffer->size() - channel_buffer_read_offset; + const auto first_chunk_size = std::min(size_to_end, read_size); + const auto first_chunk_addr = static_cast(m_channel_buffer->user_address()) + channel_buffer_read_offset; + + // Synchronize m_channel_buffer and copy to dest_buffer + auto status = m_channel_buffer->pimpl->synchronize(channel_buffer_read_offset, first_chunk_size); + CHECK_SUCCESS(status); + std::memcpy(dest_buffer, first_chunk_addr, first_chunk_size); + + const auto remaining_size = read_size - first_chunk_size; + if (remaining_size > 0) { + // Synchronize m_channel_buffer and copy remainder to dest_buffer + status = m_channel_buffer->pimpl->synchronize(0, remaining_size); + CHECK_SUCCESS(status); + std::memcpy(dest_buffer + first_chunk_size, m_channel_buffer->user_address(), remaining_size); + } + + return HAILO_SUCCESS; +} + +Expected BufferedChannel::get_buffer_state() +{ + BoundaryChannel::BufferState result; + result.num_avail = static_cast(CB_HEAD(m_state->m_descs)); + result.num_processed = static_cast(CB_TAIL(m_state->m_descs)); + auto hw_num_avail = m_host_registers.get_num_available(); + CHECK_EXPECTED(hw_num_avail); + result.hw_num_avail = hw_num_avail.release(); + auto hw_num_processed = get_hw_num_processed(); + CHECK_EXPECTED(hw_num_processed); + result.hw_num_processed = hw_num_processed.release(); + + // Get a snapshot of the channel buffer + auto channel_buffer_copy = Buffer::create(m_channel_buffer->size()); + CHECK_EXPECTED(channel_buffer_copy); + const auto status = read_from_channel_buffer_cyclic(channel_buffer_copy->data(), channel_buffer_copy->size(), 0); + CHECK_SUCCESS_AS_EXPECTED(status); + + for (size_t offset = 0; offset < channel_buffer_copy->size(); offset += m_desc_list->desc_page_size()) { + auto chunk = Buffer::create(channel_buffer_copy->data() + offset, m_desc_list->desc_page_size()); + CHECK_EXPECTED(chunk); + const auto abs_index = offset / m_desc_list->desc_page_size(); + const auto desc_num = (abs_index >= static_cast(m_state->m_previous_tail)) ? + abs_index - m_state->m_previous_tail : + m_state->m_descs.size - m_state->m_previous_tail + abs_index; + result.desc_buffer_pairing.emplace_back(static_cast(desc_num), chunk.release()); + } + + return result; +} + +hailo_status BufferedChannel::write_buffer(const MemoryView &buffer, std::chrono::milliseconds timeout, + const std::function &should_cancel) +{ + std::unique_lock state_guard(m_state->mutex()); + + // Checking in advance so as not to timeout + CHECK(buffer.size() <= m_channel_buffer->size(), HAILO_INSUFFICIENT_BUFFER, + "Can't write {} bytes to channel buffer (channel buffer size {})", + buffer.size(), m_channel_buffer->size()); + + size_t desired_desc_num = m_desc_list->descriptors_in_buffer(buffer.size()); + hailo_status channel_completion_status = HAILO_SUCCESS; + bool was_successful = m_state->transfer_buffer_cv().wait_for(state_guard, timeout, [this, desired_desc_num, + &should_cancel, &channel_completion_status] () { + if (m_state->m_is_aborted) { + return true; + } + + if (should_cancel()) { + channel_completion_status = HAILO_STREAM_ABORTED_BY_USER; + return true; + } + // Limit writes to not surpass size of m_pending_buffers + size_t written_buffers_count = m_pending_buffers_sizes.size(); + size_t sent_buffers_count = m_state->m_pending_buffers.size(); + if (written_buffers_count + sent_buffers_count >= m_state->m_pending_buffers.capacity()) { + return false; + } + + return is_ready_for_write(static_cast(desired_desc_num)); + }); + if (m_state->m_is_aborted || (HAILO_STREAM_ABORTED_BY_USER == channel_completion_status)) { + LOGGER__INFO("wait_for in write_buffer was aborted!"); + return HAILO_STREAM_ABORTED_BY_USER; + } + CHECK(was_successful, HAILO_TIMEOUT, "Got HAILO_TIMEOUT while waiting for descriptors in write_buffer (channel_id={})", + m_channel_id); + CHECK_SUCCESS(channel_completion_status); + + return write_buffer_impl(buffer); +} + +hailo_status BufferedChannel::send_pending_buffer_impl() +{ + CHECK(!m_pending_buffers_sizes.empty(), HAILO_INVALID_OPERATION, "There are no pending buffers to send!"); + + // For h2d, only the host need to get transfer done interrupts + InterruptsDomain last_desc_interrupts_domain = InterruptsDomain::HOST; + // If we measure latency, we need interrupt on the first descriptor + InterruptsDomain first_desc_interrupts_domain = (m_latency_meter != nullptr) ? + InterruptsDomain::HOST : InterruptsDomain::NONE; + + auto status = prepare_descriptors(m_pending_buffers_sizes.front(), first_desc_interrupts_domain, last_desc_interrupts_domain); + if (HAILO_STREAM_NOT_ACTIVATED == status) { + LOGGER__INFO("sending pending buffer failed because stream is not activated"); + // Stream was aborted during transfer - reset pending buffers + m_pending_num_avail_offset = 0; + while (m_pending_buffers_sizes.size() > 0) { + m_pending_buffers_sizes.pop_front(); + } + return status; + } + CHECK_SUCCESS(status); + m_state->m_accumulated_transfers = (m_state->m_accumulated_transfers + 1) % m_transfers_per_axi_intr; + + size_t desired_desc_num = m_desc_list->descriptors_in_buffer(m_pending_buffers_sizes.front()); + m_pending_num_avail_offset = static_cast(m_pending_num_avail_offset - desired_desc_num); + + m_pending_buffers_sizes.pop_front(); + + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::send_pending_buffer() +{ + { + std::lock_guard state_guard(m_state->mutex()); + + auto status = send_pending_buffer_impl(); + if (HAILO_STREAM_NOT_ACTIVATED == status) { + LOGGER__INFO("stream is not activated"); + return HAILO_STREAM_NOT_ACTIVATED; + } else { + CHECK_SUCCESS(status); + } + } + m_state->transfer_buffer_cv().notify_one(); + + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::transfer(std::shared_ptr, const TransferDoneCallback &, void *) +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status BufferedChannel::cancel_pending_transfers() +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status BufferedChannel::transfer_h2d(void *buf, size_t count) +{ + auto status = write_buffer_impl(MemoryView(buf, count)); + CHECK_SUCCESS(status); + + status = send_pending_buffer_impl(); + if (HAILO_STREAM_NOT_ACTIVATED == status) { + return status; + } else { + CHECK_SUCCESS(status); + } + + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::transfer_d2h(void *buf, size_t count) +{ + hailo_status status = HAILO_UNINITIALIZED; + // Provide FW interrupt only in the end of the last transfer in the batch + InterruptsDomain first_desc_interrupts_domain = InterruptsDomain::NONE; + InterruptsDomain last_desc_interrupts_domain = (m_state->m_accumulated_transfers + 1 == m_transfers_per_axi_intr) ? + InterruptsDomain::BOTH : InterruptsDomain::HOST; + + auto desired_desc_num = m_desc_list->descriptors_in_buffer(count); + assert(desired_desc_num <= MAX_DESCS_COUNT); + int desc_num = static_cast(desired_desc_num); + + int num_processed = CB_TAIL(m_state->m_descs); + int num_ready = CB_PROG(m_state->m_descs, num_processed, m_state->m_d2h_read_desc_index); + CHECK(num_ready >= desc_num, HAILO_OUT_OF_DESCRIPTORS, + "{} descriptors desired but only {} available", desc_num, num_ready); + + const auto channel_buffer_read_offset = m_state->m_d2h_read_desc_index_abs * m_desc_list->desc_page_size(); + status = read_from_channel_buffer_cyclic(static_cast(buf), count, channel_buffer_read_offset); + CHECK_SUCCESS(status); + + m_state->m_d2h_read_desc_index = (m_state->m_d2h_read_desc_index + desc_num) & m_state->m_descs.size_mask; + m_state->m_d2h_read_desc_index_abs = (m_state->m_d2h_read_desc_index_abs + desc_num) & m_state->m_descs.size_mask; + + // prepare descriptors for next recv + if (m_state->m_is_channel_activated) { + status = prepare_descriptors(count, first_desc_interrupts_domain, last_desc_interrupts_domain); + if (HAILO_STREAM_NOT_ACTIVATED == status) { + LOGGER__INFO("transfer d2h failed because stream is not activated"); + return status; + } + CHECK_SUCCESS(status); + } + + m_state->m_accumulated_transfers = (m_state->m_accumulated_transfers + 1) % m_transfers_per_axi_intr; + + return HAILO_SUCCESS; +} + +hailo_status BufferedChannel::prepare_descriptors(size_t transfer_size, InterruptsDomain first_desc_interrupts_domain, + InterruptsDomain last_desc_interrupts_domain) +{ + if (!m_state->m_is_channel_activated) { + return HAILO_STREAM_NOT_ACTIVATED; + } + + // Calculate desired descriptors for the buffer + size_t desired_desc_num = m_desc_list->descriptors_in_buffer(transfer_size); + assert(desired_desc_num <= MAX_DESCS_COUNT); + uint16_t desc_num = static_cast(desired_desc_num); + + int num_available = get_num_available(); + int num_processed = CB_TAIL(m_state->m_descs); + int num_free = CB_AVAIL(m_state->m_descs, num_available, num_processed); + if (num_free < desc_num) { + return HAILO_OUT_OF_DESCRIPTORS; + } + + if (nullptr != m_latency_meter) { + // Program first descriptor + m_desc_list->program_single_descriptor((*m_desc_list)[num_available], m_desc_list->desc_page_size(), + first_desc_interrupts_domain); + } + auto actual_desc_count = m_desc_list->program_last_descriptor(transfer_size, last_desc_interrupts_domain, + num_available, true); + if (!actual_desc_count) { + LOGGER__ERROR("Failed to program desc_list for channel {}", m_channel_id); + return actual_desc_count.status(); + } + assert (actual_desc_count.value() == desc_num); + int last_desc_avail = ((num_available + desc_num - 1) & m_state->m_descs.size_mask); + + m_state->add_pending_buffer(num_available, last_desc_avail, m_direction, m_transfer_done_callback); + return inc_num_available(desc_num); +} + +bool BufferedChannel::is_ready_for_write(const uint16_t desired_desc_num) +{ + const auto has_space_in_buffers = !m_state->m_pending_buffers.full(); + const uint32_t desc_avail = (get_num_available() + m_pending_num_avail_offset) & m_state->m_descs.size_mask; + const int num_free = CB_AVAIL(m_state->m_descs, desc_avail, CB_TAIL(m_state->m_descs)); + const auto has_desc_space = (num_free >= desired_desc_num); + + return (has_space_in_buffers && has_desc_space); +} + +bool BufferedChannel::is_ready_for_transfer_d2h(size_t buffer_size) +{ + size_t desired_desc_num = m_desc_list->descriptors_in_buffer(buffer_size); + assert(desired_desc_num <= MAX_DESCS_COUNT); + int desc_num = static_cast(desired_desc_num); + + if (m_state->m_pending_buffers.full()) { + return false; + } + + int num_processed = CB_TAIL(m_state->m_descs); + int num_ready = CB_PROG(m_state->m_descs, num_processed, m_state->m_d2h_read_desc_index); + if (num_ready < desc_num) { + return false; + } + return true; +} + +void BufferedChannel::notify_all() +{ + { + // Acquire mutex to make sure the notify_all will wake the blocking threads on the cv + std::lock_guard state_guard(m_state->mutex()); + } + m_state->transfer_buffer_cv().notify_all(); +} + +} /* namespace vdma */ +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/channel/buffered_channel.hpp b/hailort/libhailort/src/vdma/channel/buffered_channel.hpp new file mode 100644 index 0000000..d46ba7f --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/buffered_channel.hpp @@ -0,0 +1,90 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file buffered_channel.hpp + * @brief BufferedChannel - Implements the BoundaryChannel interface, allowing for buffering of frames + * by managing a vdma buffer + **/ + +#ifndef _HAILO_VDMA_BUFFERED_CHANNEL_HPP_ +#define _HAILO_VDMA_BUFFERED_CHANNEL_HPP_ + +#include "hailo/hailort.h" +#include "hailo/dma_mapped_buffer.hpp" + +#include "vdma/channel/boundary_channel.hpp" + + +namespace hailort { +namespace vdma { + +class BufferedChannel; +using BufferedChannelPtr = std::shared_ptr; + +class BufferedChannel : public BoundaryChannel +{ +public: + static Expected create(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, + uint32_t descs_count, uint16_t desc_page_size, const std::string &stream_name = "", LatencyMeterPtr latency_meter = nullptr, + uint16_t transfers_per_axi_intr = 1); + + BufferedChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, uint32_t descs_count, + uint16_t desc_page_size, const std::string &stream_name, LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr, hailo_status &status); + BufferedChannel(const BufferedChannel &other) = delete; + BufferedChannel &operator=(const BufferedChannel &other) = delete; + BufferedChannel(BufferedChannel &&other) = delete; + BufferedChannel &operator=(BufferedChannel &&other) = delete; + virtual ~BufferedChannel() = default; + + virtual hailo_status transfer(void *buf, size_t count) override; + // Either write_buffer + send_pending_buffer or transfer (h2d) should be used on a given channel, not both + virtual hailo_status write_buffer(const MemoryView &buffer, std::chrono::milliseconds timeout, + const std::function &should_cancel) override; + virtual hailo_status send_pending_buffer() override; + virtual hailo_status transfer(std::shared_ptr, const TransferDoneCallback &, void *) override; + virtual hailo_status cancel_pending_transfers() override; + virtual hailo_status complete_channel_activation(uint32_t transfer_size, bool resume_pending_transfers) override; + virtual hailo_status complete_channel_deactivation() override; + + // Assumes that the channel is idle; doesn't block changes to the channel + // To be used for debugging purposes + virtual Expected get_buffer_state() override; + virtual Expected get_h2d_pending_frames_count() override; + virtual Expected get_d2h_pending_descs_count() override; + + virtual void notify_all() override; + +private: + static Expected> create_mapped_buffer(uint32_t descs_count, uint16_t desc_page_size, + Direction direction, HailoRTDriver &driver); + + hailo_status transfer_h2d(void *buf, size_t count); + hailo_status write_buffer_impl(const MemoryView &buffer); + hailo_status write_to_channel_buffer_cyclic(const MemoryView &buffer, size_t channel_buffer_write_offset); + hailo_status read_from_channel_buffer_cyclic(uint8_t *dest_buffer, size_t read_size, size_t channel_buffer_read_offset); + hailo_status send_pending_buffer_impl(); + hailo_status transfer_d2h(void *buf, size_t count); + hailo_status prepare_descriptors(size_t transfer_size, InterruptsDomain first_desc_interrupts_domain, + InterruptsDomain last_desc_interrupts_domain); + hailo_status prepare_d2h_pending_descriptors(uint32_t transfer_size, uint32_t transfers_count); + bool is_ready_for_write(const uint16_t desired_desc_num); + virtual bool is_ready_for_transfer_d2h(size_t buffer_size) override; + hailo_status store_channel_buffer_state(); + + // TODO: m_channel_buffer gets bound to ChannelBase::m_desc_list meaning the desc in that list point to dma addrs + // that back m_channel_buffer. Because ChannelBase gets dtor'd after BufferedChannel, m_channel_buffer ChannelBase::m_desc_list + // will point to a freed buffer. This is ok because the channel objects only get dtor'd after they are deactivated by the fw. + // Might want to enforce this in hailort as well (e.g. desc lists can hold shared_ptrs to DmaMappedBuffer while they are bound). + // (HRT-9110) + std::shared_ptr m_channel_buffer; + // Using CircularArray because it won't allocate or free memory wile pushing and popping. The fact that it is circular is not relevant here + CircularArray m_pending_buffers_sizes; + std::atomic_uint16_t m_pending_num_avail_offset; +}; + +} /* namespace vdma */ +} /* namespace hailort */ + +#endif // _HAILO_VDMA_BUFFERED_CHANNEL_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/channel/channel_base.cpp b/hailort/libhailort/src/vdma/channel/channel_base.cpp new file mode 100644 index 0000000..4c233fd --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/channel_base.cpp @@ -0,0 +1,219 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file channel_base.cpp + * @brief Base class of Boundary Channel - responsible for all the basic vdma channel functionality that interacts with the + * driver and the registers + * The hierarchy is as follows: + * -------------------------------------------------------------------------------------------------------------- + * | ChannelBase | (Base class - includes state and buffers) + * | | | + * | BoundaryChannel | (handles Boundary channels) + * -------------------------------------------------------------------------------------------------------------- + **/ +#include "vdma/channel/channel_base.hpp" + + +namespace hailort { +namespace vdma { + +ChannelBase::ChannelBase(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, uint32_t descs_count, + uint16_t desc_page_size, const std::string &stream_name, LatencyMeterPtr latency_meter, hailo_status &status) : + m_channel_id(channel_id), + m_direction(direction), + m_driver(driver), + m_host_registers(driver, channel_id, direction), + m_desc_list(nullptr), + m_stream_name(stream_name), + m_latency_meter(latency_meter) +{ + if (channel_id.channel_index >= VDMA_CHANNELS_PER_ENGINE) { + LOGGER__ERROR("Invalid DMA channel index {}", channel_id.channel_index); + status = HAILO_INVALID_ARGUMENT; + return; + } + + if (channel_id.engine_index >= driver.dma_engines_count()) { + LOGGER__ERROR("Invalid DMA engine index {}, max {}", channel_id.engine_index, driver.dma_engines_count()); + status = HAILO_INVALID_ARGUMENT; + return; + } + + if (descs_count > MAX_DESCS_COUNT) { + LOGGER__ERROR("Vdma channel descs_count mustn't be larger than {}", MAX_DESCS_COUNT); + status = HAILO_INVALID_ARGUMENT; + return; + } + + auto state = VdmaChannelState::create(descs_count, (nullptr != m_latency_meter)); + if(!state) { + LOGGER__ERROR("Failed to create channel's state"); + status = state.status(); + return; + } + m_state = state.release(); + + // Allocate descriptor list (host side) + status = allocate_descriptor_list(descs_count, desc_page_size); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to allocate Vdma buffer for channel transfer! status={}", status); + return; + } + + status = HAILO_SUCCESS; +} + +hailo_status ChannelBase::set_num_avail_value(uint16_t new_value) +{ + // TODO - HRT-7885 : add check in driver + CHECK(m_state->m_is_channel_activated, HAILO_STREAM_NOT_ACTIVATED, + "Error, can't set num available when stream is not activated"); + + auto status = m_host_registers.set_num_available(new_value); + CHECK_SUCCESS(status, "Fail to write vdma num available register"); + +#ifndef NDEBUG + // Validate synchronization with HW + auto hw_num_avail = m_host_registers.get_num_available(); + assert(hw_num_avail); + assert(hw_num_avail.value() == new_value); +#endif + return HAILO_SUCCESS; +} + +hailo_status ChannelBase::inc_num_available(uint16_t value) +{ + //TODO: validate that count is added. + int num_available = get_num_available(); + int num_processed = CB_TAIL(m_state->m_descs); + int num_free = CB_AVAIL(m_state->m_descs, num_available, num_processed); + if (value > num_free) { + return HAILO_OUT_OF_DESCRIPTORS; + } + + CB_ENQUEUE(m_state->m_descs, value); + num_available = (num_available + value) & m_state->m_descs.size_mask; + return set_num_avail_value(static_cast(num_available)); +} + +bool ChannelBase::is_desc_between(uint16_t begin, uint16_t end, uint16_t desc) +{ + if (begin == end) { + // There is nothing between + return false; + } + if (begin < end) { + // desc needs to be in [begin, end) + return (begin <= desc) && (desc < end); + } + else { + // desc needs to be in [0, end) or [begin, m_state->m_descs.size()-1] + return (desc < end) || (begin <= desc); + } +} + +uint16_t ChannelBase::get_num_available() +{ + uint16_t num_available = (uint16_t)CB_HEAD(m_state->m_descs); + +#ifndef NDEBUG + // Validate synchronization with HW + auto hw_num_avail = m_host_registers.get_num_available(); + assert(hw_num_avail); + + // On case of channel aborted, the num_available is set to 0 (so we don't accept sync) + auto is_aborted_exp = m_host_registers.is_aborted(); + assert(is_aborted_exp); + + if (m_state->m_is_channel_activated && !is_aborted_exp.value()) { + assert(hw_num_avail.value() == num_available); + } +#endif + return num_available; +} + +Expected ChannelBase::get_hw_num_processed() +{ + auto hw_num_processed = m_host_registers.get_num_processed(); + CHECK_EXPECTED(hw_num_processed, "Fail to read vdma num processed register"); + + // Although the hw_num_processed should be a number between 0 and m_descs.size-1, if + // m_desc.size < 0x10000 (the maximum desc size), the actual hw_num_processed is a number + // between 1 and m_descs.size. Therefore the value can be m_descs.size, in this case we change it + // to zero. + return static_cast(hw_num_processed.value() & m_state->m_descs.size_mask); +} + +ChannelBase::Direction ChannelBase::other_direction(Direction direction) +{ + return (Direction::H2D == direction) ? Direction::D2H : Direction::H2D; +} + +hailo_status ChannelBase::allocate_descriptor_list(uint32_t descs_count, uint16_t desc_page_size) +{ + auto desc_page_size_value = m_driver.calc_desc_page_size(desc_page_size); + CHECK(is_powerof2(desc_page_size_value), HAILO_INVALID_ARGUMENT, "Descriptor page_size must be a power of two."); + + auto desc_list_exp = DescriptorList::create(descs_count, desc_page_size_value, m_driver); + CHECK_EXPECTED_AS_STATUS(desc_list_exp); + + m_desc_list = make_shared_nothrow(desc_list_exp.release()); + CHECK_NOT_NULL(m_desc_list, HAILO_OUT_OF_HOST_MEMORY); + + return HAILO_SUCCESS; +} + +size_t ChannelBase::get_transfers_count_in_buffer(size_t transfer_size) +{ + const auto descs_in_transfer = m_desc_list->descriptors_in_buffer(transfer_size); + const auto descs_count = CB_SIZE(m_state->m_descs); + return (descs_count - 1) / descs_in_transfer; +} + +Expected ChannelBase::update_latency_meter() +{ + uint16_t last_num_processed = m_state->m_last_timestamp_num_processed; + + auto timestamp_list = m_driver.vdma_interrupts_read_timestamps(m_channel_id); + CHECK_EXPECTED(timestamp_list); + + if (0 == timestamp_list->count) { + // No new timestamps for this channel, return the previous result + return Expected(last_num_processed); + } + + // TODO: now we have more iterations than we need. We know that the pending buffers + the timestamp list + // are ordered. If pending_buffer[i] is not in any of the timestamps_list[0, 1, ... k], then also pending_buffer[i+1,i+2,...] + // not in those timestamps + + for (const auto &pending_buffer : m_state->m_pending_buffers) { + uint16_t latency_desc = static_cast(pending_buffer.latency_measure_desc); + for (size_t i = 0; i < timestamp_list->count; i++) { + const auto &irq_timestamp = timestamp_list->timestamp_list[i]; + const auto desc_num_processed = static_cast(irq_timestamp.desc_num_processed & m_state->m_descs.size_mask); + if (is_desc_between(last_num_processed, desc_num_processed, latency_desc)) { + if (m_direction == Direction::H2D) { + m_latency_meter->add_start_sample(irq_timestamp.timestamp); + } + else { + m_latency_meter->add_end_sample(m_stream_name, irq_timestamp.timestamp); + } + break; + } + } + } + + m_state->m_last_timestamp_num_processed = static_cast( + timestamp_list->timestamp_list[timestamp_list->count-1].desc_num_processed & m_state->m_descs.size_mask); + return Expected(m_state->m_last_timestamp_num_processed); +} + +uint32_t ChannelBase::calculate_descriptors_count(uint32_t buffer_size) const +{ + return DescriptorList::calculate_descriptors_count(buffer_size, 1, m_desc_list->desc_page_size()); +} + +} /* namespace vdma */ +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/channel/channel_base.hpp b/hailort/libhailort/src/vdma/channel/channel_base.hpp new file mode 100644 index 0000000..5f56b81 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/channel_base.hpp @@ -0,0 +1,103 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file channel_base.hpp + * @brief Base class of Boundary Channel - responsible for all the basic vdma channel functionality that interacts with the + * driver and the registers + * The hierarchy is as follows: + * -------------------------------------------------------------------------------------------------------------- + * | ChannelBase | (Base class - includes state and buffers) + * | | | + * | BoundaryChannel | (handles Boundary channels) + * -------------------------------------------------------------------------------------------------------------- + **/ + +#ifndef _HAILO_VDMA_CHANNEL_BASE_HPP_ +#define _HAILO_VDMA_CHANNEL_BASE_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" +#include "hailo/buffer.hpp" + +#include "common/latency_meter.hpp" + +#include "vdma/channel/vdma_channel_regs.hpp" +#include "vdma/memory/sg_buffer.hpp" +#include "vdma/memory/descriptor_list.hpp" +#include "vdma/channel/channel_id.hpp" +#include "vdma/channel/channel_state.hpp" + +#include +#include + + +namespace hailort { +namespace vdma { + +class ChannelBase +{ +public: + using Direction = HailoRTDriver::DmaDirection; + + ChannelBase(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, uint32_t descs_count, + uint16_t desc_page_size, const std::string &stream_name, LatencyMeterPtr latency_meter, hailo_status &status); + ChannelBase(const ChannelBase &other) = delete; + ChannelBase &operator=(const ChannelBase &other) = delete; + ChannelBase(ChannelBase &&other) = delete; + ChannelBase &operator=(ChannelBase &&other) = delete; + virtual ~ChannelBase() = default; + + vdma::ChannelId get_channel_id() const + { + return m_channel_id; + } + + uint16_t get_page_size() + { + return m_desc_list->desc_page_size(); + } + + const std::string &stream_name() const + { + return m_stream_name; + } + + size_t get_transfers_count_in_buffer(size_t transfer_size); + size_t get_buffer_size() const; + uint32_t calculate_descriptors_count(uint32_t buffer_size) const; + + std::shared_ptr get_desc_list() + { + return m_desc_list; + } + +protected: + const vdma::ChannelId m_channel_id; + const Direction m_direction; + HailoRTDriver &m_driver; + VdmaChannelRegs m_host_registers; + std::shared_ptr m_desc_list; // Host side descriptor list + const std::string m_stream_name; + std::unique_ptr m_state; + LatencyMeterPtr m_latency_meter; + + static bool is_desc_between(uint16_t begin, uint16_t end, uint16_t desc); + // Returns the desc index of the last desc whose timestamp was measured in the driver + Expected update_latency_meter(); + Expected is_aborted(); + hailo_status set_num_avail_value(uint16_t new_value); + uint16_t get_num_available(); + Expected get_hw_num_processed(); + hailo_status inc_num_available(uint16_t value); + static Direction other_direction(const Direction direction); + +private: + hailo_status allocate_descriptor_list(uint32_t descs_count, uint16_t desc_page_size); +}; + +} /* namespace vdma */ +} /* namespace hailort */ + +#endif /* _HAILO_VDMA_CHANNEL_BASE_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/channel_id.hpp b/hailort/libhailort/src/vdma/channel/channel_id.hpp similarity index 100% rename from hailort/libhailort/src/vdma/channel_id.hpp rename to hailort/libhailort/src/vdma/channel/channel_id.hpp diff --git a/hailort/libhailort/src/vdma/channel/channel_state.cpp b/hailort/libhailort/src/vdma/channel/channel_state.cpp new file mode 100644 index 0000000..0880f04 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/channel_state.cpp @@ -0,0 +1,240 @@ +#include "vdma/channel/channel_state.hpp" + + +namespace hailort { +namespace vdma { + +#ifndef _MSC_VER +RecursiveSharedMutex::RecursiveSharedMutex() +{ + // Make sharable mutex + pthread_mutexattr_t mutex_attrs{}; + int err = pthread_mutexattr_init(&mutex_attrs); + if (0 != err) { + LOGGER__CRITICAL("Failed init mutex attr, aborting"); + std::abort(); + } + + err = pthread_mutexattr_setpshared(&mutex_attrs, PTHREAD_PROCESS_SHARED); + if (0 != err) { + LOGGER__CRITICAL("pthread_mutexattr_setpshared failed"); + std::abort(); + } + + err = pthread_mutexattr_settype(&mutex_attrs, PTHREAD_MUTEX_RECURSIVE); + if (0 != err) { + LOGGER__CRITICAL("pthread_mutexattr_settype failed"); + std::abort(); + } + + err = pthread_mutex_init(&m_mutex, &mutex_attrs); + if (0 != pthread_mutexattr_destroy(&mutex_attrs)) { + LOGGER__CRITICAL("Failed destroy mutexattr"); + // continue + } + if (0 != err) { + LOGGER__CRITICAL("Failed init mutex, aborting"); + std::abort(); + } +} + +RecursiveSharedMutex::~RecursiveSharedMutex() +{ + int err = pthread_mutex_destroy(&m_mutex); + if (0 != err) { + LOGGER__ERROR("Failed destroy shared mutex, errno {}", err); + } +} + +void RecursiveSharedMutex::lock() +{ + int err = pthread_mutex_lock(&m_mutex); + if (0 != err) { + LOGGER__ERROR("Failed lock shared mutex, errno {}", err); + std::abort(); + } +} + +void RecursiveSharedMutex::unlock() +{ + int err = pthread_mutex_unlock(&m_mutex); + if (0 != err) { + LOGGER__ERROR("Failed unlock shared mutex, errno {}", err); + std::abort(); + } +} + +SharedConditionVariable::SharedConditionVariable() +{ + // Make sharable condvar + pthread_condattr_t cond_attrs{}; + int err = pthread_condattr_init(&cond_attrs); + if (0 != err) { + LOGGER__CRITICAL("Failed init condition variable attr, aborting"); + std::abort(); + } + + err = pthread_condattr_setpshared(&cond_attrs, PTHREAD_PROCESS_SHARED); + if (0 != err) { + LOGGER__CRITICAL("pthread_condattr_setpshared failed"); + std::abort(); + } + + err = pthread_condattr_setclock(&cond_attrs, CLOCK_MONOTONIC); + if (0 != err) { + LOGGER__CRITICAL("pthread_condattr_setclock failed"); + std::abort(); + } + + err = pthread_cond_init(&m_cond, &cond_attrs); + if (0 != pthread_condattr_destroy(&cond_attrs)) { + LOGGER__CRITICAL("Failed destroy condattr"); + // continue + } + if (0 != err) { + LOGGER__CRITICAL("Failed init mutex, aborting"); + std::abort(); + } +} + +SharedConditionVariable::~SharedConditionVariable() +{ + int err = pthread_cond_destroy(&m_cond); + if (0 != err) { + LOGGER__ERROR("Failed destory vdma channel condition varialbe, errno {}", err); + } +} + +// Get the absolute time for the given timeout - calculate now() + timeout_ns +// using system CLOCK_MONOTONIC (Used for pthread condition variable wait) +static struct timespec get_absolute_time(std::chrono::nanoseconds timeout_ns) +{ + // Using chrono with timespec types to avoid casts + using ts_seconds = std::chrono::duration; + using ts_nanoseconds = std::chrono::duration; + + struct timespec current_ts{}; + clock_gettime(CLOCK_MONOTONIC, ¤t_ts); + + assert((current_ts.tv_sec + std::chrono::duration_cast(timeout_ns).count()) < + std::numeric_limits::max()); + auto absolute_sec = ts_seconds(current_ts.tv_sec) + std::chrono::duration_cast(timeout_ns); + assert(current_ts.tv_nsec <= std::nano::den); + auto absolute_nsec = ts_nanoseconds(current_ts.tv_nsec) + + std::chrono::duration_cast(timeout_ns % std::chrono::seconds(1)); + + // Nanos overflow + if (absolute_nsec.count() >= std::nano::den) { + absolute_sec += ts_seconds(1); + absolute_nsec = absolute_nsec % ts_seconds(1); + } + + return timespec { + .tv_sec = absolute_sec.count(), + .tv_nsec = absolute_nsec.count() + }; +} + +bool SharedConditionVariable::wait_for(std::unique_lock &lock, std::chrono::milliseconds timeout, std::function condition) +{ + if (UINT32_MAX == timeout.count()) { + // Infinity wait + int err = 0; + while (!condition() && err == 0) { + err = pthread_cond_wait(&m_cond, lock.mutex()->native_handle()); + } + if (err != 0) { + LOGGER__CRITICAL("Error waiting for shared condition variable: {}", err); + std::abort(); + } + return true; + } + else if (0 == timeout.count()) { + // Special case for 0 timeout - we don't want to mess with absolute time + return condition(); + } else { + // Timed wait + auto ts = get_absolute_time(timeout); + + int err = 0; + while (!condition() && err == 0) { + err = pthread_cond_timedwait(&m_cond, lock.mutex()->native_handle(), &ts); + } + if ((err != 0) && (err != ETIMEDOUT)) { + LOGGER__CRITICAL("Error waiting for shared condition variable: {}", err); + std::abort(); + } + return err == 0; + } +} + +void SharedConditionVariable::notify_one() +{ + pthread_cond_signal(&m_cond); +} + +void SharedConditionVariable::notify_all() +{ + pthread_cond_broadcast(&m_cond); +} + +#endif /* _MSC_VER */ + +Expected> VdmaChannelState::create(uint32_t descs_count, bool measure_latency) +{ + // Note: we implement operator new so the state object will be shared with forked processes. + auto state = make_unique_nothrow(descs_count, measure_latency); + CHECK_NOT_NULL_AS_EXPECTED(state, HAILO_OUT_OF_HOST_MEMORY); + return state; +} + +VdmaChannelState::VdmaChannelState(uint32_t descs_count, bool measure_latency) : + m_is_channel_activated(false), + // If we measuring latency, we may get 2 interrupts for each input channel (first descriptor and last descriptor). + // Hence we must limit the transfers count to half of the actual transfers count. + m_pending_buffers(measure_latency ? PENDING_BUFFERS_SIZE/2 : PENDING_BUFFERS_SIZE), + m_d2h_read_desc_index(0), + m_d2h_read_desc_index_abs(0), + m_is_aborted(false), + m_previous_tail(0), + m_desc_list_delta(0), + m_last_timestamp_num_processed(0), + m_accumulated_transfers(0) +{ + CB_INIT(m_descs, descs_count); +} + +void VdmaChannelState::reset_counters() +{ + CB_RESET(m_descs); + m_pending_buffers.reset(); + m_last_timestamp_num_processed = 0; + m_accumulated_transfers = 0; +} + +void VdmaChannelState::reset_previous_state_counters() +{ + m_previous_tail = 0; + m_desc_list_delta = 0; + m_d2h_read_desc_index = 0; + m_d2h_read_desc_index_abs = 0; +} + +void VdmaChannelState::add_pending_buffer(uint32_t first_desc, uint32_t last_desc, HailoRTDriver::DmaDirection direction, + const TransferDoneCallback &on_transfer_done, std::shared_ptr buffer, void *opaque) +{ + if (m_pending_buffers.full()) { + // TODO- HRT-8900 : Fix log and check if should return error + LOGGER__ERROR("no avail space"); + } + PendingBuffer pending_buffer{}; + pending_buffer.last_desc = last_desc; + pending_buffer.latency_measure_desc = (direction == HailoRTDriver::DmaDirection::H2D) ? first_desc : last_desc; + pending_buffer.on_transfer_done = on_transfer_done; + pending_buffer.buffer = buffer; + pending_buffer.opaque = opaque; + m_pending_buffers.push_back(std::move(pending_buffer)); +} + +} /* namespace vdma */ +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/channel/channel_state.hpp b/hailort/libhailort/src/vdma/channel/channel_state.hpp new file mode 100644 index 0000000..ece1e27 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/channel_state.hpp @@ -0,0 +1,178 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file channel_state.hpp + * @brief Current state of Vdma Channel + * + * + **/ + +#ifndef _HAILO_VDMA_CHANNEL_STATE_HPP_ +#define _HAILO_VDMA_CHANNEL_STATE_HPP_ + +#include "hailo/hailort.h" +#include "os/hailort_driver.hpp" +#include "common/circular_buffer.hpp" +#include "hailo/dma_mapped_buffer.hpp" +#include "hailo/stream.hpp" + +#include +#include + +#ifndef _MSC_VER +#include +#endif + + +namespace hailort { +namespace vdma { + +struct PendingBuffer { + uint32_t last_desc; + uint32_t latency_measure_desc; + TransferDoneCallback on_transfer_done; + std::shared_ptr buffer; + void *opaque; +}; + +class ChannelBase; +class BoundaryChannel; +class AsyncChannel; +class BufferedChannel; + + +#ifndef _MSC_VER +// Special mutex and condition variable objects that can be shared between forked processes (Not needed on windows, +// because there is no fork). +class RecursiveSharedMutex final { +public: + RecursiveSharedMutex(); + ~RecursiveSharedMutex(); + + RecursiveSharedMutex(const RecursiveSharedMutex &) = delete; + RecursiveSharedMutex &operator=(const RecursiveSharedMutex &) = delete; + RecursiveSharedMutex(RecursiveSharedMutex &&) = delete; + RecursiveSharedMutex &operator=(RecursiveSharedMutex &&) = delete; + + void lock(); + void unlock(); + + pthread_mutex_t *native_handle() + { + return &m_mutex; + } + +private: + pthread_mutex_t m_mutex; +}; + +class SharedConditionVariable final { +public: + + SharedConditionVariable(); + ~SharedConditionVariable(); + + SharedConditionVariable(const SharedConditionVariable &) = delete; + SharedConditionVariable &operator=(const SharedConditionVariable &) = delete; + SharedConditionVariable(SharedConditionVariable &&) = delete; + SharedConditionVariable &operator=(SharedConditionVariable &&) = delete; + + bool wait_for(std::unique_lock &lock, std::chrono::milliseconds timeout, std::function condition); + void notify_one(); + void notify_all(); + +private: + pthread_cond_t m_cond; +}; +#else /* _MSC_VER */ +using RecursiveSharedMutex = std::recursive_mutex; +using SharedConditionVariable = std::condition_variable_any; +#endif + +class VdmaChannelState final +{ +public: + static Expected> create(uint32_t descs_count, bool measure_latency); + + VdmaChannelState(uint32_t descs_count, bool measure_latency); + VdmaChannelState(const VdmaChannelState &other) = delete; + VdmaChannelState(VdmaChannelState &&other) = delete; + ~VdmaChannelState() = default; + + void reset_counters(); + void reset_previous_state_counters(); + // Each transfer on the channel is logged by a PendingBuffer: + // - first_desc/last_desc - first and last descriptors of the transfer + // - direction - transfer's direction + // - on_transfer_done - callback to be called once the transfer is complete (i.e. when an interrupt is received on last_desc) + // - buffer - points to the vdma mapped buffer being transferred (may be null) + // - opaque - context to be transferred to the callback (may be null) + void add_pending_buffer(uint32_t first_desc, uint32_t last_desc, HailoRTDriver::DmaDirection direction, + const TransferDoneCallback &on_transfer_done, std::shared_ptr buffer = nullptr, void *opaque = nullptr); + + RecursiveSharedMutex &mutex() + { + return m_state_lock; + } + + SharedConditionVariable &transfer_buffer_cv() + { + return m_can_transfer_buffer_cv; + } + +#ifndef _MSC_VER + // The VdmaChannelState must remain in a shared memory scope, so we implement the new/delete operators (only on + // non-windows machines). + void* operator new(std::size_t size) = delete; + void* operator new(std::size_t size, const std::nothrow_t&) throw() { + // Map a shared memory region into the virtual memory of the process + void* ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED) { + return nullptr; + } + return ptr; + } + + // Custom operator delete function that unmaps the shared memory region + void operator delete(void* ptr, std::size_t size) { + munmap(ptr, size); + } +#endif /* _MSC_VER */ + + friend class ChannelBase; + friend class BoundaryChannel; + friend class AsyncChannel; + friend class BufferedChannel; + +private: + RecursiveSharedMutex m_state_lock; + SharedConditionVariable m_can_transfer_buffer_cv; + + bool m_is_channel_activated; + + // On pending buffer with must use std::array because it relays on the shared memory (and std::vector uses new malloc) + CircularArray> m_pending_buffers; + // TODO: describe why we must have our own num_available and num_proc. + // it's not just for efficiency but its critical to avoid a potential bug - see Avigail email. + // TODO: Consider C11 stdatomic + circbuf_t m_descs; + // m_d2h_read_desc_index and m_d2h_read_desc_index_abs are the index of the first desc containing frames to be + // copied to the user ("ready" frames in a D2H buffered channel). m_d2h_read_desc_index is relative to the + // first desc in the desc list, whereas m_d2h_read_desc_index_abs is relative to the start of the vdma buffer. + int m_d2h_read_desc_index; + int m_d2h_read_desc_index_abs; + bool m_is_aborted; + // Points to the tail of the desc list when the channel is stopped (starts at zero) + int m_previous_tail; + int m_desc_list_delta; + // Contains the last num_processed of the last interrupt (only used on latency measurement) + uint16_t m_last_timestamp_num_processed; + size_t m_accumulated_transfers; +}; + +} /* namespace hailort */ +} /* namespace hailort */ + +#endif /* _HAILO_VDMA_CHANNEL_STATE_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.cpp b/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.cpp new file mode 100644 index 0000000..99ad909 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.cpp @@ -0,0 +1,106 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file interrupts_dispatcher.cpp + * @brief Manages a thread that is waiting for channel interrupts. + **/ + +#include "interrupts_dispatcher.hpp" +#include "hailo/hailort_common.hpp" +#include "common/os_utils.hpp" + +namespace hailort { +namespace vdma { + +Expected> InterruptsDispatcher::create(std::reference_wrapper driver) +{ + auto thread = make_unique_nothrow(driver); + CHECK_NOT_NULL_AS_EXPECTED(thread, HAILO_OUT_OF_HOST_MEMORY); + return thread; +} + +InterruptsDispatcher::InterruptsDispatcher(std::reference_wrapper driver) : + m_driver(driver), + m_is_running(false), + m_channels_bitmap() +{} + +InterruptsDispatcher::~InterruptsDispatcher() +{ + if (m_is_running) { + stop(); + } +} + +hailo_status InterruptsDispatcher::start(const ChannelsBitmap &channels_bitmap, bool enable_timestamp_measure, + const ProcessIrqCallback &process_irq) +{ + CHECK(!m_is_running, HAILO_INVALID_OPERATION, "Interrupt thread already running"); + assert(m_channel_threads.empty()); + assert(m_channels_bitmap == ChannelsBitmap{}); + + m_channels_bitmap = channels_bitmap; + + auto status = m_driver.get().vdma_interrupts_enable(m_channels_bitmap, enable_timestamp_measure); + CHECK_SUCCESS(status, "Failed to enable vdma interrupts"); + + // Setting m_is_running will allow the threads to run + m_is_running = true; + m_channel_threads.emplace_back([this, process_irq]() { + // m_channels_bitmap may be changed by InterruptsDispatcher::stop. To avoid wait for 0 channels, + // we use copy of m_channels_bitmap. + ChannelsBitmap channels_bitmap_local = m_channels_bitmap; + wait_interrupts(channels_bitmap_local, process_irq); + }); + + return HAILO_SUCCESS; +} + +hailo_status InterruptsDispatcher::stop() +{ + CHECK(m_is_running, HAILO_INVALID_OPERATION, "Interrupts thread not started"); + assert(!m_channel_threads.empty()); + assert(m_channels_bitmap != ChannelsBitmap{}); + + // Signal threads to stop execution + m_is_running = false; + + // Calling disable interrupts will cause the vdma_interrupts_wait to return. + auto status = m_driver.get().vdma_interrupts_disable(m_channels_bitmap); + CHECK_SUCCESS(status, "Failed to disable vdma interrupts"); + + m_channels_bitmap = ChannelsBitmap{}; + for (auto &thread : m_channel_threads) { + if (thread.joinable()) { + thread.join(); + } + } + m_channel_threads.clear(); + + return HAILO_SUCCESS; +} + +void InterruptsDispatcher::wait_interrupts(const ChannelsBitmap &channels_bitmap, const ProcessIrqCallback &process_irq) +{ + OsUtils::set_current_thread_name("CHANNEL_INTR"); + while (m_is_running) { + // vdma_interrupts_wait is a blocking function that returns in this scenarios: + // 1. We got a new interrupts, irq_data will be passed to the process_irq callback + // 2. vdma_interrupts_disable will be called, vdma_interrupts_wait will return with an empty list. + // 3. Other error returns - shouldn't really happen, we exit the interrupt thread. + auto irq_data = m_driver.get().vdma_interrupts_wait(channels_bitmap); + if (!irq_data.has_value()) { + LOGGER__ERROR("Interrupt thread exit with {}", irq_data.status()); + break; + } + + if (irq_data->channels_count > 0) { + process_irq(irq_data.release()); + } + } +} + +} /* namespace vdma */ +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.hpp b/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.hpp new file mode 100644 index 0000000..c02f428 --- /dev/null +++ b/hailort/libhailort/src/vdma/channel/interrupts_dispatcher.hpp @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file interrupts_dispatcher.hpp + * @brief Manages a thread that is waiting for channel interrupts. + **/ + +#ifndef _HAILO_VDMA_INTERRUPTS_DISPATCHER_HPP_ +#define _HAILO_VDMA_INTERRUPTS_DISPATCHER_HPP_ + +#include "os/hailort_driver.hpp" +#include +#include + +namespace hailort { +namespace vdma { + + +/// When needed, creates thread (or threads) that waits for interrupts on all channels. +class InterruptsDispatcher final { +public: + // The actual irq process callback, should run quickly (blocks the interrupts thread). + using ProcessIrqCallback = std::function; + + static Expected> create(std::reference_wrapper driver); + explicit InterruptsDispatcher(std::reference_wrapper driver); + ~InterruptsDispatcher(); + + InterruptsDispatcher(const InterruptsDispatcher &) = delete; + InterruptsDispatcher &operator=(const InterruptsDispatcher &) = delete; + InterruptsDispatcher(InterruptsDispatcher &&) = delete; + InterruptsDispatcher &operator=(InterruptsDispatcher &&) = delete; + + // TODO: HRT-9590 remove interrupt_thread_per_channel, use it by default + hailo_status start(const ChannelsBitmap &channels_bitmap, bool enable_timestamp_measure, + const ProcessIrqCallback &process_irq); + hailo_status stop(); + +private: + + void wait_interrupts(const ChannelsBitmap &channels_bitmap, const ProcessIrqCallback &process_irq); + + const std::reference_wrapper m_driver; + std::atomic m_is_running; + ChannelsBitmap m_channels_bitmap; + std::vector m_channel_threads; +}; + +} /* namespace vdma */ +} /* namespace hailort */ + +#endif /* _HAILO_VDMA_INTERRUPTS_DISPATCHER_HPP_ */ diff --git a/hailort/libhailort/src/vdma_channel_regs.hpp b/hailort/libhailort/src/vdma/channel/vdma_channel_regs.hpp similarity index 64% rename from hailort/libhailort/src/vdma_channel_regs.hpp rename to hailort/libhailort/src/vdma/channel/vdma_channel_regs.hpp index dc0bb7c..afd95be 100644 --- a/hailort/libhailort/src/vdma_channel_regs.hpp +++ b/hailort/libhailort/src/vdma/channel/vdma_channel_regs.hpp @@ -21,15 +21,12 @@ namespace hailort #define DESCPRIPTOR_LIST_MAX_DEPTH (16) + inline bool vdma_channel_control_is_aborted(uint8_t control_reg) { return (control_reg & 1) == 0; } -inline bool vdma_channel_control_is_paused(uint8_t control_reg) -{ - return (control_reg & 2) == 2; -} class VdmaChannelRegs final { public: @@ -39,11 +36,6 @@ public: m_direction(direction) {} - Expected get_control() - { - return read_integer(VDMA_CHANNEL_CONTROL_OFFSET); - } - Expected get_num_available() { return read_integer(VDMA_CHANNEL_NUM_AVAIL_OFFSET); @@ -59,28 +51,14 @@ public: return read_integer(VDMA_CHANNEL_NUM_PROC_OFFSET); } - Expected get_channel_error() - { - return read_integer(VDMA_CHANNEL_ERROR_OFFSET); - } - - hailo_status stop_channel() +#ifndef NDEBUG + Expected is_aborted() { - auto reg_control = get_control(); - CHECK_EXPECTED_AS_STATUS(reg_control, "Fail to read vdma control register"); - - // First pause channel - auto status = set_control((reg_control.value() & 0xFC) | 0x3); - CHECK_SUCCESS(status, "Fail to write vdma control register"); - - std::this_thread::sleep_for(std::chrono::microseconds(2)); - - // Then abort - status = set_control((reg_control.value() & 0xFC) | 0x0); - CHECK_SUCCESS(status, "Fail to write vdma control register"); - - return HAILO_SUCCESS; + const auto control_reg = read_integer(VDMA_CHANNEL_CONTROL_OFFSET); + CHECK_EXPECTED(control_reg); + return vdma_channel_control_is_aborted(*control_reg); } +#endif /* NDEBUG */ private: @@ -92,11 +70,6 @@ private: return static_cast(value.release()); } - hailo_status set_control(uint8_t value) - { - return write_integer(VDMA_CHANNEL_CONTROL_OFFSET, value); - } - template hailo_status write_integer(uint32_t offset, IntegerType value) { diff --git a/hailort/libhailort/src/vdma/integrated/integrated_device.cpp b/hailort/libhailort/src/vdma/integrated/integrated_device.cpp new file mode 100644 index 0000000..b6406d5 --- /dev/null +++ b/hailort/libhailort/src/vdma/integrated/integrated_device.cpp @@ -0,0 +1,78 @@ +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "common/logger_macros.hpp" + +#include "vdma/integrated/integrated_device.hpp" +#include "vdma/vdma_config_manager.hpp" + +#include "md5.h" +#include + +static const std::string INTEGRATED_NNC_DRIVER_PATH = "/dev/hailo_integrated_nnc"; + +namespace hailort +{ + +bool IntegratedDevice::is_loaded() +{ +#if defined(_MSC_VER) + // windows is not supported for core driver + return false; +#else + return (access(INTEGRATED_NNC_DRIVER_PATH.c_str(), F_OK) == 0); +#endif // defined(_MSC_VER) +} + +Expected> IntegratedDevice::create() +{ + hailo_status status = HAILO_UNINITIALIZED; + + auto driver = HailoRTDriver::create(INTEGRATED_NNC_DRIVER_PATH); + CHECK_EXPECTED(driver, "Failed to initialize HailoRTDriver"); + + auto device = std::unique_ptr(new (std::nothrow) IntegratedDevice(driver.release(), status, DEVICE_ID)); + CHECK_AS_EXPECTED((nullptr != device), HAILO_OUT_OF_HOST_MEMORY); + CHECK_SUCCESS_AS_EXPECTED(status, "Failed creating IntegratedDevice"); + + return device; +} + + +IntegratedDevice::IntegratedDevice(HailoRTDriver &&driver, hailo_status &status, const std::string &device_id) : + VdmaDevice::VdmaDevice(std::move(driver), Device::Type::INTEGRATED, device_id) +{ + status = update_fw_state(); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("update_fw_state() failed with status {}", status); + return; + } + + status = HAILO_SUCCESS; +} + +Expected IntegratedDevice::get_architecture() const { + return Expected(m_device_architecture); +} + +hailo_status IntegratedDevice::reset_impl(CONTROL_PROTOCOL__reset_type_t reset_type) +{ + if (CONTROL_PROTOCOL__RESET_TYPE__NN_CORE == reset_type) { + return m_driver.reset_nn_core(); + } + + LOGGER__ERROR("Can't reset IntegratedDevice, please use linux reboot"); + return HAILO_NOT_IMPLEMENTED; +} + +Expected IntegratedDevice::read_log(MemoryView &buffer, hailo_cpu_id_t cpu_id) +{ + if (hailo_cpu_id_t::HAILO_CPU_ID_0 == cpu_id) { + LOGGER__ERROR("Read FW log is supported only on core CPU"); + return make_unexpected(HAILO_INVALID_ARGUMENT); + } + + return VdmaDevice::read_log(buffer, cpu_id); +} + +} /* namespace hailort */ \ No newline at end of file diff --git a/hailort/libhailort/src/core_device.hpp b/hailort/libhailort/src/vdma/integrated/integrated_device.hpp similarity index 68% rename from hailort/libhailort/src/core_device.hpp rename to hailort/libhailort/src/vdma/integrated/integrated_device.hpp index 05014fc..5bb07fe 100644 --- a/hailort/libhailort/src/core_device.hpp +++ b/hailort/libhailort/src/vdma/integrated/integrated_device.hpp @@ -3,28 +3,30 @@ * Distributed under the MIT license (https://opensource.org/licenses/MIT) **/ /** - * @file core_device + * @file integrated_device * @brief Device used by Hailo-15 * **/ -#ifndef _HAILO_CORE_DEVICE_HPP_ -#define _HAILO_CORE_DEVICE_HPP_ +#ifndef _HAILO_INTEGRATED_DEVICE_HPP_ +#define _HAILO_INTEGRATED_DEVICE_HPP_ #include "hailo/expected.hpp" #include "hailo/hailort.h" -#include "vdma_device.hpp" + +#include "vdma/vdma_device.hpp" #include + namespace hailort { -class CoreDevice : public VdmaDevice { +class IntegratedDevice : public VdmaDevice { public: - virtual ~CoreDevice() = default; + virtual ~IntegratedDevice() = default; static bool is_loaded(); - static Expected> create(); + static Expected> create(); virtual Expected get_architecture() const override; virtual const char* get_dev_id() const override {return DEVICE_ID;} @@ -33,7 +35,7 @@ public: virtual bool is_stream_interface_supported(const hailo_stream_interface_t &stream_interface) const override { switch (stream_interface) { - case HAILO_STREAM_INTERFACE_CORE: + case HAILO_STREAM_INTERFACE_INTEGRATED: return true; case HAILO_STREAM_INTERFACE_PCIE: case HAILO_STREAM_INTERFACE_ETH: @@ -45,16 +47,16 @@ public: } } - static constexpr const char *DEVICE_ID = "[core]"; + static constexpr const char *DEVICE_ID = "[integrated]"; protected: virtual hailo_status reset_impl(CONTROL_PROTOCOL__reset_type_t reset_type) override; private: - CoreDevice(HailoRTDriver &&driver, hailo_status &status, const std::string &device_id); + IntegratedDevice(HailoRTDriver &&driver, hailo_status &status, const std::string &device_id); }; } /* namespace hailort */ -#endif /* _HAILO_CORE_DEVICE_HPP_ */ \ No newline at end of file +#endif /* _HAILO_INTEGRATED_DEVICE_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/mapped_buffer.cpp b/hailort/libhailort/src/vdma/mapped_buffer.cpp deleted file mode 100644 index bfe0e40..0000000 --- a/hailort/libhailort/src/vdma/mapped_buffer.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include "mapped_buffer.hpp" - -namespace hailort { -namespace vdma { - -Expected MappedBuffer::create(size_t required_size, HailoRTDriver::DmaDirection data_direction, - HailoRTDriver &driver) -{ - hailo_status status = HAILO_UNINITIALIZED; - MappedBuffer object(required_size, data_direction, driver, status); - if (HAILO_SUCCESS != status) { - return make_unexpected(status); - } - - return object; -} - -MappedBuffer::MappedBuffer( - size_t required_size, HailoRTDriver::DmaDirection data_direction, HailoRTDriver &driver, hailo_status &status) - : m_size(required_size), m_driver(driver) -{ - auto buffer = VdmaMappedBufferImpl::allocate_vdma_buffer(driver, required_size); - if (! buffer) { - status = buffer.status(); - return; - } - - auto expected_handle = m_driver.vdma_buffer_map(buffer->get(), required_size, data_direction, - buffer->get_mapped_buffer_identifier()); - if (!expected_handle) { - status = expected_handle.status(); - return; - } - - m_vdma_mapped_buffer = make_unique_nothrow(buffer.release()); - if (nullptr == m_vdma_mapped_buffer) { - m_driver.vdma_buffer_unmap(expected_handle.value()); - status = HAILO_OUT_OF_HOST_MEMORY; - return; - } - - m_handle = expected_handle.release(); - status = HAILO_SUCCESS; -} - -MappedBuffer::~MappedBuffer() -{ - if (m_vdma_mapped_buffer && *m_vdma_mapped_buffer) { - m_driver.vdma_buffer_unmap(m_handle); - } -} - -hailo_status MappedBuffer::write(const void *buf_src, size_t count, size_t offset) -{ - if ((count + offset) > m_size) { - LOGGER__ERROR("Requested size {} from offset {} is more than the MappedBuffer size {}", count, offset, m_size); - return HAILO_INSUFFICIENT_BUFFER; - } - - if (count > 0) { - auto dst_vdma_address = (uint8_t*)m_vdma_mapped_buffer->get() + offset; - memcpy(dst_vdma_address, buf_src, count); - - auto status = m_driver.vdma_buffer_sync(m_handle, HailoRTDriver::DmaDirection::H2D, dst_vdma_address, count); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed synching vdma buffer on write"); - return status; - } - } - - return HAILO_SUCCESS; -} - -hailo_status MappedBuffer::read(void *buf_dst, size_t count, size_t offset, bool should_sync) -{ - if ((count + offset) > m_size) { - LOGGER__ERROR("Requested size {} from offset {} is more than the MappedBuffer size {}", count, offset, m_size); - return HAILO_INSUFFICIENT_BUFFER; - } - - if (count > 0) { - const auto dst_vdma_address = (uint8_t*)m_vdma_mapped_buffer->get() + offset; - if (should_sync) { - const auto status = m_driver.vdma_buffer_sync(m_handle, HailoRTDriver::DmaDirection::D2H, dst_vdma_address, count); - CHECK_SUCCESS(status, "Failed synching vdma buffer on read"); - } - - memcpy(buf_dst, dst_vdma_address, count); - } - - return HAILO_SUCCESS; -} - -hailo_status MappedBuffer::write_cyclic(const void *buf_src, size_t count, size_t offset) -{ - if (count > m_size) { - LOGGER__ERROR("Requested size({}) is more than the MappedBuffer size {}", count, m_size); - return HAILO_INSUFFICIENT_BUFFER; - } - - auto size_to_end = m_size - offset; - auto copy_size = std::min(size_to_end, count); - auto status = write(buf_src, copy_size, offset); - if (HAILO_SUCCESS != status) { - return status; - } - - auto remaining_size = count - copy_size; - if (remaining_size > 0) { - status = write((uint8_t*)buf_src + copy_size, remaining_size, 0); - if (HAILO_SUCCESS != status) { - return status; - } - } - - return HAILO_SUCCESS; -} - -hailo_status MappedBuffer::read_cyclic(void *buf_dst, size_t count, size_t offset, bool should_sync) -{ - if (count > m_size) { - LOGGER__ERROR("Requested size({}) is more than the MappedBuffer size {}", count, m_size); - return HAILO_INSUFFICIENT_BUFFER; - } - - auto size_to_end = m_size - offset; - auto copy_size = std::min(size_to_end, count); - auto status = read(buf_dst, copy_size, offset, should_sync); - if (HAILO_SUCCESS != status) { - return status; - } - - auto remaining_size = count - copy_size; - if (remaining_size > 0) { - status = read((uint8_t*)buf_dst + copy_size, remaining_size, 0, should_sync); - if (HAILO_SUCCESS != status) { - return status; - } - } - - return HAILO_SUCCESS; -} - -} /* namespace vdma */ -} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/mapped_buffer.hpp b/hailort/libhailort/src/vdma/mapped_buffer.hpp deleted file mode 100644 index 5fa81d7..0000000 --- a/hailort/libhailort/src/vdma/mapped_buffer.hpp +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file mapped_buffer.hpp - * @brief The mapped buffer that is continuous in virtual memory, but not on physical memory. - * We map the buffer to the IOMMU. - * - * The buffer can be used only with the help of a descriptors list that contains pointers to a physical - * continuous "dma pages". - * - * There are 2 options to allocated the buffer: - * 1. User mode allocation - the user mode calls `malloc` or `mmap` to allocate the buffer, then - * using HailoRTDriver we map the driver to the IOMMU (and pin the pages to avoid pagigs). - * This is the default option - * 2. Kernel mode allocation - on some systems, the user mode doesn't allocate the memory in a "dma-able" address, - * so we need to allocate the pages in driver. - **/ - -#ifndef _HAILO_VDMA_MAPPED_BUFFER_HPP_ -#define _HAILO_VDMA_MAPPED_BUFFER_HPP_ - -#include "os/mmap_buffer.hpp" -#include "os/hailort_driver.hpp" -#include "hailo/expected.hpp" -#include "vdma_mapped_buffer_impl.hpp" - -namespace hailort { -namespace vdma { - -class MappedBuffer final -{ -public: - static Expected create(size_t required_size, HailoRTDriver::DmaDirection data_direction, - HailoRTDriver &driver); - - MappedBuffer(size_t required_size, HailoRTDriver::DmaDirection data_direction, - HailoRTDriver &driver, hailo_status &status); - ~MappedBuffer(); - - MappedBuffer(const MappedBuffer &other) = delete; - MappedBuffer &operator=(const MappedBuffer &other) = delete; - MappedBuffer(MappedBuffer &&other) noexcept = default; - MappedBuffer &operator=(MappedBuffer &&other) = delete; - - void *user_address() { return m_vdma_mapped_buffer->get(); } - HailoRTDriver::VdmaBufferHandle handle() { return m_handle; } - size_t size() const { return m_size; } - - /** - * Copy data from buf_src parameter to this MappedBuffer. - * - * @note (offset + count) MUST be smaller than this MappedBuffer size - * - * @param[in] buf_src The buffer to copy the data from - * @param[in] count Number of bytes to copy from buf_src - * @param[in] offset The offset relative to this MappedBuffer to copy the data to - */ - hailo_status write(const void *buf_src, size_t count, size_t offset); - - /** - * Copy data from this MappedBuffer to buf_dst. - * - * @note (offset + count) MUST be smaller than this MappedBuffer size - * - * @param[out] buf_dst The buffer to copy the data to - * @param[in] count Number of bytes to copy to buf_dst - * @param[in] offset The offset relative to this MappedBuffer to copy the data from - * @param[in] should_sync If the backing memory is vdma and it's written to by a device, sync should be true - * so that the read will be consistent with the backing memory - */ - hailo_status read(void *buf_dst, size_t count, size_t offset, bool should_sync = true); - - /** - * Copy data from buf_src parameter to this MappedBuffer. - * - * Similar to 'write' but if (offset + count) is larger than the MappedBuffer size, the copy continues - * from the start of the MappedBuffer. - * - * @note count MUST be smaller than this MappedBuffer size - * - * @param[in] buf_src The buffer to copy the data from - * @param[in] count Number of bytes to copy from buf_src - * @param[in] offset The offset relative to this MappedBuffer to copy the data to - */ - hailo_status write_cyclic(const void *buf_src, size_t count, size_t offset); - - /** - * Copy data from this MappedBuffer to buf_dst. - * - * Similar to 'read' but if (offset + count) is larger than the MappedBuffer size, the copy continues - * from the start of the MappedBuffer. - * - * @note count MUST be smaller than this MappedBuffer size - * - * @param[out] buf_dst The buffer to copy the data to - * @param[in] count Number of bytes to copy to buf_dst - * @param[in] offset The offset relative to this MappedBuffer to copy the data from - * @param[in] should_sync If the backing memory is vdma and it's written to by a device, sync should be true - * so that the read will be consistent with the backing memory - */ - hailo_status read_cyclic(void *buf_dst, size_t count, size_t offset, bool should_sync = true); - -private: - - std::unique_ptr m_vdma_mapped_buffer; - HailoRTDriver::VdmaBufferHandle m_handle; - size_t m_size; - HailoRTDriver &m_driver; -}; - -} /* namespace vdma */ -} /* namespace hailort */ - -#endif /* _HAILO_VDMA_MAPPED_BUFFER_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/continuous_buffer.cpp b/hailort/libhailort/src/vdma/memory/continuous_buffer.cpp similarity index 89% rename from hailort/libhailort/src/vdma/continuous_buffer.cpp rename to hailort/libhailort/src/vdma/memory/continuous_buffer.cpp index 53a26da..bff2809 100644 --- a/hailort/libhailort/src/vdma/continuous_buffer.cpp +++ b/hailort/libhailort/src/vdma/memory/continuous_buffer.cpp @@ -4,7 +4,7 @@ **/ /** * @file continuous_buffer.hpp - * @brief Continuous physical vdma buffer. + * @brief Continuous physical vdma buffer. **/ #include "continuous_buffer.hpp" @@ -53,7 +53,7 @@ uint32_t ContinuousBuffer::get_buffer_size(uint32_t buffer_size) uint32_t ContinuousBuffer::get_buffer_size_desc_power2(uint32_t buffer_size) { const uint16_t page_size = DEFAULT_DESC_PAGE_SIZE; - const auto descriptors_in_buffer = DESCRIPTORS_IN_BUFFER(buffer_size, page_size); + const auto descriptors_in_buffer = DIV_ROUND_UP(buffer_size, page_size); const auto actual_descriptors_count = get_nearest_powerof_2(descriptors_in_buffer, MIN_CCB_DESCRIPTORS_COUNT); return actual_descriptors_count * page_size; } @@ -101,7 +101,7 @@ hailo_status ContinuousBuffer::read(void *buf_dst, size_t count, size_t offset, CHECK((count + offset) <= m_size, HAILO_INSUFFICIENT_BUFFER, "Requested size {} from offset {} is more than the buffer size {}", count, offset, m_size); // We use dma coherent mmap, so no need to sync the buffer after the memcpy. - const auto src_address = reinterpret_cast(m_mmap.get()) + offset; + const auto src_address = reinterpret_cast(m_mmap.address()) + offset; memcpy(buf_dst, src_address, count); return HAILO_SUCCESS; } @@ -111,15 +111,14 @@ hailo_status ContinuousBuffer::write(const void *buf_src, size_t count, size_t o CHECK((count + offset) <= m_size, HAILO_INSUFFICIENT_BUFFER, "Requested size {} from offset {} is more than the buffer size {}", count, offset, m_size); // We use dma coherent mmap, so no need to sync the buffer after the memcpy. - const auto dst_address = reinterpret_cast(m_mmap.get()) + offset; + const auto dst_address = reinterpret_cast(m_mmap.address()) + offset; memcpy(dst_address, buf_src, count); return HAILO_SUCCESS; } -Expected ContinuousBuffer::program_descriptors(size_t transfer_size, VdmaInterruptsDomain first_desc_interrupts_domain, - VdmaInterruptsDomain last_desc_interrupts_domain, size_t desc_offset, bool is_circular) +Expected ContinuousBuffer::program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, + size_t desc_offset, bool is_circular) { - (void)first_desc_interrupts_domain; (void)last_desc_interrupts_domain; (void)desc_offset; (void)is_circular; @@ -129,7 +128,7 @@ Expected ContinuousBuffer::program_descriptors(size_t transfer_size, V } hailo_status ContinuousBuffer::reprogram_device_interrupts_for_end_of_batch(size_t transfer_size, uint16_t batch_size, - VdmaInterruptsDomain new_interrupts_domain) + InterruptsDomain new_interrupts_domain) { (void)transfer_size; (void)batch_size; diff --git a/hailort/libhailort/src/vdma/continuous_buffer.hpp b/hailort/libhailort/src/vdma/memory/continuous_buffer.hpp similarity index 90% rename from hailort/libhailort/src/vdma/continuous_buffer.hpp rename to hailort/libhailort/src/vdma/memory/continuous_buffer.hpp index 0e5f088..58afefb 100644 --- a/hailort/libhailort/src/vdma/continuous_buffer.hpp +++ b/hailort/libhailort/src/vdma/memory/continuous_buffer.hpp @@ -12,7 +12,8 @@ #include "os/hailort_driver.hpp" #include "os/mmap_buffer.hpp" -#include "vdma/vdma_buffer.hpp" +#include "vdma/memory/vdma_buffer.hpp" + namespace hailort { namespace vdma { @@ -53,10 +54,10 @@ public: virtual hailo_status read(void *buf_dst, size_t count, size_t offset, bool should_sync) override; virtual hailo_status write(const void *buf_src, size_t count, size_t offset) override; - virtual Expected program_descriptors(size_t transfer_size, VdmaInterruptsDomain first_desc_interrupts_domain, - VdmaInterruptsDomain last_desc_interrupts_domain, size_t desc_offset, bool is_circular) override; + virtual Expected program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, + size_t desc_offset, bool is_circular) override; virtual hailo_status reprogram_device_interrupts_for_end_of_batch(size_t transfer_size, uint16_t batch_size, - VdmaInterruptsDomain new_interrupts_domain) override; + InterruptsDomain new_interrupts_domain) override; private: ContinuousBuffer(size_t size, HailoRTDriver &driver, uintptr_t handle, uint64_t dma_address, diff --git a/hailort/libhailort/src/vdma_descriptor_list.cpp b/hailort/libhailort/src/vdma/memory/descriptor_list.cpp similarity index 64% rename from hailort/libhailort/src/vdma_descriptor_list.cpp rename to hailort/libhailort/src/vdma/memory/descriptor_list.cpp index b160246..995a27c 100644 --- a/hailort/libhailort/src/vdma_descriptor_list.cpp +++ b/hailort/libhailort/src/vdma/memory/descriptor_list.cpp @@ -1,4 +1,17 @@ -#include "vdma_descriptor_list.hpp" +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file descriptor_list.cpp + * @brief Implements vdma descriptor list class + **/ + +#include "vdma/memory/descriptor_list.hpp" +#include "vdma/memory/mapped_buffer_impl.hpp" + +#include "utils.h" + #define DESC_STATUS_REQ (1 << 0) #define DESC_STATUS_REQ_ERR (1 << 1) @@ -15,23 +28,26 @@ #define DESC_PAGE_SIZE_MASK (0xFFFFFF00) #define DESC_IRQ_MASK (0x0000003C) -namespace hailort -{ +namespace hailort { +namespace vdma { + -Expected VdmaDescriptorList::create(uint32_t desc_count, uint16_t requested_desc_page_size, +Expected DescriptorList::create(uint32_t desc_count, uint16_t requested_desc_page_size, HailoRTDriver &driver) { hailo_status status = HAILO_UNINITIALIZED; auto desc_page_size_value = driver.calc_desc_page_size(requested_desc_page_size); - VdmaDescriptorList object(desc_count, driver, desc_page_size_value, status); + DescriptorList object(desc_count, driver, desc_page_size_value, status); if (HAILO_SUCCESS != status) { return make_unexpected(status); } + // No need to initialize descripotrs here because they are initialized in driver in hailo_vdma_program_descriptor() + return object; } -VdmaDescriptorList::VdmaDescriptorList(uint32_t desc_count, HailoRTDriver &driver, uint16_t desc_page_size, +DescriptorList::DescriptorList(uint32_t desc_count, HailoRTDriver &driver, uint16_t desc_page_size, hailo_status &status) : m_mapped_list(), m_count(desc_count), @@ -74,7 +90,7 @@ VdmaDescriptorList::VdmaDescriptorList(uint32_t desc_count, HailoRTDriver &drive status = HAILO_SUCCESS; } -VdmaDescriptorList::~VdmaDescriptorList() +DescriptorList::~DescriptorList() { if (HAILO_SUCCESS != m_mapped_list.unmap()) { LOGGER__ERROR("Failed to release descriptors mapping"); @@ -88,7 +104,7 @@ VdmaDescriptorList::~VdmaDescriptorList() } } -VdmaDescriptorList::VdmaDescriptorList(VdmaDescriptorList &&other) noexcept : +DescriptorList::DescriptorList(DescriptorList &&other) noexcept : m_mapped_list(std::move(other.m_mapped_list)), m_count(std::move(other.m_count)), m_depth(std::move(other.m_depth)), @@ -97,7 +113,7 @@ VdmaDescriptorList::VdmaDescriptorList(VdmaDescriptorList &&other) noexcept : m_driver(other.m_driver), m_desc_page_size(other.m_desc_page_size) {} -Expected VdmaDescriptorList::calculate_desc_list_depth(size_t count) +Expected DescriptorList::calculate_desc_list_depth(size_t count) { // Calculate log2 of m_count (by finding the offset of the MSB) uint32_t depth = 0; @@ -108,46 +124,40 @@ Expected VdmaDescriptorList::calculate_desc_list_depth(size_t count) return static_cast(depth); } -hailo_status VdmaDescriptorList::configure_to_use_buffer(vdma::MappedBuffer& buffer, uint8_t channel_index, size_t offset) +hailo_status DescriptorList::configure_to_use_buffer(DmaMappedBuffer& buffer, ChannelId channel_id, uint32_t starting_desc) { - return m_driver.descriptors_list_bind_vdma_buffer(m_desc_handle, buffer.handle(), m_desc_page_size, - channel_index, offset); -} + const auto desc_list_capacity = m_desc_page_size * m_count; + CHECK(buffer.size() <= desc_list_capacity, HAILO_INVALID_ARGUMENT, + "Can't bind a buffer larger than the descriptor list's capacity. Buffer size {}, descriptor list capacity {}", + buffer.size(), desc_list_capacity); -hailo_status VdmaDescriptorList::configure_to_use_buffer(vdma::MappedBuffer& buffer, size_t offset) -{ - return configure_to_use_buffer(buffer, HailoRTDriver::INVALID_VDMA_CHANNEL_INDEX, offset); + return m_driver.descriptors_list_bind_vdma_buffer(m_desc_handle, buffer.pimpl->handle(), m_desc_page_size, + channel_id.channel_index, starting_desc); } -Expected VdmaDescriptorList::program_descriptors(size_t transfer_size, - VdmaInterruptsDomain first_desc_interrupts_domain, VdmaInterruptsDomain last_desc_interrupts_domain, - size_t desc_offset, bool is_circular) +Expected DescriptorList::program_last_descriptor(size_t transfer_size, + InterruptsDomain last_desc_interrupts_domain, size_t desc_offset, bool is_circular) { assert(transfer_size > 0); const auto required_descriptors = descriptors_in_buffer(transfer_size); - // Required_descriptors + desc_offset can't reach m_count. We need to keep at least 1 free desc at all time. - if ((!is_circular) && ((required_descriptors + desc_offset) >= m_count)){ + // Required_descriptors + desc_offset can't reach m_count. + if ((!is_circular) && ((required_descriptors + desc_offset) > m_count)){ LOGGER__ERROR("Requested transfer size ({}) result in more descriptors than available ({})", transfer_size, m_count); return make_unexpected(HAILO_OUT_OF_DESCRIPTORS); } - size_t desc_index = desc_offset; - for (size_t i = 0; i < required_descriptors - 1; ++i) { - const auto interrupts_domain = (i == 0) ? first_desc_interrupts_domain : VdmaInterruptsDomain::NONE; - program_single_descriptor((*this)[desc_index], m_desc_page_size, interrupts_domain); - desc_index = (desc_index + 1) & (m_count - 1); - } - + // Program last descriptor of the transfer size /* write residue page with the remaining buffer size*/ auto resuide = transfer_size - (required_descriptors - 1) * m_desc_page_size; assert(IS_FIT_IN_UINT16(resuide)); - program_single_descriptor((*this)[desc_index], static_cast(resuide), last_desc_interrupts_domain); + size_t last_desc = (desc_offset + required_descriptors - 1) & (m_count - 1); + program_single_descriptor((*this)[last_desc], static_cast(resuide), last_desc_interrupts_domain); return std::move(static_cast(required_descriptors)); } -hailo_status VdmaDescriptorList::reprogram_descriptor_interrupts_domain(size_t desc_index, - VdmaInterruptsDomain interrupts_domain) +hailo_status DescriptorList::reprogram_descriptor_interrupts_domain(size_t desc_index, + InterruptsDomain interrupts_domain) { if (desc_index >= m_count){ LOGGER__ERROR("Requested desc (index={}) exceeds the number of descriptors in the list ({})", desc_index, m_count); @@ -157,18 +167,18 @@ hailo_status VdmaDescriptorList::reprogram_descriptor_interrupts_domain(size_t d return HAILO_SUCCESS; } -uint32_t VdmaDescriptorList::descriptors_in_buffer(size_t buffer_size) const +uint32_t DescriptorList::descriptors_in_buffer(size_t buffer_size) const { return descriptors_in_buffer(buffer_size, m_desc_page_size); } -uint32_t VdmaDescriptorList::descriptors_in_buffer(size_t buffer_size, uint16_t desc_page_size) +uint32_t DescriptorList::descriptors_in_buffer(size_t buffer_size, uint16_t desc_page_size) { assert(buffer_size < std::numeric_limits::max()); - return static_cast(DESCRIPTORS_IN_BUFFER(buffer_size, desc_page_size)); + return static_cast(DIV_ROUND_UP(buffer_size, desc_page_size)); } -uint32_t VdmaDescriptorList::calculate_descriptors_count(uint32_t buffer_size, uint16_t batch_size, uint16_t desc_page_size) +uint32_t DescriptorList::calculate_descriptors_count(uint32_t buffer_size, uint16_t batch_size, uint16_t desc_page_size) { // Because we use cyclic buffer, the amount of active descs is lower by one that the amount // of descs given (Otherwise we won't be able to determine if the buffer is empty or full). @@ -179,7 +189,7 @@ uint32_t VdmaDescriptorList::calculate_descriptors_count(uint32_t buffer_size, u return get_nearest_powerof_2(descs_count, MIN_DESCS_COUNT); } -Expected> VdmaDescriptorList::get_desc_buffer_sizes_for_single_transfer( +Expected> DescriptorList::get_desc_buffer_sizes_for_single_transfer( const HailoRTDriver &driver, uint16_t min_batch_size, uint16_t max_batch_size, uint32_t transfer_size) { // Note: If the pages pointed to by the descriptors are copied in their entirety, then DEFAULT_DESC_PAGE_SIZE @@ -199,30 +209,30 @@ Expected> VdmaDescriptorList::get_desc_buffer_size static_cast(initial_desc_page_size)); } -Expected> VdmaDescriptorList::get_desc_buffer_sizes_for_multiple_transfers( +Expected> DescriptorList::get_desc_buffer_sizes_for_multiple_transfers( const HailoRTDriver &driver, uint16_t batch_size, const std::vector &transfer_sizes) { return get_desc_buffer_sizes_for_multiple_transfers_impl(driver, batch_size, transfer_sizes, DEFAULT_DESC_PAGE_SIZE); } -Expected> VdmaDescriptorList::get_desc_buffer_sizes_for_single_transfer_impl( +Expected> DescriptorList::get_desc_buffer_sizes_for_single_transfer_impl( const HailoRTDriver &driver, uint16_t min_batch_size, uint16_t max_batch_size, uint32_t transfer_size, uint16_t initial_desc_page_size) { - auto results = VdmaDescriptorList::get_desc_buffer_sizes_for_multiple_transfers_impl(driver, min_batch_size, + auto results = DescriptorList::get_desc_buffer_sizes_for_multiple_transfers_impl(driver, min_batch_size, {transfer_size}, initial_desc_page_size); CHECK_EXPECTED(results); auto page_size = results->first; auto desc_count = std::min(MAX_DESCS_COUNT, - VdmaDescriptorList::calculate_descriptors_count(transfer_size, max_batch_size, page_size)); + DescriptorList::calculate_descriptors_count(transfer_size, max_batch_size, page_size)); return std::make_pair(page_size, desc_count); } -Expected> VdmaDescriptorList::get_desc_buffer_sizes_for_multiple_transfers_impl( +Expected> DescriptorList::get_desc_buffer_sizes_for_multiple_transfers_impl( const HailoRTDriver &driver, uint16_t batch_size, const std::vector &transfer_sizes, uint16_t initial_desc_page_size) { @@ -239,11 +249,7 @@ Expected> VdmaDescriptorList::get_desc_buffer_size "Initial descriptor page size ({}) is smaller than minimum descriptor page size ({})", local_desc_page_size, min_desc_page_size); - uint32_t acc_desc_count = 0; - for (const auto &transfer_size : transfer_sizes) { - acc_desc_count += - VdmaDescriptorList::descriptors_in_buffer(transfer_size, static_cast(local_desc_page_size)); - } + uint32_t acc_desc_count = get_descriptors_count_needed(transfer_sizes, static_cast(local_desc_page_size)); // Too many descriptors; try a larger desc_page_size which will lead to less descriptors used while ((acc_desc_count * batch_size) > (MAX_DESCS_COUNT - 1)) { @@ -251,18 +257,14 @@ Expected> VdmaDescriptorList::get_desc_buffer_size CHECK_AS_EXPECTED(local_desc_page_size <= max_desc_page_size, HAILO_OUT_OF_DESCRIPTORS, "Network shapes and batch size exceeds driver descriptors capabilities. " - "Required descriptors count: {}, max allowed on the driver: {}. (A common cause for this error could be the" - "Batch size - which is {}).", - (batch_size * acc_desc_count), MAX_DESCS_COUNT, batch_size); + "Required descriptors count: {}, max allowed on the driver: {}. " + "(A common cause for this error could be the batch size - which is {}).", + (batch_size * acc_desc_count), (MAX_DESCS_COUNT - 1), batch_size); CHECK_AS_EXPECTED(IS_FIT_IN_UINT16(local_desc_page_size), HAILO_INTERNAL_FAILURE, "Descriptor page size needs to fit in 16B"); - acc_desc_count = 0; - for (auto &transfer_size : transfer_sizes) { - acc_desc_count += - VdmaDescriptorList::descriptors_in_buffer(transfer_size, static_cast(local_desc_page_size)); - } + acc_desc_count = get_descriptors_count_needed(transfer_sizes, static_cast(local_desc_page_size)); } // Found desc_page_size and acc_desc_count @@ -279,7 +281,20 @@ Expected> VdmaDescriptorList::get_desc_buffer_size return std::make_pair(desc_page_size, descs_count); } -uint32_t VdmaDescriptorList::get_interrupts_bitmask(VdmaInterruptsDomain interrupts_domain) +uint32_t DescriptorList::get_descriptors_count_needed(const std::vector &transfer_sizes, + uint16_t desc_page_size) +{ + uint32_t desc_count = 0; + for (auto &transfer_size : transfer_sizes) { + desc_count += descriptors_in_buffer(transfer_size, desc_page_size); + } + + // One extra descriptor is needed, because the amount of available descriptors is (desc_count - 1) + desc_count += 1; + return desc_count; +} + +uint32_t DescriptorList::get_interrupts_bitmask(InterruptsDomain interrupts_domain) { uint32_t host_bitmask = 0; uint32_t device_bitmask = 0; @@ -308,40 +323,54 @@ uint32_t VdmaDescriptorList::get_interrupts_bitmask(VdmaInterruptsDomain interru return bitmask; } -void VdmaDescriptorList::program_single_descriptor(VdmaDescriptor &descriptor, uint16_t page_size, - VdmaInterruptsDomain interrupts_domain) +void DescriptorList::program_single_descriptor(VdmaDescriptor &descriptor, uint16_t page_size, + InterruptsDomain interrupts_domain) { - descriptor.PageSize_DescControl = 0; // Update the descriptor's PAGE_SIZE field in the control register with the maximum size of the DMA page. - descriptor.PageSize_DescControl |= - (uint32_t)(page_size << DESC_PAGE_SIZE_SHIFT) & (uint32_t)DESC_PAGE_SIZE_MASK; + // Make all edits to the local variable local_pagesize_desc_ctrl that is on the stack to save read/writes to DDR + auto local_pagesize_desc_ctrl = static_cast(page_size << DESC_PAGE_SIZE_SHIFT) & DESC_PAGE_SIZE_MASK; - if (VdmaInterruptsDomain::NONE != interrupts_domain) { + if (InterruptsDomain::NONE != interrupts_domain) { // Update the desc_control - descriptor.PageSize_DescControl |= (DESC_REQUREST_IRQ_PROCESSED | DESC_REQUREST_IRQ_ERR | + local_pagesize_desc_ctrl |= (DESC_REQUREST_IRQ_PROCESSED | DESC_REQUREST_IRQ_ERR | get_interrupts_bitmask(interrupts_domain)); #ifndef NDEBUG - descriptor.PageSize_DescControl |= (DESC_STATUS_REQ | DESC_STATUS_REQ_ERR); + local_pagesize_desc_ctrl |= (DESC_STATUS_REQ | DESC_STATUS_REQ_ERR); #endif } + descriptor.PageSize_DescControl = local_pagesize_desc_ctrl; + +#ifndef NDEBUG // Clear status descriptor.RemainingPageSize_Status = 0; +#endif } -void VdmaDescriptorList::reprogram_single_descriptor_interrupts_domain(VdmaDescriptor &descriptor, - VdmaInterruptsDomain interrupts_domain) +void DescriptorList::reprogram_single_descriptor_interrupts_domain(VdmaDescriptor &descriptor, + InterruptsDomain interrupts_domain) { // Set the IRQ control bits to zero - descriptor.PageSize_DescControl &= ~DESC_IRQ_MASK; + // Make all edits to the local variable local_pagesize_desc_ctrl that is on the stack to save read/writes to DDR + auto local_pagesize_desc_ctrl = (descriptor.PageSize_DescControl & ~DESC_IRQ_MASK); - if (VdmaInterruptsDomain::NONE == interrupts_domain) { + if (InterruptsDomain::NONE == interrupts_domain) { // Nothing else to do + descriptor.PageSize_DescControl = local_pagesize_desc_ctrl; return; } - descriptor.PageSize_DescControl |= (DESC_REQUREST_IRQ_PROCESSED | DESC_REQUREST_IRQ_ERR | + local_pagesize_desc_ctrl |= (DESC_REQUREST_IRQ_PROCESSED | DESC_REQUREST_IRQ_ERR | get_interrupts_bitmask(interrupts_domain)); + + descriptor.PageSize_DescControl = local_pagesize_desc_ctrl; +} + +void DescriptorList::clear_descriptor(const size_t desc_index) +{ + // Clear previous descriptor properties + program_single_descriptor((*this)[desc_index], m_desc_page_size, InterruptsDomain::NONE); } +} /* namespace vdma */ } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma_descriptor_list.hpp b/hailort/libhailort/src/vdma/memory/descriptor_list.hpp similarity index 66% rename from hailort/libhailort/src/vdma_descriptor_list.hpp rename to hailort/libhailort/src/vdma/memory/descriptor_list.hpp index 6a42241..6800b32 100644 --- a/hailort/libhailort/src/vdma_descriptor_list.hpp +++ b/hailort/libhailort/src/vdma/memory/descriptor_list.hpp @@ -3,22 +3,25 @@ * Distributed under the MIT license (https://opensource.org/licenses/MIT) **/ /** - * @file vdma_descriptor_list.hpp + * @file descriptor_list.hpp * @brief Allocates a list of buffer descriptors used for VDMA - * **/ #ifndef _HAILO_VDMA_DESCRIPTOR_LIST_HPP_ #define _HAILO_VDMA_DESCRIPTOR_LIST_HPP_ -#include "os/hailort_driver.hpp" #include "hailo/expected.hpp" -#include "os/mmap_buffer.hpp" -#include "vdma/mapped_buffer.hpp" +#include "hailo/dma_mapped_buffer.hpp" + #include "common/utils.hpp" -namespace hailort -{ +#include "vdma/channel/channel_id.hpp" +#include "os/hailort_driver.hpp" +#include "os/mmap_buffer.hpp" + + +namespace hailort { +namespace vdma { #define MAX_DESCS_COUNT (64 * 1024u) @@ -40,7 +43,7 @@ static_assert(DEFAULT_DESC_COUNT <= MAX_DESCS_COUNT && DEFAULT_DESC_COUNT >= MIN #define MIN_DESC_PAGE_SIZE (64u) // TODO: Calculate from G_PAGE_SIZE_MAX (I.e. read the reg etc.) #define MAX_DESC_PAGE_SIZE (4096u) -#define DEFAULT_DESC_PAGE_SIZE (512u) +static constexpr uint16_t DEFAULT_DESC_PAGE_SIZE = 512; static_assert(is_powerof2(MIN_DESC_PAGE_SIZE), "MIN_DESC_PAGE_SIZE must be a power of 2"); static_assert(MIN_DESC_PAGE_SIZE > 0, "MIN_DESC_PAGE_SIZE must be larger then 0"); @@ -58,7 +61,7 @@ struct VdmaDescriptor uint32_t RemainingPageSize_Status; }; -enum class VdmaInterruptsDomain +enum class InterruptsDomain { NONE = 0, DEVICE = 1 << 0, @@ -66,28 +69,28 @@ enum class VdmaInterruptsDomain BOTH = DEVICE | HOST }; -inline bool host_interuptes_enabled(VdmaInterruptsDomain interrupts_domain) +inline bool host_interuptes_enabled(InterruptsDomain interrupts_domain) { - return 0 != (static_cast(interrupts_domain) & static_cast(VdmaInterruptsDomain::HOST)); + return 0 != (static_cast(interrupts_domain) & static_cast(InterruptsDomain::HOST)); } -inline bool device_interuptes_enabled(VdmaInterruptsDomain interrupts_domain) +inline bool device_interuptes_enabled(InterruptsDomain interrupts_domain) { - return 0 != (static_cast(interrupts_domain) & static_cast(VdmaInterruptsDomain::DEVICE)); + return 0 != (static_cast(interrupts_domain) & static_cast(InterruptsDomain::DEVICE)); } -class VdmaDescriptorList +class DescriptorList { public: - static Expected create(uint32_t desc_count, uint16_t requested_desc_page_size, + static Expected create(uint32_t desc_count, uint16_t requested_desc_page_size, HailoRTDriver &driver); - ~VdmaDescriptorList(); + ~DescriptorList(); - VdmaDescriptorList(const VdmaDescriptorList &other) = delete; - VdmaDescriptorList &operator=(const VdmaDescriptorList &other) = delete; - VdmaDescriptorList(VdmaDescriptorList &&other) noexcept; - VdmaDescriptorList &operator=(VdmaDescriptorList &&other) = delete; + DescriptorList(const DescriptorList &other) = delete; + DescriptorList &operator=(const DescriptorList &other) = delete; + DescriptorList(DescriptorList &&other) noexcept; + DescriptorList &operator=(DescriptorList &&other) = delete; uint8_t depth() const { @@ -120,15 +123,23 @@ public: return m_desc_handle; } - // offset in buffer to which the first desc in this VdmaDescriptorList will point - // offset must be a multiple of desc_page_size() - hailo_status configure_to_use_buffer(vdma::MappedBuffer& buffer, uint8_t channel_index, size_t offset = 0); - // On hailo8, we allow configuring buffer without specific channel index. - hailo_status configure_to_use_buffer(vdma::MappedBuffer& buffer, size_t offset = 0); + uint16_t max_transfers(uint32_t transfer_size) + { + // We need to keep at least 1 free desc at all time. + return static_cast((m_count - 1) / descriptors_in_buffer(transfer_size)); + } - Expected program_descriptors(size_t transfer_size, VdmaInterruptsDomain first_desc_interrupts_domain, - VdmaInterruptsDomain last_desc_interrupts_domain, size_t desc_offset, bool is_circular); - hailo_status reprogram_descriptor_interrupts_domain(size_t desc_index, VdmaInterruptsDomain interrupts_domain); + // Map descriptors starting at offset to the start of buffer, wrapping around the descriptor list as needed + // On hailo8, we allow configuring buffer without specific channel index (default is INVALID_VDMA_CHANNEL_INDEX). + hailo_status configure_to_use_buffer(DmaMappedBuffer& buffer, ChannelId channel_id, uint32_t starting_desc = 0); + // All descritors are initialized to have size of m_desc_page_size - so all we do is set the last descritor for the + // Interrupt - and then after transfer has finished clear the previously used first and last decsriptors. + // This saves us write/ reads to the desscriptor list which is DMA memory. + Expected program_last_descriptor(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, + size_t desc_offset, bool is_circular); + void program_single_descriptor(VdmaDescriptor &descriptor, uint16_t page_size, InterruptsDomain interrupts_domain); + hailo_status reprogram_descriptor_interrupts_domain(size_t desc_index, InterruptsDomain interrupts_domain); + void clear_descriptor(const size_t desc_index); uint32_t descriptors_in_buffer(size_t buffer_size) const; static uint32_t descriptors_in_buffer(size_t buffer_size, uint16_t desc_page_size); @@ -139,11 +150,9 @@ public: uint16_t batch_size, const std::vector &transfer_sizes); private: - VdmaDescriptorList(uint32_t desc_count, HailoRTDriver &driver, uint16_t desc_page_size, hailo_status &status); - uint32_t get_interrupts_bitmask(VdmaInterruptsDomain interrupts_domain); - void program_single_descriptor(VdmaDescriptor &descriptor, uint16_t page_size, - VdmaInterruptsDomain interrupts_domain); - void reprogram_single_descriptor_interrupts_domain(VdmaDescriptor &descriptor, VdmaInterruptsDomain interrupts_domain); + DescriptorList(uint32_t desc_count, HailoRTDriver &driver, uint16_t desc_page_size, hailo_status &status); + uint32_t get_interrupts_bitmask(InterruptsDomain interrupts_domain); + void reprogram_single_descriptor_interrupts_domain(VdmaDescriptor &descriptor, InterruptsDomain interrupts_domain); static Expected calculate_desc_list_depth(size_t count); // Note: initial_desc_page_size should be the optimal descriptor page size. static Expected> get_desc_buffer_sizes_for_single_transfer_impl( @@ -152,6 +161,8 @@ private: static Expected> get_desc_buffer_sizes_for_multiple_transfers_impl( const HailoRTDriver &driver, uint16_t batch_size, const std::vector &transfer_sizes, uint16_t initial_desc_page_size); + static uint32_t get_descriptors_count_needed(const std::vector &transfer_sizes, + uint16_t desc_page_size); MmapBuffer m_mapped_list; uint32_t m_count; @@ -162,6 +173,7 @@ private: const uint16_t m_desc_page_size; }; +} /* namespace vdma */ } /* namespace hailort */ #endif //_HAILO_VDMA_DESCRIPTOR_LIST_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/memory/dma_mapped_buffer.cpp b/hailort/libhailort/src/vdma/memory/dma_mapped_buffer.cpp new file mode 100644 index 0000000..bd3a270 --- /dev/null +++ b/hailort/libhailort/src/vdma/memory/dma_mapped_buffer.cpp @@ -0,0 +1,96 @@ +/** + * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file vmda_mapped_buffer.cpp + * @brief Vdma mapped buffer implementation + **/ + +#include "hailo/dma_mapped_buffer.hpp" + +#include "vdma/memory/mapped_buffer_impl.hpp" +#include "vdma/vdma_device.hpp" + + +namespace hailort { + +static Expected convert_flags_to_driver_enum(hailo_vdma_buffer_direction_flags_t data_direction) +{ + static const auto BOTH_DIRECTIONS = HAILO_VDMA_BUFFER_DIRECTION_FLAGS_H2D | HAILO_VDMA_BUFFER_DIRECTION_FLAGS_D2H; + if ((data_direction & BOTH_DIRECTIONS) == BOTH_DIRECTIONS) { + return HailoRTDriver::DmaDirection::BOTH; + } + + if ((data_direction & HAILO_VDMA_BUFFER_DIRECTION_FLAGS_H2D) == HAILO_VDMA_BUFFER_DIRECTION_FLAGS_H2D) { + return HailoRTDriver::DmaDirection::H2D; + } + + if ((data_direction & HAILO_VDMA_BUFFER_DIRECTION_FLAGS_D2H) == HAILO_VDMA_BUFFER_DIRECTION_FLAGS_D2H) { + return HailoRTDriver::DmaDirection::D2H; + } + + return make_unexpected(HAILO_INVALID_ARGUMENT); +} + +// TODO: this should maybe be a vdevice (for mapping buffers to multiple devs) +// TODO: a helper function for the cast to VdmaDevice +Expected DmaMappedBuffer::create(size_t size, + hailo_vdma_buffer_direction_flags_t data_direction_flags, Device &device) +{ + static const auto ALLOCATE_BUFFER = nullptr; + return create(ALLOCATE_BUFFER, size, data_direction_flags, device); +} + +Expected DmaMappedBuffer::create_from_user_address(void *user_address, size_t size, + hailo_vdma_buffer_direction_flags_t data_direction_flags, Device &device) +{ + CHECK_ARG_NOT_NULL_AS_EXPECTED(user_address); + return create(user_address, size, data_direction_flags, device); +} + +Expected DmaMappedBuffer::create(void *user_address, size_t size, + hailo_vdma_buffer_direction_flags_t data_direction_flags, Device &device) +{ + const auto device_type = device.get_type(); + CHECK_AS_EXPECTED(((Device::Type::INTEGRATED == device_type) || (Device::Type::PCIE == device_type)), + HAILO_INVALID_ARGUMENT, "Invalid device type (expected integrated/pcie, received {})", device_type); + VdmaDevice *vdma_device = reinterpret_cast(&device); + + auto data_direction = convert_flags_to_driver_enum(data_direction_flags); + CHECK_EXPECTED(data_direction, "Invalid direction flags received {}", data_direction_flags); + + auto pimpl_exp = Impl::create(vdma_device->get_driver(), data_direction.release(), size, user_address); + CHECK_EXPECTED(pimpl_exp); + + auto pimpl = make_unique_nothrow(pimpl_exp.release()); + CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); + + return DmaMappedBuffer(std::move(pimpl)); +} + +DmaMappedBuffer::DmaMappedBuffer(std::unique_ptr pimpl) : + pimpl(std::move(pimpl)) +{} + +// Note: These can't be defined in the header due to the use of pimpl (it'll cause a compilation error) +DmaMappedBuffer::DmaMappedBuffer(DmaMappedBuffer &&other) noexcept = default; +DmaMappedBuffer::~DmaMappedBuffer() = default; + +void *DmaMappedBuffer::user_address() +{ + return pimpl->user_address(); +} + +size_t DmaMappedBuffer::size() const +{ + return pimpl->size(); +} + +hailo_status DmaMappedBuffer::synchronize() +{ + static constexpr auto BUFFER_START = 0; + return pimpl->synchronize(BUFFER_START, size()); +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/memory/mapped_buffer_factory.cpp b/hailort/libhailort/src/vdma/memory/mapped_buffer_factory.cpp new file mode 100644 index 0000000..095243e --- /dev/null +++ b/hailort/libhailort/src/vdma/memory/mapped_buffer_factory.cpp @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file mapped_buffer_factory.cpp + * @brief Static utility class for creating DmaMappedBuffers internally in hailort + **/ + +#include "vdma/memory/mapped_buffer_factory.hpp" +#include "vdma/memory/mapped_buffer_impl.hpp" + +namespace hailort +{ +namespace vdma +{ + +Expected MappedBufferFactory::create_mapped_buffer(size_t size, + HailoRTDriver::DmaDirection data_direction, HailoRTDriver &driver) +{ + auto pimpl_exp = DmaMappedBuffer::Impl::create(driver, data_direction, size); + CHECK_EXPECTED(pimpl_exp); + + auto pimpl = make_unique_nothrow(pimpl_exp.release()); + CHECK_NOT_NULL_AS_EXPECTED(pimpl, HAILO_OUT_OF_HOST_MEMORY); + return DmaMappedBuffer(std::move(pimpl)); +} + +} /* namespace vdma */ +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/memory/mapped_buffer_factory.hpp b/hailort/libhailort/src/vdma/memory/mapped_buffer_factory.hpp new file mode 100644 index 0000000..8cad51f --- /dev/null +++ b/hailort/libhailort/src/vdma/memory/mapped_buffer_factory.hpp @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file mapped_buffer_factory.hpp + * @brief Static utility class for creating DmaMappedBuffers internally in hailort + **/ + +#ifndef _HAILO_MAPPED_BUFFER_FACTORY_HPP_ +#define _HAILO_MAPPED_BUFFER_FACTORY_HPP_ + +#include "hailo/hailort.h" +#include "hailo/dma_mapped_buffer.hpp" +#include "os/hailort_driver.hpp" + +namespace hailort +{ +namespace vdma +{ + +class MappedBufferFactory +{ +public: + MappedBufferFactory() = delete; + static Expected create_mapped_buffer(size_t size, + HailoRTDriver::DmaDirection data_direction, HailoRTDriver &driver); +}; + +} /* namespace vdma */ +} /* namespace hailort */ + +#endif /* _HAILO_MAPPED_BUFFER_FACTORY_HPP_ */ diff --git a/hailort/libhailort/src/vdma/memory/mapped_buffer_impl.cpp b/hailort/libhailort/src/vdma/memory/mapped_buffer_impl.cpp new file mode 100644 index 0000000..2d7193f --- /dev/null +++ b/hailort/libhailort/src/vdma/memory/mapped_buffer_impl.cpp @@ -0,0 +1,279 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file mapped_buffer_impl.cpp + * @brief Dma mapped buffer pimpl class implementation + **/ +#include "mapped_buffer_impl.hpp" + +namespace hailort { + +#if defined(__linux__) || defined(_MSC_VER) + +Expected DmaMappedBuffer::Impl::create(HailoRTDriver &driver, + HailoRTDriver::DmaDirection data_direction, size_t size, void *user_address) +{ + if (nullptr != user_address) { + // User allocated buffer - create an empty MmapBuffer (it doesn't hold the buffer) + auto status = HAILO_UNINITIALIZED; + auto result = DmaMappedBuffer::Impl(HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE, size, + data_direction, user_address, MmapBuffer(), driver, status); + CHECK_SUCCESS_AS_EXPECTED(status); + + return result; + } else if (driver.allocate_driver_buffer()) { + // Allocate buffer via driver + auto driver_buffer_handle = driver.vdma_low_memory_buffer_alloc(size); + CHECK_EXPECTED(driver_buffer_handle); + + uintptr_t driver_buff_handle = driver_buffer_handle.release(); + + auto mapped_buffer = MmapBuffer::create_file_map(size, driver.fd(), driver_buff_handle); + CHECK_EXPECTED(mapped_buffer); + + auto status = HAILO_UNINITIALIZED; + auto result = DmaMappedBuffer::Impl(driver_buff_handle, size, data_direction, mapped_buffer.release(), + driver, status); + CHECK_SUCCESS_AS_EXPECTED(status); + + return result; + } else { + // Standard userspace allocation + auto mapped_buffer = MmapBuffer::create_shared_memory(size); + CHECK_EXPECTED(mapped_buffer); + + auto status = HAILO_UNINITIALIZED; + auto result = DmaMappedBuffer::Impl(HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE, size, + data_direction, mapped_buffer.release(), driver, status); + CHECK_SUCCESS_AS_EXPECTED(status); + + return result; + } +} + +DmaMappedBuffer::Impl::Impl(vdma_mapped_buffer_driver_identifier driver_allocated_buffer_id, + size_t size, HailoRTDriver::DmaDirection data_direction, void *user_address, + MmapBuffer &&mapped_buffer, HailoRTDriver &driver, hailo_status &status) : + m_driver(driver), + m_driver_allocated_buffer_id(driver_allocated_buffer_id), + m_mapping_handle(HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE), + m_mapped_buffer(std::move(mapped_buffer)), + m_size(size), + m_data_direction(data_direction), + m_user_address(user_address) +{ + if (m_mapped_buffer.is_mapped() && (m_user_address != m_mapped_buffer.address())) { + status = HAILO_INVALID_ARGUMENT; + return; + } + + auto expected_handle = driver.vdma_buffer_map(m_user_address, m_size, m_data_direction, + m_driver_allocated_buffer_id); + if (!expected_handle) { + status = expected_handle.status(); + return; + } + + m_mapping_handle = expected_handle.release(); + status = HAILO_SUCCESS; +} + +DmaMappedBuffer::Impl::Impl(vdma_mapped_buffer_driver_identifier driver_allocated_buffer_id, + size_t size, HailoRTDriver::DmaDirection data_direction, + MmapBuffer &&mapped_buffer, HailoRTDriver &driver, hailo_status &status) : + Impl(driver_allocated_buffer_id, size, data_direction, mapped_buffer.address(), std::move(mapped_buffer), driver, status) +{} + +DmaMappedBuffer::Impl::Impl(Impl &&other) noexcept : + m_driver(other.m_driver), + m_driver_allocated_buffer_id(std::exchange(other.m_driver_allocated_buffer_id, HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE)), + m_mapping_handle(std::exchange(other.m_mapping_handle, HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE)), + m_mapped_buffer(std::move(other.m_mapped_buffer)), + m_size(std::move(other.m_size)), + m_data_direction(std::move(other.m_data_direction)), + m_user_address(std::move(other.m_user_address)) +{} + +DmaMappedBuffer::Impl::~Impl() +{ + if (HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE != m_mapping_handle) { + m_driver.vdma_buffer_unmap(m_mapping_handle); + m_mapping_handle = HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE; + } + + if (HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE != m_driver_allocated_buffer_id) { + m_driver.vdma_low_memory_buffer_free(m_driver_allocated_buffer_id); + m_driver_allocated_buffer_id = HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE; + } +} + +void* DmaMappedBuffer::Impl::user_address() +{ + return m_user_address; +} + +size_t DmaMappedBuffer::Impl::size() const +{ + return m_size; +} + +HailoRTDriver::VdmaBufferHandle DmaMappedBuffer::Impl::handle() +{ + return m_mapping_handle; +} + +hailo_status DmaMappedBuffer::Impl::synchronize(size_t offset, size_t count) +{ + CHECK(offset + count <= size(), HAILO_INVALID_ARGUMENT, + "Synchronizing {} bytes starting at offset {} will overflow (buffer size {})", + offset, count, size()); + return m_driver.vdma_buffer_sync(m_mapping_handle, m_data_direction, offset, count); +} + +#elif defined(__QNX__) + +#include + +const int DmaMappedBuffer::Impl::INVALID_FD = -1; +const shm_handle_t DmaMappedBuffer::Impl::INVALID_HANDLE = (shm_handle_t)-1; +const char* DmaMappedBuffer::Impl::VDMA_BUFFER_TYPE_MEMORY_NAME = "/memory/below4G/ram/below1G"; + +Expected DmaMappedBuffer::Impl::create(HailoRTDriver &driver, + HailoRTDriver::DmaDirection data_direction, size_t size, void *user_address) +{ + // TODO: HRT-9508 + CHECK_AS_EXPECTED(user_address == nullptr, HAILO_NOT_IMPLEMENTED, "User allocated buffers not supported on qnx"); + + // Destructor of type_mem_fd will close fd + FileDescriptor type_mem_fd(posix_typed_mem_open(VDMA_BUFFER_TYPE_MEMORY_NAME, O_RDWR, POSIX_TYPED_MEM_ALLOCATE)); + if (INVALID_FD == type_mem_fd) { + LOGGER__ERROR("Error getting fd from typed memory of type {}, errno {}\n", VDMA_BUFFER_TYPE_MEMORY_NAME, + errno); + return make_unexpected(HAILO_INTERNAL_FAILURE); + } + + vdma_mapped_buffer_driver_identifier driver_buff_handle; + driver_buff_handle.shm_fd = shm_open(SHM_ANON, O_RDWR | O_CREAT, 0777); + CHECK_AS_EXPECTED(INVALID_FD != driver_buff_handle.shm_fd, HAILO_INTERNAL_FAILURE, + "Error creating shm object, errno is: {}", errno); + + // backs the shared memory object with physical memory + int err = shm_ctl(driver_buff_handle.shm_fd, SHMCTL_ANON | SHMCTL_TYMEM, (uint64_t)type_mem_fd, + size); + if (-1 == err) { + LOGGER__ERROR("Error backing shm object in physical memory, errno is: {}", errno); + close(driver_buff_handle.shm_fd); + return make_unexpected(HAILO_INTERNAL_FAILURE); + } + + // Create shared memory handle to send to driver + err = shm_create_handle(driver_buff_handle.shm_fd, driver.resource_manager_pid(), O_RDWR, + &driver_buff_handle.shm_handle, 0); + if (0 != err) { + LOGGER__ERROR("Error creating shm object handle, errno is: {}", errno); + close(driver_buff_handle.shm_fd); + return make_unexpected(HAILO_INTERNAL_FAILURE); + } + + void *address = mmap(0, size, PROT_WRITE | PROT_READ | PROT_NOCACHE, MAP_SHARED, driver_buff_handle.shm_fd, 0); + if (MAP_FAILED == address) { + LOGGER__ERROR("Failed to mmap buffer with errno:{}", errno); + shm_delete_handle(driver_buff_handle.shm_handle); + close(driver_buff_handle.shm_fd); + return make_unexpected(HAILO_OUT_OF_HOST_MEMORY); + } + + hailo_status status = HAILO_UNINITIALIZED; + auto result = DmaMappedBuffer::Impl(address, size, data_direction, driver_buff_handle.shm_handle, + driver_buff_handle.shm_fd, driver, status); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to map buffer to vdma"); + munmap(address, size); + shm_delete_handle(driver_buff_handle.shm_handle); + close(driver_buff_handle.shm_fd); + return make_unexpected(status); + } + + return result; +} + +DmaMappedBuffer::Impl::Impl(void *addr, size_t size, HailoRTDriver::DmaDirection data_direction, + shm_handle_t shm_handle, int shm_fd, HailoRTDriver &driver, hailo_status &status) : + m_driver(driver), + m_address(addr), + m_size(size), + m_data_direction(data_direction) +{ + m_driver_allocated_buffer_id.shm_handle = shm_handle; + m_driver_allocated_buffer_id.shm_fd = shm_fd; + + auto expected_handle = driver.vdma_buffer_map(addr, size, data_direction, m_driver_allocated_buffer_id); + if (!expected_handle) { + status = expected_handle.status(); + return; + } + + m_mapping_handle = expected_handle.release(); + status = HAILO_SUCCESS; +} + +DmaMappedBuffer::Impl::Impl(Impl &&other) noexcept : + m_driver(other.m_driver), + m_address(std::exchange(other.m_address, nullptr)), + m_size(std::move(other.m_size)), + m_data_direction(std::move(other.m_data_direction)), + m_mapping_handle(std::exchange(other.m_mapping_handle, HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE)) +{ + m_driver_allocated_buffer_id.shm_handle = std::exchange(other.m_driver_allocated_buffer_id.shm_handle, INVALID_HANDLE); + m_driver_allocated_buffer_id.shm_fd = std::exchange(other.m_driver_allocated_buffer_id.shm_fd, INVALID_FD); +} + +DmaMappedBuffer::Impl::~Impl() +{ + if (HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE != m_mapping_handle) { + m_driver.vdma_buffer_unmap(m_mapping_handle); + m_mapping_handle = HailoRTDriver::INVALID_DRIVER_VDMA_MAPPING_HANDLE_VALUE; + } + + if (nullptr != m_address) { + if (0 != munmap(m_address, m_size)) { + LOGGER__ERROR("Error unmapping memory at address {}, Errno: {}", m_address, errno); + } + } + + if (INVALID_FD != m_driver_allocated_buffer_id.shm_fd) { + if (0 != close(m_driver_allocated_buffer_id.shm_fd)) { + LOGGER__ERROR("Error closing shared memory fd, Errno: {}", errno); + } + } +} + +void* DmaMappedBuffer::Impl::user_address() +{ + return m_address; +} +size_t DmaMappedBuffer::Impl::size() const +{ + return m_size; +} + +HailoRTDriver::VdmaBufferHandle DmaMappedBuffer::Impl::handle() +{ + return m_mapping_handle; +} + +hailo_status DmaMappedBuffer::Impl::synchronize(size_t offset, size_t count) +{ + CHECK(offset + count <= size(), HAILO_INVALID_ARGUMENT, + "Synchronizing {} bytes starting at offset {} will overflow (buffer size {})", + offset, count, size()); + return m_driver.vdma_buffer_sync(m_mapping_handle, m_data_direction, offset, count); +} + +#else +#error "unsupported platform!" +#endif // defined(__linux__) || defined(_MSC_VER) + +} /* namespace hailort */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/memory/mapped_buffer_impl.hpp b/hailort/libhailort/src/vdma/memory/mapped_buffer_impl.hpp new file mode 100644 index 0000000..7643db8 --- /dev/null +++ b/hailort/libhailort/src/vdma/memory/mapped_buffer_impl.hpp @@ -0,0 +1,98 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file mapped_buffer_impl.hpp + * @brief Vdma mapped buffer pimpl class defintion + **/ +#ifndef _HAILO_VDMA_MAPPED_BUFFER_IMPL_HPP_ +#define _HAILO_VDMA_MAPPED_BUFFER_IMPL_HPP_ + +#include "hailo/dma_mapped_buffer.hpp" +#include "os/mmap_buffer.hpp" +#include "os/hailort_driver.hpp" +#include "hailo/expected.hpp" + +namespace hailort { + +#if defined(__linux__) || defined(_MSC_VER) + +class DmaMappedBuffer::Impl final { +public: + // If user_address is nullptr, a buffer of size 'size' will be allocated and mapped to dma in 'data_direction' + // Otherwise, the buffer pointed to by user_address will be mapped to dma in 'data_direction' + static Expected create(HailoRTDriver &driver, HailoRTDriver::DmaDirection data_direction, + size_t size, void *user_address = nullptr); + + Impl(Impl &&other) noexcept; + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl &operator=(Impl &&other) = delete; + ~Impl(); + + void* user_address(); + size_t size() const; + HailoRTDriver::VdmaBufferHandle handle(); + // TODO: validate that offset is cache aligned (HRT-9811) + hailo_status synchronize(size_t offset, size_t count); + +private: + Impl(vdma_mapped_buffer_driver_identifier driver_allocated_buffer_id, size_t size, + HailoRTDriver::DmaDirection data_direction, void *user_address, MmapBuffer &&mapped_buffer, + HailoRTDriver &driver, hailo_status &status); + Impl(vdma_mapped_buffer_driver_identifier driver_allocated_buffer_id, size_t size, + HailoRTDriver::DmaDirection data_direction, MmapBuffer &&mapped_buffer, HailoRTDriver &driver, + hailo_status &status); + + HailoRTDriver &m_driver; + vdma_mapped_buffer_driver_identifier m_driver_allocated_buffer_id; + HailoRTDriver::VdmaBufferHandle m_mapping_handle; + MmapBuffer m_mapped_buffer; + const size_t m_size; + const HailoRTDriver::DmaDirection m_data_direction; + void *const m_user_address; +}; + +#elif defined(__QNX__) + +// TODO: merge qnx and non-qnx impls (HRT-9508) +class DmaMappedBuffer::Impl final { +public: + static Expected create(HailoRTDriver &driver, HailoRTDriver::DmaDirection data_direction, + size_t size, void *user_address = nullptr); + + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl &operator=(Impl &&other) = delete; + Impl(Impl &&other) noexcept; + ~Impl(); + + void* user_address(); + size_t size() const; + HailoRTDriver::VdmaBufferHandle handle(); + hailo_status synchronize(size_t offset, size_t count); + +private: + Impl(void *addr, size_t size, HailoRTDriver::DmaDirection data_direction, + shm_handle_t shm_handle, int shm_fd, HailoRTDriver &driver, hailo_status &status); + + static const int INVALID_FD; + static const shm_handle_t INVALID_HANDLE; + static const char* VDMA_BUFFER_TYPE_MEMORY_NAME; + + HailoRTDriver &m_driver; + void *m_address; + const size_t m_size; + const HailoRTDriver::DmaDirection m_data_direction; + vdma_mapped_buffer_driver_identifier m_driver_allocated_buffer_id; + HailoRTDriver::VdmaBufferHandle m_mapping_handle; +}; + +#else +#error "unsupported platform!" +#endif // defined(__linux__) || defined(_MSC_VER) + +} /* namespace hailort */ + +#endif /* _HAILO_VDMA_MAPPED_BUFFER_IMPL_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/memory/sg_buffer.cpp b/hailort/libhailort/src/vdma/memory/sg_buffer.cpp new file mode 100644 index 0000000..9d6b97b --- /dev/null +++ b/hailort/libhailort/src/vdma/memory/sg_buffer.cpp @@ -0,0 +1,137 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_sg_buffer.cpp + * @brief Scatter-gather vdma buffer. + **/ + +#include "vdma/memory/sg_buffer.hpp" +#include "vdma/channel/channel_id.hpp" +#include "vdma/memory/mapped_buffer_factory.hpp" + + +namespace hailort { +namespace vdma { + +Expected SgBuffer::create(HailoRTDriver &driver, size_t size, uint32_t desc_count, uint16_t desc_page_size, + HailoRTDriver::DmaDirection data_direction, ChannelId channel_id) +{ + CHECK_AS_EXPECTED(size <= (desc_count * desc_page_size), HAILO_INTERNAL_FAILURE, + "Requested buffer size {} must be smaller than {}", size, (desc_count * desc_page_size)); + CHECK_AS_EXPECTED((size % desc_page_size) == 0, HAILO_INTERNAL_FAILURE, + "SgBuffer size must be a multiple of descriptors page size (size {})", size); + + auto mapped_buffer_exp = MappedBufferFactory::create_mapped_buffer(size, + data_direction, driver); + CHECK_EXPECTED(mapped_buffer_exp); + + auto mapped_buffer = make_shared_nothrow(mapped_buffer_exp.release()); + CHECK_NOT_NULL_AS_EXPECTED(mapped_buffer, HAILO_OUT_OF_HOST_MEMORY); + + auto desc_list_exp = DescriptorList::create(desc_count, desc_page_size, driver); + CHECK_EXPECTED(desc_list_exp); + + auto desc_list = make_shared_nothrow(desc_list_exp.release()); + CHECK_NOT_NULL_AS_EXPECTED(desc_list, HAILO_OUT_OF_HOST_MEMORY); + + assert((desc_count * desc_page_size) <= std::numeric_limits::max()); + + auto status = desc_list->configure_to_use_buffer(*mapped_buffer, channel_id); + CHECK_SUCCESS_AS_EXPECTED(status); + + return SgBuffer(mapped_buffer, desc_list); +} + +SgBuffer::SgBuffer(std::shared_ptr mapped_buffer, std::shared_ptr desc_list) : + m_mapped_buffer(mapped_buffer), + m_desc_list(desc_list) +{} + +size_t SgBuffer::size() const +{ + return m_mapped_buffer->size(); +} + +uint64_t SgBuffer::dma_address() const +{ + return m_desc_list->dma_address(); +} + +uint16_t SgBuffer::desc_page_size() const +{ + return m_desc_list->desc_page_size(); +} + +uint32_t SgBuffer::descs_count() const +{ + return static_cast(m_desc_list->count()); +} + +uint8_t SgBuffer::depth() const +{ + return m_desc_list->depth(); +} + +std::shared_ptr SgBuffer::get_desc_list() +{ + return m_desc_list; +} + +// TODO: Remove after HRT-7838 +void* SgBuffer::get_user_address() +{ + return m_mapped_buffer->user_address(); +} + +hailo_status SgBuffer::read(void *buf_dst, size_t count, size_t offset, bool should_sync) +{ + CHECK(count + offset <= m_mapped_buffer->size(), HAILO_INSUFFICIENT_BUFFER); + if (count == 0) { + return HAILO_SUCCESS; + } + + if (should_sync) { + const auto status = m_mapped_buffer->synchronize(); + CHECK_SUCCESS(status, "Failed synching SgBuffer buffer on read"); + } + + const auto src_addr = static_cast(m_mapped_buffer->user_address()) + offset; + memcpy(buf_dst, src_addr, count); + + return HAILO_SUCCESS; +} +hailo_status SgBuffer::write(const void *buf_src, size_t count, size_t offset) +{ + CHECK(count + offset <= m_mapped_buffer->size(), HAILO_INSUFFICIENT_BUFFER); + if (count == 0) { + return HAILO_SUCCESS; + } + + const auto dst_addr = static_cast(m_mapped_buffer->user_address()) + offset; + std::memcpy(dst_addr, buf_src, count); + + const auto status = m_mapped_buffer->synchronize(); + CHECK_SUCCESS(status, "Failed synching SgBuffer buffer on write"); + + return HAILO_SUCCESS; +} + +Expected SgBuffer::program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, + size_t desc_offset, bool is_circular) +{ + return m_desc_list->program_last_descriptor(transfer_size, last_desc_interrupts_domain, desc_offset, is_circular); +} + +hailo_status SgBuffer::reprogram_device_interrupts_for_end_of_batch(size_t transfer_size, uint16_t batch_size, + InterruptsDomain new_interrupts_domain) +{ + const auto desc_per_transfer = m_desc_list->descriptors_in_buffer(transfer_size); + const auto num_desc_in_batch = desc_per_transfer * batch_size; + const auto last_desc_index_in_batch = num_desc_in_batch - 1; + return m_desc_list->reprogram_descriptor_interrupts_domain(last_desc_index_in_batch, new_interrupts_domain); +} + +} +} \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/sg_buffer.hpp b/hailort/libhailort/src/vdma/memory/sg_buffer.hpp similarity index 53% rename from hailort/libhailort/src/vdma/sg_buffer.hpp rename to hailort/libhailort/src/vdma/memory/sg_buffer.hpp index 6f097b6..36e2041 100644 --- a/hailort/libhailort/src/vdma/sg_buffer.hpp +++ b/hailort/libhailort/src/vdma/memory/sg_buffer.hpp @@ -7,26 +7,28 @@ * @brief Scatter-gather vdma buffer, from the user-mode point of view the buffer is continuous, * but not from the physical-memory point of view. * The sg buffer contains 2 parts: - * - MappedBuffer - the actual buffer stores the data. - * - Descriptors list - each descritpor points to a single "dma page" in the MappedBuffer. + * - DmaMappedBuffer - the actual buffer stores the data. + * - Descriptors list - each descritpor points to a single "dma page" in the DmaMappedBuffer. * The hw accept the descriptors list address and parses it to get the actual data. **/ #ifndef _HAILO_VDMA_SG_BUFFER_HPP_ #define _HAILO_VDMA_SG_BUFFER_HPP_ +#include "hailo/dma_mapped_buffer.hpp" + #include "os/hailort_driver.hpp" -#include "vdma/vdma_buffer.hpp" -#include "vdma_descriptor_list.hpp" -#include "vdma/mapped_buffer.hpp" +#include "vdma/memory/vdma_buffer.hpp" +#include "vdma/memory/descriptor_list.hpp" + namespace hailort { namespace vdma { class SgBuffer final : public VdmaBuffer { public: - static Expected create(HailoRTDriver &driver, uint32_t desc_count, uint16_t desc_page_size, - HailoRTDriver::DmaDirection data_direction, uint8_t channel_index = HailoRTDriver::INVALID_VDMA_CHANNEL_INDEX); + static Expected create(HailoRTDriver &driver, size_t size, uint32_t desc_count, uint16_t desc_page_size, + HailoRTDriver::DmaDirection data_direction, vdma::ChannelId channel_id); virtual ~SgBuffer() = default; @@ -46,33 +48,24 @@ public: virtual uint32_t descs_count() const override; uint8_t depth() const; - ExpectedRef get_desc_list(); - // TODO: Remove after HRT-7838 + std::shared_ptr get_desc_list(); + // TODO: Remove after HRT-7838 void *get_user_address(); virtual hailo_status read(void *buf_dst, size_t count, size_t offset, bool should_sync) override; virtual hailo_status write(const void *buf_src, size_t count, size_t offset) override; - hailo_status read_cyclic(void *buf_dst, size_t count, size_t offset, bool should_sync = true); - hailo_status write_cyclic(const void *buf_src, size_t count, size_t offset); - - virtual Expected program_descriptors(size_t transfer_size, VdmaInterruptsDomain first_desc_interrupts_domain, - VdmaInterruptsDomain last_desc_interrupts_domain, size_t desc_offset, bool is_circular) override; + virtual Expected program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, + size_t desc_offset, bool is_circular) override; virtual hailo_status reprogram_device_interrupts_for_end_of_batch(size_t transfer_size, uint16_t batch_size, - VdmaInterruptsDomain new_interrupts_domain) override; - - // TODO: after HRT-8519 the VdmaDescriptorList will be owned by the vdma channel and this function can be removed - // (VdmaChannel::reprogram_buffer_offset will call VdmaDescriptorList::configure_to_use_buffer directly) - hailo_status reprogram_buffer_offset(size_t new_start_offset, uint8_t channel_index); + InterruptsDomain new_interrupts_domain) override; private: - SgBuffer(VdmaDescriptorList &&desc_list, MappedBuffer &&mapped_buffer) : - m_desc_list(std::move(desc_list)), - m_mapped_buffer(std::move(mapped_buffer)) - {} + SgBuffer(std::shared_ptr mapped_buffer, std::shared_ptr desc_list); - VdmaDescriptorList m_desc_list; - MappedBuffer m_mapped_buffer; + // Initialization Dependency: The descriptor list points into the mapped buffer so it must be freed before it + std::shared_ptr m_mapped_buffer; + std::shared_ptr m_desc_list; }; } /* vdma */ diff --git a/hailort/libhailort/src/vdma/vdma_buffer.cpp b/hailort/libhailort/src/vdma/memory/vdma_buffer.cpp similarity index 50% rename from hailort/libhailort/src/vdma/vdma_buffer.cpp rename to hailort/libhailort/src/vdma/memory/vdma_buffer.cpp index 500e80d..97a00d9 100644 --- a/hailort/libhailort/src/vdma/vdma_buffer.cpp +++ b/hailort/libhailort/src/vdma/memory/vdma_buffer.cpp @@ -15,14 +15,19 @@ namespace vdma { CONTROL_PROTOCOL__host_buffer_info_t VdmaBuffer::get_host_buffer_info(uint32_t transfer_size) { - CONTROL_PROTOCOL__host_buffer_info_t buffer_info = {}; + return get_host_buffer_info(type(), dma_address(), desc_page_size(), descs_count(), transfer_size); +} - buffer_info.buffer_type = static_cast((type() == vdma::VdmaBuffer::Type::SCATTER_GATHER) ? +CONTROL_PROTOCOL__host_buffer_info_t VdmaBuffer::get_host_buffer_info(Type type, uint64_t dma_address, + uint16_t desc_page_size, uint32_t desc_count, uint32_t transfer_size) +{ + CONTROL_PROTOCOL__host_buffer_info_t buffer_info{}; + buffer_info.buffer_type = static_cast((type == vdma::VdmaBuffer::Type::SCATTER_GATHER) ? CONTROL_PROTOCOL__HOST_BUFFER_TYPE_EXTERNAL_DESC : CONTROL_PROTOCOL__HOST_BUFFER_TYPE_CCB); - buffer_info.dma_address = dma_address(); - buffer_info.desc_page_size = desc_page_size(); - buffer_info.total_desc_count = descs_count(); + buffer_info.dma_address = dma_address; + buffer_info.desc_page_size = desc_page_size; + buffer_info.total_desc_count = desc_count; buffer_info.bytes_in_pattern = transfer_size; return buffer_info; diff --git a/hailort/libhailort/src/vdma/vdma_buffer.hpp b/hailort/libhailort/src/vdma/memory/vdma_buffer.hpp similarity index 78% rename from hailort/libhailort/src/vdma/vdma_buffer.hpp rename to hailort/libhailort/src/vdma/memory/vdma_buffer.hpp index fabb4cb..78171ab 100644 --- a/hailort/libhailort/src/vdma/vdma_buffer.hpp +++ b/hailort/libhailort/src/vdma/memory/vdma_buffer.hpp @@ -12,9 +12,10 @@ #define _HAILO_VDMA_VDMA_BUFFER_HPP_ #include "os/hailort_driver.hpp" -#include "vdma_descriptor_list.hpp" +#include "vdma/memory/descriptor_list.hpp" #include "control_protocol.h" + namespace hailort { namespace vdma { @@ -44,18 +45,20 @@ public: { assert(buffer_size < std::numeric_limits::max()); const auto page_size = desc_page_size(); - return static_cast(DESCRIPTORS_IN_BUFFER(buffer_size, page_size)); + return static_cast(DIV_ROUND_UP(buffer_size, page_size)); } virtual hailo_status read(void *buf_dst, size_t count, size_t offset, bool should_sync = true) = 0; virtual hailo_status write(const void *buf_src, size_t count, size_t offset) = 0; - virtual Expected program_descriptors(size_t transfer_size, VdmaInterruptsDomain first_desc_interrupts_domain, - VdmaInterruptsDomain last_desc_interrupts_domain, size_t desc_offset, bool is_circular) = 0; + virtual Expected program_descriptors(size_t transfer_size, InterruptsDomain last_desc_interrupts_domain, + size_t desc_offset, bool is_circular) = 0; virtual hailo_status reprogram_device_interrupts_for_end_of_batch(size_t transfer_size, uint16_t batch_size, - VdmaInterruptsDomain new_interrupts_domain) = 0; + InterruptsDomain new_interrupts_domain) = 0; CONTROL_PROTOCOL__host_buffer_info_t get_host_buffer_info(uint32_t transfer_size); + static CONTROL_PROTOCOL__host_buffer_info_t get_host_buffer_info(Type type, uint64_t dma_address, + uint16_t desc_page_size, uint32_t total_desc_count, uint32_t transfer_size); }; } /* vdma */ diff --git a/hailort/libhailort/src/pcie_device.cpp b/hailort/libhailort/src/vdma/pcie/pcie_device.cpp similarity index 85% rename from hailort/libhailort/src/pcie_device.cpp rename to hailort/libhailort/src/vdma/pcie/pcie_device.cpp index 44ca5b2..2fe9e93 100644 --- a/hailort/libhailort/src/pcie_device.cpp +++ b/hailort/libhailort/src/vdma/pcie/pcie_device.cpp @@ -9,20 +9,23 @@ * TODO: doc **/ -#include "pcie_device.hpp" #include "hailo/hailort.h" -#include "common/utils.hpp" #include "hailo/device.hpp" #include "hailo/hef.hpp" -#include "control.hpp" + +#include "common/utils.hpp" #include "common/compiler_extensions_compat.hpp" + +#include "vdma/pcie/pcie_device.hpp" +#include "device_common/control.hpp" #include "os/hailort_driver.hpp" -#include "context_switch/multi_context/resource_manager.hpp" -#include "context_switch/multi_context/vdma_config_manager.hpp" +#include "core_op/resource_manager/resource_manager.hpp" +#include "vdma/vdma_config_manager.hpp" #include #include + namespace hailort { @@ -178,37 +181,6 @@ const char *PcieDevice::get_dev_id() const return m_device_id.c_str(); } -hailo_status PcieDevice::close_all_vdma_channels() -{ - auto status = HAILO_UNINITIALIZED; - - // TODO: Add one icotl to stop all channels at once (HRT-6097) - constexpr uint8_t PCIE_DEFAULT_ENGINE_INDEX = 0; - for (uint8_t channel_index = 0; channel_index <= MAX_H2D_CHANNEL_INDEX; channel_index++) { - const vdma::ChannelId channel_id = { PCIE_DEFAULT_ENGINE_INDEX, channel_index }; - auto host_registers = VdmaChannelRegs(m_driver, channel_id, HailoRTDriver::DmaDirection::H2D); - status = host_registers.stop_channel(); - CHECK_SUCCESS(status); - - auto device_registers = VdmaChannelRegs(m_driver, channel_id, HailoRTDriver::DmaDirection::D2H); - status = device_registers.stop_channel(); - CHECK_SUCCESS(status); - } - - for (uint8_t channel_index = MIN_D2H_CHANNEL_INDEX; channel_index <= MAX_D2H_CHANNEL_INDEX; channel_index++) { - const vdma::ChannelId channel_id = { PCIE_DEFAULT_ENGINE_INDEX, channel_index }; - auto host_registers = VdmaChannelRegs(m_driver, channel_id, HailoRTDriver::DmaDirection::D2H); - status = host_registers.stop_channel(); - CHECK_SUCCESS(status); - - auto device_registers = VdmaChannelRegs(m_driver, channel_id, HailoRTDriver::DmaDirection::H2D); - status = device_registers.stop_channel(); - CHECK_SUCCESS(status); - } - - return HAILO_SUCCESS; -} - hailo_status PcieDevice::reset_impl(CONTROL_PROTOCOL__reset_type_t reset_type) { hailo_status status = HAILO_UNINITIALIZED; @@ -226,8 +198,6 @@ hailo_status PcieDevice::reset_impl(CONTROL_PROTOCOL__reset_type_t reset_type) if ((CONTROL_PROTOCOL__RESET_TYPE__FORCED_SOFT == reset_type) || (CONTROL_PROTOCOL__RESET_TYPE__SOFT == reset_type)) { is_expecting_response = false; // TODO: Check boot source, set is_expecting_response = (boot_source != pcie) - status = close_all_vdma_channels(); - CHECK_SUCCESS(status); } common_status = CONTROL_PROTOCOL__pack_reset_request(&request, &request_size, m_control_sequence, reset_type); diff --git a/hailort/libhailort/src/pcie_device.hpp b/hailort/libhailort/src/vdma/pcie/pcie_device.hpp similarity index 94% rename from hailort/libhailort/src/pcie_device.hpp rename to hailort/libhailort/src/vdma/pcie/pcie_device.hpp index 1755563..ed909e0 100644 --- a/hailort/libhailort/src/pcie_device.hpp +++ b/hailort/libhailort/src/vdma/pcie/pcie_device.hpp @@ -14,8 +14,10 @@ #include "hailo/hailort.h" #include "hailo/expected.hpp" -#include "vdma_channel.hpp" -#include "vdma_device.hpp" + +#include "vdma/channel/boundary_channel.hpp" +#include "vdma/vdma_device.hpp" + namespace hailort { @@ -38,7 +40,7 @@ public: { switch (stream_interface) { case HAILO_STREAM_INTERFACE_ETH: - case HAILO_STREAM_INTERFACE_CORE: + case HAILO_STREAM_INTERFACE_INTEGRATED: return false; case HAILO_STREAM_INTERFACE_PCIE: case HAILO_STREAM_INTERFACE_MIPI: @@ -63,8 +65,6 @@ private: PcieDevice(HailoRTDriver &&driver, const hailo_pcie_device_info_t &device_info, hailo_status &status, const std::string &device_id); - hailo_status close_all_vdma_channels(); - static Expected find_device_info(const hailo_pcie_device_info_t &pcie_device_info); const hailo_pcie_device_info_t m_device_info; diff --git a/hailort/libhailort/src/vdma/sg_buffer.cpp b/hailort/libhailort/src/vdma/sg_buffer.cpp deleted file mode 100644 index d2d0f68..0000000 --- a/hailort/libhailort/src/vdma/sg_buffer.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file vdma_sg_buffer.cpp - * @brief Scatter-gather vdma buffer. - **/ - -#include "sg_buffer.hpp" - -namespace hailort { -namespace vdma { - -Expected SgBuffer::create(HailoRTDriver &driver, uint32_t desc_count, uint16_t desc_page_size, - HailoRTDriver::DmaDirection data_direction, uint8_t channel_index) -{ - auto desc_list = VdmaDescriptorList::create(desc_count, desc_page_size, driver); - CHECK_EXPECTED(desc_list); - - assert((desc_count * desc_page_size) <= std::numeric_limits::max()); - auto mapped_buffer = MappedBuffer::create(desc_count * desc_page_size, data_direction, driver); - CHECK_EXPECTED(mapped_buffer); - - auto status = desc_list->configure_to_use_buffer(mapped_buffer.value(), channel_index); - CHECK_SUCCESS_AS_EXPECTED(status); - - return SgBuffer(desc_list.release(), mapped_buffer.release()); -} - -size_t SgBuffer::size() const -{ - return m_mapped_buffer.size(); -} - -uint64_t SgBuffer::dma_address() const -{ - return m_desc_list.dma_address(); -} - -uint16_t SgBuffer::desc_page_size() const -{ - return m_desc_list.desc_page_size(); -} - -uint32_t SgBuffer::descs_count() const -{ - return (uint32_t)m_desc_list.count(); -} - -uint8_t SgBuffer::depth() const -{ - return m_desc_list.depth(); -} - -ExpectedRef SgBuffer::get_desc_list() -{ - return std::ref(m_desc_list); -} - -hailo_status SgBuffer::read(void *buf_dst, size_t count, size_t offset, bool should_sync) -{ - return m_mapped_buffer.read(buf_dst, count, offset, should_sync); -} - -hailo_status SgBuffer::write(const void *buf_src, size_t count, size_t offset) -{ - return m_mapped_buffer.write(buf_src, count, offset); -} - -hailo_status SgBuffer::read_cyclic(void *buf_dst, size_t count, size_t offset, bool should_sync) -{ - return m_mapped_buffer.read_cyclic(buf_dst, count, offset, should_sync); -} - -hailo_status SgBuffer::write_cyclic(const void *buf_src, size_t count, size_t offset) -{ - return m_mapped_buffer.write_cyclic(buf_src, count, offset); -} - -// TODO: Remove after HRT-7838 -void* SgBuffer::get_user_address() -{ - return m_mapped_buffer.user_address(); -} - -Expected SgBuffer::program_descriptors(size_t transfer_size, VdmaInterruptsDomain first_desc_interrupts_domain, - VdmaInterruptsDomain last_desc_interrupts_domain, size_t desc_offset, bool is_circular) -{ - return m_desc_list.program_descriptors(transfer_size, first_desc_interrupts_domain, last_desc_interrupts_domain, - desc_offset, is_circular); -} - -hailo_status SgBuffer::reprogram_device_interrupts_for_end_of_batch(size_t transfer_size, uint16_t batch_size, - VdmaInterruptsDomain new_interrupts_domain) -{ - const auto desc_per_transfer = m_desc_list.descriptors_in_buffer(transfer_size); - const auto num_desc_in_batch = desc_per_transfer * batch_size; - const auto last_desc_index_in_batch = num_desc_in_batch - 1; - return m_desc_list.reprogram_descriptor_interrupts_domain(last_desc_index_in_batch, new_interrupts_domain); -} - -hailo_status SgBuffer::reprogram_buffer_offset(size_t new_start_offset, uint8_t channel_index) -{ - return m_desc_list.configure_to_use_buffer(m_mapped_buffer, channel_index, new_start_offset); -} - -} -} \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/vdma_async_stream.cpp b/hailort/libhailort/src/vdma/vdma_async_stream.cpp new file mode 100644 index 0000000..1b55e27 --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_async_stream.cpp @@ -0,0 +1,90 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_async_stream.cpp + * @brief Async vdma stream implementation + **/ + +#include "hailo/hailort_common.hpp" + +#include "vdma/vdma_async_stream.hpp" + + +namespace hailort +{ + +VdmaAsyncInputStream::VdmaAsyncInputStream(VdmaDevice &device, vdma::BoundaryChannelPtr channel, + const LayerInfo &edge_layer, EventPtr core_op_activated_event, + uint16_t batch_size, std::chrono::milliseconds transfer_timeout, + hailo_stream_interface_t stream_interface, hailo_status &status) : + VdmaInputStreamBase(device, channel, edge_layer, core_op_activated_event, batch_size, + transfer_timeout, stream_interface, status) +{ + // Checking status for base class c'tor + if (HAILO_SUCCESS != status) { + return; + } + + status = HAILO_SUCCESS; +} + +Expected VdmaAsyncInputStream::sync_write_raw_buffer(const MemoryView &) +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +hailo_status VdmaAsyncInputStream::sync_write_all_raw_buffer_no_transform_impl(void *, size_t, size_t) +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status VdmaAsyncInputStream::wait_for_ready(size_t transfer_size, std::chrono::milliseconds timeout) +{ + return m_channel->wait(transfer_size, timeout); +} + +hailo_status VdmaAsyncInputStream::write_async(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque) +{ + return m_channel->transfer(buffer, user_callback, opaque); +} + +/** Output stream **/ + +VdmaAsyncOutputStream::VdmaAsyncOutputStream(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t interface, + hailo_status &status) : + VdmaOutputStreamBase(device, channel, edge_layer, core_op_activated_event, batch_size, + transfer_timeout, interface, status) +{ + // Check status for base class c'tor + if (HAILO_SUCCESS != status) { + return; + } + + status = HAILO_SUCCESS; +} + +Expected VdmaAsyncOutputStream::sync_read_raw_buffer(MemoryView &) +{ + return make_unexpected(HAILO_NOT_IMPLEMENTED); +} + +hailo_status VdmaAsyncOutputStream::read_all(MemoryView &) +{ + return HAILO_NOT_IMPLEMENTED; +} + +hailo_status VdmaAsyncOutputStream::wait_for_ready(size_t transfer_size, std::chrono::milliseconds timeout) +{ + return m_channel->wait(transfer_size, timeout); +} + +hailo_status VdmaAsyncOutputStream::read_async(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque) +{ + return m_channel->transfer(buffer, user_callback, opaque); +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_async_stream.hpp b/hailort/libhailort/src/vdma/vdma_async_stream.hpp new file mode 100644 index 0000000..5086c55 --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_async_stream.hpp @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_stream.hpp + * @brief Async stream object over vDMA channel + **/ + +#ifndef _HAILO_VDMA_ASYNC_STREAM_HPP_ +#define _HAILO_VDMA_ASYNC_STREAM_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" +#include "hailo/stream.hpp" + +#include "vdma/vdma_stream_base.hpp" +#include "vdma/vdma_device.hpp" +#include "vdma/channel/async_channel.hpp" + + +namespace hailort +{ + +class VdmaAsyncInputStream : public VdmaInputStreamBase +{ +public: + VdmaAsyncInputStream(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t stream_interface, + hailo_status &status); + virtual ~VdmaAsyncInputStream() = default; + + virtual hailo_status wait_for_ready(size_t transfer_size, std::chrono::milliseconds timeout) override; + virtual hailo_status write_async(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque); + +private: + virtual Expected sync_write_raw_buffer(const MemoryView &buffer) override; + virtual hailo_status sync_write_all_raw_buffer_no_transform_impl(void *buffer, size_t offset, size_t size) override; +}; + +class VdmaAsyncOutputStream : public VdmaOutputStreamBase +{ +public: + VdmaAsyncOutputStream(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t interface, + hailo_status &status); + virtual ~VdmaAsyncOutputStream() = default; + + virtual hailo_status wait_for_ready(size_t transfer_size, std::chrono::milliseconds timeout) override; + virtual hailo_status read_async(std::shared_ptr buffer, const TransferDoneCallback &user_callback, void *opaque = nullptr) override; + +private: + virtual Expected sync_read_raw_buffer(MemoryView &buffer); + virtual hailo_status read_all(MemoryView &buffer) override; +}; + + +} /* namespace hailort */ + +#endif /* _HAILO_VDMA_ASYNC_STREAM_HPP_ */ diff --git a/hailort/libhailort/src/vdma/vdma_config_activated_core_op.cpp b/hailort/libhailort/src/vdma/vdma_config_activated_core_op.cpp new file mode 100644 index 0000000..7d7810c --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_config_activated_core_op.cpp @@ -0,0 +1,140 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_config_activated_core_op.cpp + * @brief VdmaConfigActivatedCoreOp implementation + **/ + +#include "vdma/vdma_config_activated_core_op.hpp" +#include "device_common/control.hpp" + +#include + + +namespace hailort +{ + +Expected VdmaConfigActivatedCoreOp::create( + ActiveCoreOpHolder &active_core_op_holder, + const std::string &core_op_name, + std::shared_ptr resources_manager, + // hailo_activate_network_group_params_t is currently an empty holder, if anything will be added to it , + // it will require a check that these params will be relevant for this one core op only. + const hailo_activate_network_group_params_t &network_group_params, + uint16_t dynamic_batch_size, + std::map> &input_streams, + std::map> &output_streams, + EventPtr core_op_activated_event, + AccumulatorPtr deactivation_time_accumulator, + bool resume_pending_stream_transfers, + CoreOp &core_op) +{ + CHECK(!active_core_op_holder.is_any_active(), make_unexpected(HAILO_INVALID_OPERATION), + "core-op is currently active. You must deactivate before activating another core-op"); + + CHECK_ARG_NOT_NULL_AS_EXPECTED(deactivation_time_accumulator); + + auto status = HAILO_UNINITIALIZED; + VdmaConfigActivatedCoreOp object(core_op_name, network_group_params, dynamic_batch_size, input_streams, output_streams, + std::move(resources_manager), active_core_op_holder, std::move(core_op_activated_event), + deactivation_time_accumulator, resume_pending_stream_transfers, core_op, status); + CHECK_SUCCESS_AS_EXPECTED(status); + + return object; +} + +VdmaConfigActivatedCoreOp::VdmaConfigActivatedCoreOp( + const std::string &core_op_name, + const hailo_activate_network_group_params_t &network_group_params, + uint16_t dynamic_batch_size, + std::map> &input_streams, + std::map> &output_streams, + std::shared_ptr &&resources_manager, + ActiveCoreOpHolder &active_core_op_holder, + EventPtr &&core_op_activated_event, + AccumulatorPtr deactivation_time_accumulator, + bool resume_pending_stream_transfers, + CoreOp &core_op, + hailo_status &status) : + ActivatedCoreOp(network_group_params, input_streams, output_streams, + std::move(core_op_activated_event), status), + m_core_op_name(core_op_name), + m_should_reset_core_op(true), + m_active_core_op_holder(active_core_op_holder), + m_resources_manager(std::move(resources_manager)), + m_deactivation_time_accumulator(deactivation_time_accumulator), + m_keep_nn_config_during_reset(false) +{ + // Validate ActivatedCoreOp status + if (HAILO_SUCCESS != status) { + return; + } + + // We know core_op is a VdmaConfigCoreOp + status = core_op.activate_impl(dynamic_batch_size, resume_pending_stream_transfers); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Error activating core-op"); + return; + } +} + +VdmaConfigActivatedCoreOp::VdmaConfigActivatedCoreOp(VdmaConfigActivatedCoreOp &&other) noexcept : + ActivatedCoreOp(std::move(other)), + m_core_op_name(std::move(other.m_core_op_name)), + m_should_reset_core_op(std::exchange(other.m_should_reset_core_op, false)), + m_active_core_op_holder(other.m_active_core_op_holder), + m_resources_manager(std::move(other.m_resources_manager)), + m_deactivation_time_accumulator(std::move(other.m_deactivation_time_accumulator)), + m_keep_nn_config_during_reset(std::move(other.m_keep_nn_config_during_reset)) +{} + +VdmaConfigActivatedCoreOp::~VdmaConfigActivatedCoreOp() +{ + if (!m_should_reset_core_op) { + return; + } + + auto status = HAILO_UNINITIALIZED; + const auto start_time = std::chrono::steady_clock::now(); + + auto core_op_ref = m_active_core_op_holder.get(); + if (!core_op_ref.has_value()) { + LOGGER__ERROR("Error getting core-op (status {})", status); + return; + } + + auto vdma_config_core_op = core_op_ref.value(); + + status = vdma_config_core_op.get().deactivate_impl(m_keep_nn_config_during_reset); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed deactivating core-op (status {})", status); + } + + const auto elapsed_time_ms = std::chrono::duration( + std::chrono::steady_clock::now() - start_time).count(); + LOGGER__INFO("Deactivating took {} ms", elapsed_time_ms); + m_deactivation_time_accumulator->add_data_point(elapsed_time_ms); +} + +// TODO: add get_core_op_name() for better code readability? +const std::string &VdmaConfigActivatedCoreOp::get_network_group_name() const +{ + // network_group name is the same as core_op name in this case. + // VdmaConfigActivatedCoreOp should be used only for single core ops network groups. + return m_core_op_name; +} + +Expected VdmaConfigActivatedCoreOp::get_intermediate_buffer(const IntermediateBufferKey &key) +{ + return m_resources_manager->read_intermediate_buffer(key); +} + +hailo_status VdmaConfigActivatedCoreOp::set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) +{ + m_keep_nn_config_during_reset = keep_nn_config_during_reset; + return HAILO_SUCCESS; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_config_activated_core_op.hpp b/hailort/libhailort/src/vdma/vdma_config_activated_core_op.hpp new file mode 100644 index 0000000..336e534 --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_config_activated_core_op.hpp @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_config_activated_core_op.hpp + * @brief Represent activated core-op from HEF + **/ + +#ifndef _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_ACTIVATED_CORE_OP_HPP_ +#define _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_ACTIVATED_CORE_OP_HPP_ + +#include "hailo/expected.hpp" + +#include "vdma/channel/boundary_channel.hpp" +#include "core_op/active_core_op_holder.hpp" +#include "core_op/resource_manager/resource_manager.hpp" + +#include +#include +#include + + +namespace hailort +{ + +class VdmaConfigActivatedCoreOp : public ActivatedCoreOp +{ +public: + + static Expected create( + ActiveCoreOpHolder &active_core_op_holder, + const std::string &core_op_name, + std::shared_ptr resources_manager, + const hailo_activate_network_group_params_t &network_group_params, + uint16_t dynamic_batch_size, + std::map> &input_streams, + std::map> &output_streams, + EventPtr core_op_activated_event, + AccumulatorPtr deactivation_time_accumulator, + bool resume_pending_stream_transfers, + CoreOp &core_op); + + virtual ~VdmaConfigActivatedCoreOp(); + + VdmaConfigActivatedCoreOp(const VdmaConfigActivatedCoreOp &other) = delete; + VdmaConfigActivatedCoreOp &operator=(const VdmaConfigActivatedCoreOp &other) = delete; + VdmaConfigActivatedCoreOp &operator=(VdmaConfigActivatedCoreOp &&other) = delete; + VdmaConfigActivatedCoreOp(VdmaConfigActivatedCoreOp &&other) noexcept; + + virtual const std::string &get_network_group_name() const override; + virtual Expected get_intermediate_buffer(const IntermediateBufferKey &key) override; + virtual hailo_status set_keep_nn_config_during_reset(const bool keep_nn_config_during_reset) override; + +private: + VdmaConfigActivatedCoreOp( + const std::string &core_op_name, + const hailo_activate_network_group_params_t &network_group_params, + uint16_t dynamic_batch_size, + std::map> &input_streams, + std::map> &output_streams, + std::shared_ptr &&resources_manager, + ActiveCoreOpHolder &active_core_op_holder, + EventPtr &&core_op_activated_event, + AccumulatorPtr deactivation_time_accumulator, + bool resume_pending_stream_transfers, + CoreOp &core_op, + hailo_status &status); + + std::string m_core_op_name; + bool m_should_reset_core_op; + ActiveCoreOpHolder &m_active_core_op_holder; + std::shared_ptr m_resources_manager; + AccumulatorPtr m_deactivation_time_accumulator; + bool m_keep_nn_config_during_reset; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_ACTIVATED_CORE_OP_HPP_ */ diff --git a/hailort/libhailort/src/vdma/vdma_config_core_op.cpp b/hailort/libhailort/src/vdma/vdma_config_core_op.cpp new file mode 100644 index 0000000..88c0968 --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_config_core_op.cpp @@ -0,0 +1,168 @@ +#include "utils/profiler/tracer_macros.hpp" +#include "vdma/vdma_config_core_op.hpp" +#include "network_group/network_group_internal.hpp" +#include "net_flow/pipeline/vstream_internal.hpp" + + +namespace hailort +{ + +Expected VdmaConfigCoreOp::create(ActiveCoreOpHolder &active_core_op_holder, + const ConfigureNetworkParams &config_params, + std::shared_ptr resources_manager, + std::shared_ptr metadata) +{ + auto status = HAILO_UNINITIALIZED; + + VdmaConfigCoreOp object(active_core_op_holder, config_params, + std::move(resources_manager), metadata, status); + CHECK_SUCCESS_AS_EXPECTED(status); + + return object; +} + +VdmaConfigCoreOp::VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder, + const ConfigureNetworkParams &config_params, + std::shared_ptr &&resources_manager, + std::shared_ptr metadata, hailo_status &status) : + CoreOp(config_params, metadata, status), + m_active_core_op_holder(active_core_op_holder), + m_resources_manager(std::move(resources_manager)) +{} + +hailo_status VdmaConfigCoreOp::activate_impl(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) +{ + auto status = HAILO_UNINITIALIZED; + + // Check that no network is currently activated + CHECK(!m_active_core_op_holder.is_any_active(), HAILO_INTERNAL_FAILURE, + "Cant activate network because a network is already activated"); + + m_active_core_op_holder.set(*this); + + status = m_resources_manager->set_inter_context_channels_dynamic_batch_size(dynamic_batch_size); + CHECK_SUCCESS(status, "Failed to set inter-context channels dynamic batch size."); + + status = m_resources_manager->enable_state_machine(dynamic_batch_size); + CHECK_SUCCESS(status, "Failed to activate state-machine"); + + status = m_resources_manager->start_vdma_interrupts_dispatcher(); + CHECK_SUCCESS(status, "Failed to start vdma interrupts"); + + // Low-level streams assume that the vdma channels are enabled (happens in `enable_state_machine`), and that + // the interrupt dispatcher is running (so they can wait for interrupts). + status = activate_low_level_streams(dynamic_batch_size, resume_pending_stream_transfers); + CHECK_SUCCESS(status, "Failed to activate low level streams"); + + status = m_core_op_activated_event->signal(); + CHECK_SUCCESS(status, "Failed to signal network activation event"); + + return HAILO_SUCCESS; +} + +hailo_status VdmaConfigCoreOp::deactivate_impl(bool keep_nn_config_during_reset) +{ + auto status = deactivate_host_resources(); + CHECK_SUCCESS(status); + + status = m_resources_manager->reset_state_machine(keep_nn_config_during_reset); + CHECK_SUCCESS(status, "Failed to reset context switch state machine"); + + // After the state machine has been reset the vdma channels are no longer active, so we + // can cancel pending async transfers, thus allowing vdma buffers linked to said transfers to be freed + status = m_resources_manager->cancel_pending_async_transfers(); + CHECK_SUCCESS(status, "Failed to cancel pending async transfers"); + + return HAILO_SUCCESS; +} + +hailo_status VdmaConfigCoreOp::deactivate_host_resources() +{ + // Check that network is currently activated + CHECK(m_active_core_op_holder.is_any_active(), HAILO_INTERNAL_FAILURE, + "Cant Deactivate network because no network is already activated"); + + // Make sure the core op we are deactivating is this object + auto active_core_op_ref = m_active_core_op_holder.get().value(); + CHECK(this == std::addressof(active_core_op_ref.get()), HAILO_INTERNAL_FAILURE, + "Trying to deactivate different network goup"); + + m_active_core_op_holder.clear(); + + m_core_op_activated_event->reset(); + + auto status = deactivate_low_level_streams(); + CHECK_SUCCESS(status, "Failed to deactivate low level streams"); + + // After disabling the vdma interrupts, we may still get some interrupts. On HRT-9430 we need to clean them. + status = m_resources_manager->stop_vdma_interrupts_dispatcher(); + CHECK_SUCCESS(status, "Failed to stop vdma interrupts"); + + return HAILO_SUCCESS; +} + +Expected> VdmaConfigCoreOp::create_activated_network_group( + const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size, + bool resume_pending_stream_transfers) +{ + auto start_time = std::chrono::steady_clock::now(); + auto activated_net_group = VdmaConfigActivatedCoreOp::create( + m_active_core_op_holder, name(), m_resources_manager, network_group_params, dynamic_batch_size, + m_input_streams, m_output_streams, m_core_op_activated_event, m_deactivation_time_accumulator, + resume_pending_stream_transfers, *this); + const auto elapsed_time_ms = std::chrono::duration( + std::chrono::steady_clock::now() - start_time).count(); + CHECK_EXPECTED(activated_net_group); + + LOGGER__INFO("Activating {} took {} milliseconds. Note that the function is asynchronous and" + " thus the network is not fully activated yet.", name(), elapsed_time_ms); + m_activation_time_accumulator->add_data_point(elapsed_time_ms); + + std::unique_ptr activated_net_group_ptr = + make_unique_nothrow(activated_net_group.release()); + CHECK_AS_EXPECTED(nullptr != activated_net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); + + return activated_net_group_ptr; +} + +Expected VdmaConfigCoreOp::get_default_streams_interface() +{ + return m_resources_manager->get_default_streams_interface(); +} + +bool VdmaConfigCoreOp::is_scheduled() const +{ + // Scheduler allowed only when working with VDevice and scheduler enabled. + return false; +} + +hailo_status VdmaConfigCoreOp::set_scheduler_timeout(const std::chrono::milliseconds &/*timeout*/, const std::string &/*network_name*/) +{ + LOGGER__ERROR("Setting scheduler's timeout is only allowed when working with VDevice and scheduler enabled"); + return HAILO_INVALID_OPERATION; +} + +hailo_status VdmaConfigCoreOp::set_scheduler_threshold(uint32_t /*threshold*/, const std::string &/*network_name*/) +{ + LOGGER__ERROR("Setting scheduler's threshold is only allowed when working with VDevice and scheduler enabled"); + return HAILO_INVALID_OPERATION; +} + +hailo_status VdmaConfigCoreOp::set_scheduler_priority(uint8_t /*priority*/, const std::string &/*network_name*/) +{ + LOGGER__ERROR("Setting scheduler's priority is only allowed when working with VDevice and scheduler enabled"); + return HAILO_INVALID_OPERATION; +} + +Expected> VdmaConfigCoreOp::get_latency_meters() +{ + auto latency_meters = m_resources_manager->get_latency_meters(); + return make_shared_nothrow(latency_meters); +} + +Expected VdmaConfigCoreOp::get_boundary_vdma_channel_by_stream_name(const std::string &stream_name) +{ + return m_resources_manager->get_boundary_vdma_channel_by_stream_name(stream_name); +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_config_core_op.hpp b/hailort/libhailort/src/vdma/vdma_config_core_op.hpp new file mode 100644 index 0000000..740d4a3 --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_config_core_op.hpp @@ -0,0 +1,98 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_config_core_op.hpp + * @brief Represent core-op from HEF file that can be activated + * + * This core-op can be used for both single or multi context core-ops but for PCIE only + **/ + +#ifndef _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_CORE_OP_HPP_ +#define _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_CORE_OP_HPP_ + +#include "hailo/hailort.h" +#include "hailo/network_group.hpp" +#include "hailo/hailort_defaults.hpp" + +#include "common/utils.hpp" + +#include "vdma/channel/boundary_channel.hpp" +#include "core_op/resource_manager/resource_manager.hpp" +#include "vdma/vdma_config_activated_core_op.hpp" +#include "core_op/active_core_op_holder.hpp" + +#include "control_protocol.h" +#include +#include +#include +#include + + +namespace hailort +{ + + +class VdmaConfigCoreOp : public CoreOp +{ +public: + static Expected create(ActiveCoreOpHolder &active_core_op_holder, + const ConfigureNetworkParams &config_params, + std::shared_ptr resources_managers, + std::shared_ptr metadata); + + std::shared_ptr &get_resources_manager() + { + return m_resources_manager; + } + + // Functions to activate and deactivate core ops for scheduler - dont create ActivatedNetworkGroup objects + // Note: Care should be taken when calling activate_impl with resume_pending_stream_transfers = true. + // If an output stream has outstanding transfers, and the NG is deactivated (via deactivate_impl) before they + // have been completed, then these pending transfers may be overwritten upon channel activation. + // Hence, when setting resume_pending_stream_transfers = true, the caller must validate that all pending + // reads have been received (i.e. an int has been raised for this transfer) + virtual hailo_status activate_impl(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; + // Will first deactivate host resources (via deactivate_host_resources) and then reset the core-op on the fw + virtual hailo_status deactivate_impl(bool keep_nn_config_during_reset) override; + // Deactivate all resources related to the core-op on the host, but without resetting the core-op on the fw + hailo_status deactivate_host_resources(); + + virtual Expected> create_activated_network_group( + const hailo_activate_network_group_params_t &network_group_params, uint16_t dynamic_batch_size, + bool resume_pending_stream_transfers) override; + + virtual Expected get_default_streams_interface() override; + + virtual Expected> get_latency_meters() override; + virtual Expected get_boundary_vdma_channel_by_stream_name( + const std::string &stream_name) override; + + virtual bool is_scheduled() const override; + virtual hailo_status set_scheduler_timeout(const std::chrono::milliseconds &timeout, const std::string &network_name) override; + virtual hailo_status set_scheduler_threshold(uint32_t threshold, const std::string &network_name) override; + virtual hailo_status set_scheduler_priority(uint8_t priority, const std::string &network_name) override; + + virtual ~VdmaConfigCoreOp() = default; + VdmaConfigCoreOp(const VdmaConfigCoreOp &other) = delete; + VdmaConfigCoreOp &operator=(const VdmaConfigCoreOp &other) = delete; + VdmaConfigCoreOp &operator=(VdmaConfigCoreOp &&other) = delete; + VdmaConfigCoreOp(VdmaConfigCoreOp &&other) noexcept : CoreOp(std::move(other)), + m_active_core_op_holder(other.m_active_core_op_holder), + m_resources_manager(std::move(other.m_resources_manager)) + {} + +private: + VdmaConfigCoreOp(ActiveCoreOpHolder &active_core_op_holder, + const ConfigureNetworkParams &config_params, + std::shared_ptr &&resources_manager, + std::shared_ptr metadata, hailo_status &status); + + ActiveCoreOpHolder &m_active_core_op_holder; + std::shared_ptr m_resources_manager; +}; + +} /* namespace hailort */ + +#endif /* _HAILO_CONTEXT_SWITCH_VDMA_CONFIG_CORE_OP_HPP_ */ diff --git a/hailort/libhailort/src/vdma/vdma_config_manager.cpp b/hailort/libhailort/src/vdma/vdma_config_manager.cpp new file mode 100644 index 0000000..0bf8d4f --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_config_manager.cpp @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) +**/ +/** + * @file vdma_config_manager.cpp + * @brief Vdma config manager implementation + **/ + +#include "vdma_config_manager.hpp" +#include "hailo/hailort.h" + +namespace hailort +{ + +hailo_status VdmaConfigManager::switch_core_op(std::shared_ptr current_active_core_op, + std::shared_ptr next_core_op, const uint16_t batch_size, bool resume_pending_stream_transfers) +{ + static const auto RESET_NN_CONFIG = false; + CHECK((nullptr != current_active_core_op) || (nullptr != next_core_op), HAILO_INVALID_ARGUMENT); + + if (nullptr == current_active_core_op) { + // Activate first core-op + return next_core_op->activate_impl(batch_size, resume_pending_stream_transfers); + } else if (nullptr == next_core_op) { + // Deactivate last core-op + return current_active_core_op->deactivate_impl(RESET_NN_CONFIG); + } + + // We're switching from current_active_core_op to next_core_op. + // Deactivate the current core-op on the host, meaning the fw state machine won't be reset. + // This will be handled by activating the next core-op. + auto status = current_active_core_op->deactivate_host_resources(); + CHECK_SUCCESS(status, "Failed deactivating current core-op"); + + // TODO: In mercury we need to reset after deactivate. This will be fixed in MSW-762 and the "if" will be removed + // when we make the nn_manager responsible to reset the nn-core. + if (Device::Type::INTEGRATED == current_active_core_op->get_resources_manager()->get_device().get_type()) { + status = current_active_core_op->get_resources_manager()->reset_state_machine(RESET_NN_CONFIG); + CHECK_SUCCESS(status, "Failed to reset state machine in switch core-op"); + } + + // Switch from the current core-op to the next core-op. I.e. current core-op will be deactivated and + // next core-op will be activated + status = next_core_op->activate_impl(batch_size, resume_pending_stream_transfers); + CHECK_SUCCESS(status, "Failed activating next core-op"); + + // Current core-op is now deactivated, so we can cancel pending async transfers + status = current_active_core_op->get_resources_manager()->cancel_pending_async_transfers(); + CHECK_SUCCESS(status, "Failed canceling pending async transfers from previous core-op"); + + return HAILO_SUCCESS; +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_config_manager.hpp b/hailort/libhailort/src/vdma/vdma_config_manager.hpp new file mode 100644 index 0000000..c42b6a8 --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_config_manager.hpp @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_config_manager.hpp + * @brief Manager of HEF parsing and vdma-core-op resources for Pcie devices (both single and multi context) + * + **/ + +#ifndef HAILO_VDMA_CONFIG_MANAGER_HPP_ +#define HAILO_VDMA_CONFIG_MANAGER_HPP_ + +#include "hailo/hailort.h" + +#include "common/utils.hpp" + +#include "vdma/vdma_config_core_op.hpp" + + +namespace hailort +{ + +class VdmaConfigManager final +{ +public: + VdmaConfigManager() = delete; + + static hailo_status switch_core_op(std::shared_ptr current_active_core_op, + std::shared_ptr next_core_op, const uint16_t batch_size, bool resume_pending_stream_transfers); +}; + +} /* namespace hailort */ + +#endif /* HAILO_VDMA_CONFIG_MANAGER_HPP_ */ diff --git a/hailort/libhailort/src/vdma_device.cpp b/hailort/libhailort/src/vdma/vdma_device.cpp similarity index 55% rename from hailort/libhailort/src/vdma_device.cpp rename to hailort/libhailort/src/vdma/vdma_device.cpp index bf27195..c560a65 100644 --- a/hailort/libhailort/src/vdma_device.cpp +++ b/hailort/libhailort/src/vdma/vdma_device.cpp @@ -9,17 +9,19 @@ * TODO: doc **/ -#include "vdma_device.hpp" -#include "vdma_descriptor_list.hpp" -#include "context_switch/multi_context/vdma_config_manager.hpp" -#include "pcie_device.hpp" -#include "core_device.hpp" -#include "control.hpp" -#include "context_switch/resource_manager_builder.hpp" +#include "vdma/vdma_device.hpp" +#include "vdma/memory/descriptor_list.hpp" +#include "vdma/vdma_config_manager.hpp" +#include "vdma/pcie/pcie_device.hpp" +#include "vdma/integrated/integrated_device.hpp" +#include "device_common/control.hpp" +#include "core_op/resource_manager/resource_manager_builder.hpp" +#include "core_op/core_op.hpp" #include #include + namespace hailort { @@ -39,8 +41,8 @@ VdmaDevice::VdmaDevice(HailoRTDriver &&driver, Device::Type type, const std::str Expected> VdmaDevice::create(const std::string &device_id) { const bool DONT_LOG_ON_FAILURE = false; - if (CoreDevice::DEVICE_ID == device_id) { - auto device = CoreDevice::create(); + if (IntegratedDevice::DEVICE_ID == device_id) { + auto device = IntegratedDevice::create(); CHECK_EXPECTED(device);; return std::unique_ptr(device.release()); } @@ -120,126 +122,75 @@ Expected VdmaDevice::add_hef(Hef &hef, const Netwo status = Control::reset_context_switch_state_machine(*this, REMOVE_NN_CONFIG_DURING_RESET); CHECK_SUCCESS_AS_EXPECTED(status); + // In case of mercury need to reset nn core before activating network group to clear prior nn core state + if (Device::Type::INTEGRATED == get_type()) { + // On core device, the nn_manager is not responsible to reset the nn-core so + // we use the SCU control for that. + status = reset(HAILO_RESET_DEVICE_MODE_NN_CORE); + CHECK_SUCCESS_AS_EXPECTED(status); + } + status = Control::clear_configured_apps(*this); CHECK_SUCCESS_AS_EXPECTED(status, "Failed to clear configured network groups with status {}", status); + assert(nullptr == m_vdma_interrupts_dispatcher); + auto interrupts_dispatcher = vdma::InterruptsDispatcher::create(std::ref(m_driver)); + CHECK_EXPECTED(interrupts_dispatcher); + m_vdma_interrupts_dispatcher = interrupts_dispatcher.release(); + m_is_configured = true; } - auto device_arch = get_architecture(); - CHECK_EXPECTED(device_arch); - - auto partial_clusters_layout_bitmap_exp = Control::get_partial_clusters_layout_bitmap(*this); - CHECK_EXPECTED(partial_clusters_layout_bitmap_exp); - auto partial_clusters_layout_bitmap = partial_clusters_layout_bitmap_exp.release(); - - auto &hef_net_groups = hef.pimpl->network_groups(); - ConfiguredNetworkGroupVector added_network_groups; - // TODO: can be optimized (add another loop the allocate the network group we're adding) - added_network_groups.reserve(hef_net_groups.size()); - auto configure_params_copy = configure_params; - for (const auto &hef_net_group : hef_net_groups) { - const std::string &network_group_name = HefUtils::get_network_group_name(*hef_net_group, SupportedFeatures()); - auto hef_core_ops = hef.pimpl->core_ops(network_group_name); - assert(hef_core_ops.size() == 1); - std::vector> network_group_metadata_ptrs; - network_group_metadata_ptrs.reserve(hef_core_ops.size()); - const auto prev_network_group_count = m_network_groups.size(); - const auto total_network_group_count = prev_network_group_count + hef_core_ops.size(); - CHECK_AS_EXPECTED(CONTROL_PROTOCOL__MAX_CONTEXT_SWITCH_APPLICATIONS >= total_network_group_count, - HAILO_INVALID_OPERATION, - "Can't add {} network groups from HEF. Currently {} network groups are configured; maximum allowed network groups: {}.", - hef_core_ops.size(), prev_network_group_count, CONTROL_PROTOCOL__MAX_CONTEXT_SWITCH_APPLICATIONS); - - auto hef_arch = hef.pimpl->get_device_arch(); - - auto current_net_group_index = static_cast(prev_network_group_count); - for (const auto &core_op : hef_core_ops) { - auto expected_partial_core_op = Hef::Impl::get_core_op_per_arch(core_op, hef_arch, device_arch.value(), - partial_clusters_layout_bitmap); - CHECK_EXPECTED(expected_partial_core_op); - auto partial_core_op = expected_partial_core_op.release(); - status = Hef::Impl::validate_core_op_unique_layer_names(*partial_core_op); - CHECK_SUCCESS_AS_EXPECTED(status); - - // TODO: keep metadata per core_op (HRT-8639) - // TODO: decide about core_op names - align with the Compiler - auto network_group_metadata = hef.pimpl->get_network_group_metadata(network_group_name, partial_clusters_layout_bitmap); - CHECK_EXPECTED(network_group_metadata); - - auto network_group_metadata_ptr = make_shared_nothrow(network_group_metadata.release()); - CHECK_AS_EXPECTED(nullptr != network_group_metadata_ptr, HAILO_OUT_OF_HOST_MEMORY); - network_group_metadata_ptrs.push_back(network_group_metadata_ptr); - } - - /* If NG params are present, use them - If no configure params are given, use default*/ - ConfigureNetworkParams config_params{}; - if (contains(configure_params, network_group_name)) { - config_params = configure_params_copy.at(network_group_name); - configure_params_copy.erase(network_group_name); - } else if (configure_params.empty()) { - auto stream_interface = get_default_streams_interface(); - CHECK_EXPECTED(stream_interface); - auto config_params_exp = hef.create_configure_params(stream_interface.value(), network_group_name); - CHECK_EXPECTED(config_params_exp); - config_params = config_params_exp.release(); - } else { - continue; - } - /* Validate batch size (network group batch size vs network batch size) */ - status = Hef::Impl::update_network_batch_size(config_params); - CHECK_SUCCESS_AS_EXPECTED(status); - auto network_group = create_configured_network_group(network_group_metadata_ptrs, - hef, config_params, current_net_group_index); - CHECK_EXPECTED(network_group); - added_network_groups.emplace_back(network_group.release()); - current_net_group_index++; - } - std::string unmatched_keys = ""; - for (const auto &pair : configure_params_copy) { - unmatched_keys.append(" "); - unmatched_keys.append(pair.first); - } - CHECK_AS_EXPECTED(unmatched_keys.size() == 0, HAILO_INVALID_ARGUMENT, - "Some network group names in the configuration are not found in the hef file:{}", unmatched_keys); + auto added_network_groups = create_networks_group_vector(hef, configure_params); + CHECK_EXPECTED(added_network_groups); return added_network_groups; } +// TODO: HRT-9551 Create CoreOpMetadata and CoreOp in the same loop Expected> VdmaDevice::create_configured_network_group( - const std::vector> &network_group_metadatas, + std::vector> &core_ops_metadata, Hef &hef, const ConfigureNetworkParams &config_params, - uint8_t network_group_index) + uint8_t current_core_op_index) { - // TODO: keep metadata per core_op (HRT-8639) - assert(network_group_metadatas.size() == 1); - auto network_group_metadata = network_group_metadatas[0]; + std::vector> core_ops; + core_ops.reserve(core_ops_metadata.size()); + + // TODO: keep metadata per core_op (HRT-9551) + // TODO: HRT-8875 support multiple core ops + assert(core_ops_metadata.size() == 1); + auto core_op_metadata = core_ops_metadata[0]; /* build HEF supported features */ - auto resource_manager = ResourcesManagerBuilder::build(network_group_index, - *this, get_driver(), config_params, network_group_metadata, hef.pimpl->get_device_arch()); + auto resource_manager = ResourcesManagerBuilder::build(current_core_op_index, + *this, get_driver(), config_params, core_op_metadata, hef.pimpl->get_device_arch()); CHECK_EXPECTED(resource_manager); - auto net_flow_ops = hef.pimpl->post_process_ops(network_group_metadata->network_group_name()); - auto net_group = VdmaConfigNetworkGroup::create(m_active_net_group_holder, config_params, - resource_manager.release(), hef.hash(), network_group_metadata, std::move(net_flow_ops)); + auto core_op = VdmaConfigCoreOp::create(m_active_core_op_holder, config_params, + resource_manager.release(), core_op_metadata); - auto net_group_ptr = make_shared_nothrow(net_group.release()); - CHECK_AS_EXPECTED(nullptr != net_group_ptr, HAILO_OUT_OF_HOST_MEMORY); + auto core_op_ptr = make_shared_nothrow(core_op.release()); + CHECK_AS_EXPECTED(nullptr != core_op_ptr, HAILO_OUT_OF_HOST_MEMORY); - // TODO: move this func into VdmaConfigNetworkGroup c'tor - auto status = net_group_ptr->create_streams_from_config_params(*this); + // TODO: move this func into VdmaConfigCoreOp c'tor + auto status = core_op_ptr->create_streams_from_config_params(*this); CHECK_SUCCESS_AS_EXPECTED(status); - m_network_groups.emplace_back(net_group_ptr); - - // Check that all boundary streams were created - status = hef.pimpl->validate_boundary_streams_were_created(network_group_metadata->network_group_name(), *net_group_ptr); + // Check that all boundary streams were created + status = hef.pimpl->validate_boundary_streams_were_created(core_op_metadata->core_op_name(), core_op_ptr); CHECK_SUCCESS_AS_EXPECTED(status); - return Expected>(net_group_ptr); + core_ops.emplace_back(core_op_ptr); + m_core_ops.emplace_back(core_op_ptr); + + // TODO: HRT-8875 + auto net_flow_ops = hef.pimpl->post_process_ops(core_op_metadata->core_op_name()); + auto network_group_expected = ConfiguredNetworkGroupBase::create(config_params, std::move(core_ops), std::move(net_flow_ops)); + CHECK_EXPECTED(network_group_expected); + auto network_group_ptr = network_group_expected.release(); + + return Expected>(network_group_ptr); } Expected VdmaDevice::read_log(MemoryView &buffer, hailo_cpu_id_t cpu_id) @@ -266,7 +217,7 @@ hailo_reset_device_mode_t VdmaDevice::get_default_reset_mode() uint16_t VdmaDevice::get_default_desc_page_size() const { - return m_driver.calc_desc_page_size(DEFAULT_DESC_PAGE_SIZE); + return m_driver.calc_desc_page_size(vdma::DEFAULT_DESC_PAGE_SIZE); } hailo_status VdmaDevice::mark_as_used() @@ -274,6 +225,12 @@ hailo_status VdmaDevice::mark_as_used() return m_driver.mark_as_used(); } +ExpectedRef VdmaDevice::get_vdma_interrupts_dispatcher() +{ + CHECK_AS_EXPECTED(m_vdma_interrupts_dispatcher, HAILO_INTERNAL_FAILURE, "vDMA interrupt dispatcher wasn't created"); + return std::ref(*m_vdma_interrupts_dispatcher); +} + VdmaDevice::~VdmaDevice() { auto status = stop_notification_fetch_thread(); @@ -283,9 +240,107 @@ VdmaDevice::~VdmaDevice() if (m_is_configured) { status = Control::clear_configured_apps(*this); if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to clear conigured network groups with status {}", status); + LOGGER__ERROR("Failed to clear configured core-ops with status {}", status); + } + } +} + +Expected VdmaDevice::create_networks_group_vector(Hef &hef, const NetworkGroupsParamsMap &configure_params) +{ + auto partial_clusters_layout_bitmap_exp = Control::get_partial_clusters_layout_bitmap(*this); + CHECK_EXPECTED(partial_clusters_layout_bitmap_exp); + auto partial_clusters_layout_bitmap = partial_clusters_layout_bitmap_exp.release(); + + auto &hef_net_groups = hef.pimpl->network_groups(); + auto configure_params_copy = configure_params; + ConfiguredNetworkGroupVector added_network_groups; + // TODO: can be optimized (add another loop the allocate the network group we're adding) + added_network_groups.reserve(hef_net_groups.size()); + for (const auto &hef_net_group : hef_net_groups) { + const std::string &network_group_name = HefUtils::get_network_group_name(*hef_net_group, SupportedFeatures()); + const auto prev_core_op_count = m_core_ops.size(); + auto current_core_op_index = static_cast(prev_core_op_count); + + /* If NG params are present, use them + If no configure params are given, use default*/ + ConfigureNetworkParams config_params{}; + if (contains(configure_params, network_group_name)) { + config_params = configure_params_copy.at(network_group_name); + configure_params_copy.erase(network_group_name); + } else if (configure_params.empty()) { + auto stream_interface = get_default_streams_interface(); + CHECK_EXPECTED(stream_interface); + auto config_params_exp = hef.create_configure_params(stream_interface.value(), network_group_name); + CHECK_EXPECTED(config_params_exp); + config_params = config_params_exp.release(); + } else { + continue; } + + /* Validate batch size (network group batch size vs network batch size) */ + auto status = Hef::Impl::update_network_batch_size(config_params); + CHECK_SUCCESS_AS_EXPECTED(status); + + auto core_ops_metadata_ptrs = create_core_ops_metadata(hef, network_group_name, partial_clusters_layout_bitmap); + CHECK_EXPECTED(core_ops_metadata_ptrs); + + auto network_group_expected = create_configured_network_group(core_ops_metadata_ptrs.value(), + hef, config_params, current_core_op_index); + CHECK_EXPECTED(network_group_expected); + auto network_group_ptr = network_group_expected.release(); + + added_network_groups.emplace_back(network_group_ptr); + m_network_groups.push_back(network_group_ptr); + } + + std::string unmatched_keys = ""; + for (const auto &pair : configure_params_copy) { + unmatched_keys.append(" "); + unmatched_keys.append(pair.first); + } + CHECK_AS_EXPECTED(unmatched_keys.size() == 0, HAILO_INVALID_ARGUMENT, + "Some network group names in the configuration are not found in the hef file:{}", unmatched_keys); + + return added_network_groups; +} + +Expected>> VdmaDevice::create_core_ops_metadata(Hef &hef, const std::string &network_group_name, uint32_t partial_clusters_layout_bitmap) +{ + auto hef_core_ops = hef.pimpl->core_ops(network_group_name); + assert(1 == hef_core_ops.size()); + + std::vector> core_ops_metadata_ptrs; + core_ops_metadata_ptrs.reserve(hef_core_ops.size()); + const auto prev_core_ops_count = m_core_ops.size(); + const auto total_core_ops_count = prev_core_ops_count + hef_core_ops.size(); + CHECK_AS_EXPECTED(CONTROL_PROTOCOL__MAX_CONTEXT_SWITCH_APPLICATIONS >= total_core_ops_count, + HAILO_INVALID_OPERATION, + "Can't add {} core-ops from HEF. Currently {} core-ops are configured; maximum allowed core-ops: {}.", + hef_core_ops.size(), prev_core_ops_count, CONTROL_PROTOCOL__MAX_CONTEXT_SWITCH_APPLICATIONS); + + auto hef_arch = hef.pimpl->get_device_arch(); + auto device_arch = get_architecture(); + CHECK_EXPECTED(device_arch); + + for (const auto &hef_core_op : hef_core_ops) { + auto expected_partial_core_op = Hef::Impl::get_core_op_per_arch(hef_core_op, hef_arch, device_arch.value(), + partial_clusters_layout_bitmap); + CHECK_EXPECTED(expected_partial_core_op); + auto partial_core_op = expected_partial_core_op.release(); + auto status = Hef::Impl::validate_core_op_unique_layer_names(*partial_core_op); + CHECK_SUCCESS_AS_EXPECTED(status); + + // TODO: keep metadata per core_op (HRT-9551) + // TODO: decide about core_op names - align with the Compiler + auto core_op_metadata = hef.pimpl->get_core_op_metadata(network_group_name, partial_clusters_layout_bitmap); + CHECK_EXPECTED(core_op_metadata); + + auto core_op_metadata_ptr = make_shared_nothrow(core_op_metadata.release()); + CHECK_AS_EXPECTED(nullptr != core_op_metadata_ptr, HAILO_OUT_OF_HOST_MEMORY); + core_ops_metadata_ptrs.emplace_back(core_op_metadata_ptr); } + + return core_ops_metadata_ptrs; } } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma_device.hpp b/hailort/libhailort/src/vdma/vdma_device.hpp similarity index 61% rename from hailort/libhailort/src/vdma_device.hpp rename to hailort/libhailort/src/vdma/vdma_device.hpp index f25a6e0..5aea085 100644 --- a/hailort/libhailort/src/vdma_device.hpp +++ b/hailort/libhailort/src/vdma/vdma_device.hpp @@ -13,9 +13,12 @@ #include "hailo/hailort.h" #include "hailo/expected.hpp" -#include "device_internal.hpp" -#include "context_switch/network_group_internal.hpp" + +#include "device_common/device_internal.hpp" +#include "network_group/network_group_internal.hpp" #include "os/hailort_driver.hpp" +#include "vdma/channel/interrupts_dispatcher.hpp" + namespace hailort { @@ -38,6 +41,8 @@ public: return std::ref(m_driver); }; + ExpectedRef get_vdma_interrupts_dispatcher(); + protected: VdmaDevice(HailoRTDriver &&driver, Type type, const std::string &device_id); @@ -48,15 +53,24 @@ protected: virtual Expected add_hef(Hef &hef, const NetworkGroupsParamsMap &configure_params) override; HailoRTDriver m_driver; - std::vector> m_network_groups; - ActiveNetGroupHolder m_active_net_group_holder; + std::vector> m_core_ops; + std::vector> m_network_groups; // TODO: HRT-9547 - Remove when ConfiguredNetworkGroup will be kept in global context + + // The vdma interrupts dispatcher contains a callback with a reference to the current activated network group + // (reference to the ResourcesManager). Hence, it must be destructed before the networks groups are destructed. + std::unique_ptr m_vdma_interrupts_dispatcher; + + ActiveCoreOpHolder m_active_core_op_holder; bool m_is_configured; private: Expected> create_configured_network_group( - const std::vector> &network_group_metadatas, + std::vector> &core_ops, Hef &hef, const ConfigureNetworkParams &config_params, uint8_t network_group_index); + Expected create_networks_group_vector(Hef &hef, const NetworkGroupsParamsMap &configure_params); + Expected>> create_core_ops_metadata(Hef &hef, const std::string &network_group_name, + uint32_t partial_clusters_layout_bitmap); }; } /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_mapped_buffer_impl.cpp b/hailort/libhailort/src/vdma/vdma_mapped_buffer_impl.cpp deleted file mode 100644 index f299f46..0000000 --- a/hailort/libhailort/src/vdma/vdma_mapped_buffer_impl.cpp +++ /dev/null @@ -1,110 +0,0 @@ -#include "vdma_mapped_buffer_impl.hpp" - -namespace hailort { -namespace vdma { - -#if defined(__linux__) || defined(_MSC_VER) - -Expected VdmaMappedBufferImpl::allocate_vdma_buffer(HailoRTDriver &driver, size_t required_size) -{ - // Check if driver should be allocated from driver or from user - if (driver.allocate_driver_buffer()) { - auto driver_buffer_handle = driver.vdma_low_memory_buffer_alloc(required_size); - CHECK_EXPECTED(driver_buffer_handle); - - uintptr_t driver_buff_handle = driver_buffer_handle.release(); - - auto mapped_buffer = MmapBuffer::create_file_map(required_size, driver.fd(), driver_buff_handle); - CHECK_EXPECTED(mapped_buffer); - - return VdmaMappedBufferImpl(mapped_buffer.release(), driver_buff_handle, driver); - } - else { - auto mapped_buffer = MmapBuffer::create_shared_memory(required_size); - CHECK_EXPECTED(mapped_buffer); - return VdmaMappedBufferImpl(mapped_buffer.release(), HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE, driver); - } -} - -VdmaMappedBufferImpl::~VdmaMappedBufferImpl() -{ - if (m_mapped_buffer) { - if (HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE != m_driver_mapped_buffer_identifier) { - m_driver.vdma_low_memory_buffer_free(m_driver_mapped_buffer_identifier); - } - } -} - -#elif defined(__QNX__) - -#include - -const int VdmaMappedBufferImpl::INVALID_FD = -1; -const shm_handle_t VdmaMappedBufferImpl::INVALID_HANDLE = (shm_handle_t)-1; -const char* VdmaMappedBufferImpl::VDMA_BUFFER_TYPE_MEMORY_NAME = "/memory/below4G/ram/below1G"; - -Expected VdmaMappedBufferImpl::allocate_vdma_buffer(HailoRTDriver &driver, size_t required_size) -{ - // Desctructor of type_mem_fd will close fd - FileDescriptor type_mem_fd(posix_typed_mem_open(VDMA_BUFFER_TYPE_MEMORY_NAME, O_RDWR, POSIX_TYPED_MEM_ALLOCATE)); - if (INVALID_FD == type_mem_fd) { - LOGGER__ERROR("Error getting fd from typed memory of type {}, errno {}\n", VDMA_BUFFER_TYPE_MEMORY_NAME, - errno); - return make_unexpected(HAILO_INTERNAL_FAILURE); - } - - vdma_mapped_buffer_driver_identifier driver_buff_handle; - driver_buff_handle.shm_fd = shm_open(SHM_ANON, O_RDWR | O_CREAT, 0777); - CHECK_AS_EXPECTED(INVALID_FD != driver_buff_handle.shm_fd, HAILO_INTERNAL_FAILURE, - "Error creating shm object, errno is: {}", errno); - - // backs the shared memory object with physical memory - int err = shm_ctl(driver_buff_handle.shm_fd, SHMCTL_ANON | SHMCTL_TYMEM, (uint64_t)type_mem_fd, - required_size); - if (-1 == err) { - LOGGER__ERROR("Error backing shm object in physical memory, errno is: {}", errno); - close(driver_buff_handle.shm_fd); - return make_unexpected(HAILO_INTERNAL_FAILURE); - } - - // Create shared memory handle to send to driver - err = shm_create_handle(driver_buff_handle.shm_fd, driver.resource_manager_pid(), O_RDWR, - &driver_buff_handle.shm_handle, 0); - if (0 != err) { - LOGGER__ERROR("Error creating shm object handle, errno is: {}", errno); - close(driver_buff_handle.shm_fd); - return make_unexpected(HAILO_INTERNAL_FAILURE); - } - - void *address = mmap(0, required_size, PROT_WRITE | PROT_READ | PROT_NOCACHE, MAP_SHARED, driver_buff_handle.shm_fd, 0); - if (MAP_FAILED == address) { - LOGGER__ERROR("Failed to mmap buffer with errno:{}", errno); - shm_delete_handle(driver_buff_handle.shm_handle); - close(driver_buff_handle.shm_fd); - return make_unexpected(HAILO_OUT_OF_HOST_MEMORY); - } - - return VdmaMappedBufferImpl(address, required_size, driver_buff_handle.shm_handle, driver_buff_handle.shm_fd, driver); -} - -VdmaMappedBufferImpl::~VdmaMappedBufferImpl() -{ - if (nullptr != m_address) { - if (0 != munmap(m_address, m_length)) { - LOGGER__ERROR("Error unmapping memory at address {}, Errno: {}", m_address, errno); - } - - if (INVALID_FD != m_driver_mapped_buffer_identifier.shm_fd) { - if (0 != close(m_driver_mapped_buffer_identifier.shm_fd)) { - LOGGER__ERROR("Error closing shared memory fd, Errno: {}", errno); - } - } - } -} - -#else -#error "unsupported platform!" -#endif // defined(__linux__) || defined(_MSC_VER) - -} /* namespace vdma */ -} /* namespace hailort */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/vdma_mapped_buffer_impl.hpp b/hailort/libhailort/src/vdma/vdma_mapped_buffer_impl.hpp deleted file mode 100644 index 28186c3..0000000 --- a/hailort/libhailort/src/vdma/vdma_mapped_buffer_impl.hpp +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ - -#ifndef _HAILO_VDMA_MAPPED_BUFFER_IMPL_HPP_ -#define _HAILO_VDMA_MAPPED_BUFFER_IMPL_HPP_ - -#include "os/mmap_buffer.hpp" -#include "os/hailort_driver.hpp" -#include "hailo/expected.hpp" - -namespace hailort { -namespace vdma { - -#if defined(__linux__) || defined(_MSC_VER) - -class VdmaMappedBufferImpl final { -public: - VdmaMappedBufferImpl(HailoRTDriver &driver) : m_mapped_buffer(), - m_driver_mapped_buffer_identifier(HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE), m_driver(driver) {} - - ~VdmaMappedBufferImpl(); - - VdmaMappedBufferImpl(VdmaMappedBufferImpl &&other) noexcept : - m_mapped_buffer(std::move(other.m_mapped_buffer)), - m_driver_mapped_buffer_identifier(std::exchange(other.m_driver_mapped_buffer_identifier, HailoRTDriver::INVALID_DRIVER_BUFFER_HANDLE_VALUE)), - m_driver(other.m_driver) - {} - - VdmaMappedBufferImpl(const VdmaMappedBufferImpl &other) = delete; - VdmaMappedBufferImpl &operator=(const VdmaMappedBufferImpl &other) = delete; - VdmaMappedBufferImpl &operator=(VdmaMappedBufferImpl &&other) = delete; - - void* get() { return m_mapped_buffer.get(); } - - vdma_mapped_buffer_driver_identifier& get_mapped_buffer_identifier() { return m_driver_mapped_buffer_identifier; } - - explicit operator bool() - { - if (m_mapped_buffer) - return true; - return false; - } - - static Expected allocate_vdma_buffer(HailoRTDriver &driver, size_t required_size); - -private: - VdmaMappedBufferImpl(MmapBuffer&& mapped_buffer, vdma_mapped_buffer_driver_identifier driver_handle, HailoRTDriver &driver) : - m_mapped_buffer(std::move(mapped_buffer)), m_driver_mapped_buffer_identifier(driver_handle), m_driver(driver) {} - - MmapBuffer m_mapped_buffer; - vdma_mapped_buffer_driver_identifier m_driver_mapped_buffer_identifier; - HailoRTDriver &m_driver; -}; - -#elif defined(__QNX__) - -class VdmaMappedBufferImpl final { -public: - VdmaMappedBufferImpl(HailoRTDriver &driver): m_address(nullptr), m_length(0), m_driver(driver) { - m_driver_mapped_buffer_identifier.shm_handle = INVALID_HANDLE; - m_driver_mapped_buffer_identifier.shm_fd = INVALID_FD; - } - - ~VdmaMappedBufferImpl(); - - VdmaMappedBufferImpl(VdmaMappedBufferImpl &&other) noexcept : m_address(std::exchange(other.m_address, nullptr)), - m_length(std::exchange(other.m_length, 0)), m_driver(other.m_driver) - { - m_driver_mapped_buffer_identifier.shm_handle = std::exchange(other.m_driver_mapped_buffer_identifier.shm_handle, INVALID_HANDLE); - m_driver_mapped_buffer_identifier.shm_fd = std::exchange(other.m_driver_mapped_buffer_identifier.shm_fd, INVALID_FD); - - } - - VdmaMappedBufferImpl(const VdmaMappedBufferImpl &other) = delete; - VdmaMappedBufferImpl &operator=(const VdmaMappedBufferImpl &other) = delete; - VdmaMappedBufferImpl &operator=(VdmaMappedBufferImpl &&other) = delete; - - void* get() { return m_address; } - - vdma_mapped_buffer_driver_identifier& get_mapped_buffer_identifier() { return m_driver_mapped_buffer_identifier; } - - explicit operator bool() - { - return (nullptr != m_address); - } - - static Expected allocate_vdma_buffer(HailoRTDriver &driver, size_t required_size); - -private: - VdmaMappedBufferImpl(void *addr, size_t length, shm_handle_t shm_handle, int shm_fd, HailoRTDriver &driver) : - m_address(addr), m_length(length), m_driver(driver) - { - m_driver_mapped_buffer_identifier.shm_handle = shm_handle; - m_driver_mapped_buffer_identifier.shm_fd = shm_fd; - } - - static const int INVALID_FD; - static const shm_handle_t INVALID_HANDLE; - static const char* VDMA_BUFFER_TYPE_MEMORY_NAME; - - void *m_address; - size_t m_length; - vdma_mapped_buffer_driver_identifier m_driver_mapped_buffer_identifier; - HailoRTDriver &m_driver; -}; - -#else -#error "unsupported platform!" -#endif // defined(__linux__) || defined(_MSC_VER) - -} /* namespace vdma */ -} /* namespace hailort */ - -#endif /* _HAILO_VDMA_MAPPED_BUFFER_IMPL_HPP_ */ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma/vdma_stream.cpp b/hailort/libhailort/src/vdma/vdma_stream.cpp new file mode 100644 index 0000000..59ab85b --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_stream.cpp @@ -0,0 +1,134 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_stream.cpp + **/ + +#include "hailo/hailort_common.hpp" + +#include "vdma/vdma_stream.hpp" + + +namespace hailort +{ + +VdmaInputStream::VdmaInputStream(VdmaDevice &device, vdma::BoundaryChannelPtr channel, + const LayerInfo &edge_layer, EventPtr core_op_activated_event, + uint16_t batch_size, std::chrono::milliseconds transfer_timeout, + hailo_stream_interface_t stream_interface, hailo_status &status) : + VdmaInputStreamBase(device, channel, edge_layer, core_op_activated_event, batch_size, transfer_timeout, stream_interface, status), + m_write_only_mutex(), + m_send_pending_mutex() +{ + // Checking status for base class c'tor + if (HAILO_SUCCESS != status) { + return; + } + + status = HAILO_SUCCESS; +} + +Expected VdmaInputStream::sync_write_raw_buffer(const MemoryView &buffer) +{ + hailo_status status = HAILO_UNINITIALIZED; + + status = m_channel->wait(buffer.size(), m_channel_timeout); + if ((status == HAILO_STREAM_ABORTED_BY_USER) || (status == HAILO_STREAM_NOT_ACTIVATED)) { + return make_unexpected(status); + } + CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_TIMEOUT, + "{} (H2D) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_channel_timeout.count()); + CHECK_SUCCESS_AS_EXPECTED(status); + + status = m_channel->transfer((void*)buffer.data(), buffer.size()); + if ((status == HAILO_STREAM_ABORTED_BY_USER) || (status == HAILO_STREAM_NOT_ACTIVATED)) { + return make_unexpected(status); + } + CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_TIMEOUT, + "{} (H2D) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_channel_timeout.count()); + CHECK_SUCCESS_AS_EXPECTED(status); + + return buffer.size(); +} + +hailo_status VdmaInputStream::write_buffer_only(const MemoryView &buffer, + const std::function &should_cancel) +{ + std::unique_lock lock(m_write_only_mutex); + return m_channel->write_buffer(buffer, m_channel_timeout, should_cancel); +} + +hailo_status VdmaInputStream::send_pending_buffer(size_t device_index) +{ + std::unique_lock lock(m_send_pending_mutex); + CHECK(0 == device_index, HAILO_INVALID_OPERATION); + hailo_status status = m_channel->wait(get_frame_size(), m_channel_timeout); + if ((HAILO_STREAM_ABORTED_BY_USER == status) || (HAILO_STREAM_NOT_ACTIVATED == status)) { + return status; + } + CHECK(HAILO_TIMEOUT != status, HAILO_TIMEOUT, + "{} (H2D) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_channel_timeout.count()); + CHECK_SUCCESS(status); + + return m_channel->send_pending_buffer(); +} + +hailo_status VdmaInputStream::sync_write_all_raw_buffer_no_transform_impl(void *buffer, size_t offset, size_t size) +{ + ASSERT(NULL != buffer); + + return sync_write_raw_buffer(MemoryView(static_cast(buffer) + offset, size)).status(); +} + +/** Output stream **/ + +VdmaOutputStream::VdmaOutputStream(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t interface, + hailo_status &status) : + VdmaOutputStreamBase(device, channel, edge_layer, core_op_activated_event, batch_size, transfer_timeout, interface, status), + m_read_mutex() +{ + // Check status for base class c'tor + if (HAILO_SUCCESS != status) { + return; + } + + status = HAILO_SUCCESS; +} + +Expected VdmaOutputStream::sync_read_raw_buffer(MemoryView &buffer) +{ + hailo_status status = HAILO_UNINITIALIZED; + + status = m_channel->wait(buffer.size(), m_transfer_timeout); + if ((status == HAILO_STREAM_ABORTED_BY_USER) || (status == HAILO_STREAM_NOT_ACTIVATED)) { + return make_unexpected(status); + } + CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_TIMEOUT, + "{} (D2H) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_transfer_timeout.count()); + CHECK_SUCCESS_AS_EXPECTED(status); + + status = m_channel->transfer(buffer.data(), buffer.size()); + if ((status == HAILO_STREAM_NOT_ACTIVATED) || (status == HAILO_STREAM_ABORTED_BY_USER)) { + return make_unexpected(status); + } + CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_TIMEOUT, + "{} (D2H) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_transfer_timeout.count()); + CHECK_SUCCESS_AS_EXPECTED(status); + + return buffer.size(); +} + +hailo_status VdmaOutputStream::read_all(MemoryView &buffer) +{ + std::unique_lock lock(m_read_mutex); + CHECK((buffer.size() % HailoRTCommon::HW_DATA_ALIGNMENT) == 0, HAILO_INVALID_ARGUMENT, + "Size must be aligned to {} (got {})", HailoRTCommon::HW_DATA_ALIGNMENT, buffer.size()); + + return sync_read_raw_buffer(buffer).status(); +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_stream.hpp b/hailort/libhailort/src/vdma/vdma_stream.hpp new file mode 100644 index 0000000..ec6f41e --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_stream.hpp @@ -0,0 +1,74 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_stream.hpp + * @brief Stream object over vDMA channel + **/ + +#ifndef _HAILO_VDMA_STREAM_HPP_ +#define _HAILO_VDMA_STREAM_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "stream_common/stream_internal.hpp" +#include "vdma/vdma_stream_base.hpp" +#include "vdma/vdma_device.hpp" +#include "vdma/channel/boundary_channel.hpp" + + +namespace hailort +{ + +class VdmaInputStream : public VdmaInputStreamBase +{ +public: + VdmaInputStream(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t stream_interface, + hailo_status &status); + virtual ~VdmaInputStream() = default; + + hailo_status write_buffer_only(const MemoryView &buffer, const std::function &should_cancel = []() { return false; }); + hailo_status send_pending_buffer(size_t device_index = 0); + + void notify_all() + { + return m_channel->notify_all(); + } + +private: + virtual Expected sync_write_raw_buffer(const MemoryView &buffer) override; + virtual hailo_status sync_write_all_raw_buffer_no_transform_impl(void *buffer, size_t offset, size_t size) override; + + std::mutex m_write_only_mutex; + std::mutex m_send_pending_mutex; + + friend class InputVDeviceBaseStream; + friend class InputVDeviceNativeStream; +}; + +class VdmaOutputStream : public VdmaOutputStreamBase +{ +public: + VdmaOutputStream(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t interface, + hailo_status &status); + virtual ~VdmaOutputStream() = default; + +private: + virtual Expected sync_read_raw_buffer(MemoryView &buffer); + virtual hailo_status read_all(MemoryView &buffer) override; + + std::mutex m_read_mutex; + + friend class OutputVDeviceBaseStream; +}; + + +} /* namespace hailort */ + +#endif /* _HAILO_VDMA_STREAM_HPP_ */ diff --git a/hailort/libhailort/src/vdma/vdma_stream_base.cpp b/hailort/libhailort/src/vdma/vdma_stream_base.cpp new file mode 100644 index 0000000..542c438 --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_stream_base.cpp @@ -0,0 +1,443 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_stream_base.cpp + **/ + +#include "hailo/hailort_common.hpp" + +#include "vdma/vdma_stream_base.hpp" +#include "vdma/vdma_stream.hpp" +#include "vdma/vdma_async_stream.hpp" + + +namespace hailort +{ + +static bool validate_device_interface_compatibility(hailo_stream_interface_t interface, Device::Type type) +{ + bool interface_valid = false; + switch (type) + { + case Device::Type::PCIE: + interface_valid = (HAILO_STREAM_INTERFACE_PCIE == interface); + break; + + case Device::Type::INTEGRATED: + interface_valid = (HAILO_STREAM_INTERFACE_INTEGRATED == interface); + break; + + default: + LOGGER__ERROR("Invalid device type {}", type); + return false; + } + + if (interface_valid) { + return true; + } + + LOGGER__ERROR("Invalid interface {} for device of type {}", interface, type); + return false; +} + +Expected> VdmaInputStreamBase::create(hailo_stream_interface_t interface, + VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + const hailo_stream_parameters_t &stream_params, uint16_t batch_size, EventPtr core_op_activated_event) +{ + CHECK_AS_EXPECTED(validate_device_interface_compatibility(interface, device.get_type()), HAILO_INTERNAL_FAILURE); + + if ((stream_params.flags & HAILO_STREAM_FLAGS_ASYNC) != 0) { + CHECK_AS_EXPECTED(channel->type() == vdma::BoundaryChannel::Type::ASYNC, HAILO_INVALID_ARGUMENT, + "Can't create a async vdma stream with a non async channel. Received channel type {}", channel->type()); + + hailo_status status = HAILO_UNINITIALIZED; + auto result = make_shared_nothrow(device, channel, edge_layer, core_op_activated_event, + batch_size, DEFAULT_TRANSFER_TIMEOUT, interface, status); + CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + + return std::static_pointer_cast(result); + } else { + CHECK_AS_EXPECTED(channel->type() == vdma::BoundaryChannel::Type::BUFFERED, HAILO_INVALID_ARGUMENT, + "Can't create a vdma stream with a non buffered channel. Received channel type {}", channel->type()); + + hailo_status status = HAILO_UNINITIALIZED; + auto result = make_shared_nothrow(device, channel, edge_layer, core_op_activated_event, + batch_size, DEFAULT_TRANSFER_TIMEOUT, interface, status); + CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + + return std::static_pointer_cast(result); + } +} + +VdmaInputStreamBase::VdmaInputStreamBase(VdmaDevice &device, vdma::BoundaryChannelPtr channel, + const LayerInfo &edge_layer, EventPtr core_op_activated_event, + uint16_t batch_size, std::chrono::milliseconds transfer_timeout, + hailo_stream_interface_t stream_interface, hailo_status &status) : + InputStreamBase(edge_layer, stream_interface, std::move(core_op_activated_event), status), + m_device(&device), + m_channel(std::move(channel)), + m_interface(stream_interface), + is_stream_activated(false), + m_channel_timeout(transfer_timeout), + m_max_batch_size(batch_size), + m_dynamic_batch_size(batch_size) +{ + // Checking status for base class c'tor + if (HAILO_SUCCESS != status) { + return; + } + + status = HAILO_SUCCESS; +} + +VdmaInputStreamBase::~VdmaInputStreamBase() +{ + // We want to stop the vdma channel before closing the stream in the firmware + // because sending data to a closed stream may terminate the dma engine + if (this->is_stream_activated) { + const auto status = VdmaInputStreamBase::deactivate_stream(); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to deactivate stream with error status {}", status); + } + } +} + +hailo_stream_interface_t VdmaInputStreamBase::get_interface() const +{ + return m_interface; +} + +std::chrono::milliseconds VdmaInputStreamBase::get_timeout() const +{ + return this->m_channel_timeout; +} + +hailo_status VdmaInputStreamBase::set_timeout(std::chrono::milliseconds timeout) +{ + this->m_channel_timeout = timeout; + return HAILO_SUCCESS; +} + +hailo_status VdmaInputStreamBase::abort() +{ + return m_channel->abort(); +} + +hailo_status VdmaInputStreamBase::clear_abort() +{ + return m_channel->clear_abort(); +} + +hailo_status VdmaInputStreamBase::flush() +{ + const auto dynamic_batch_size = (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == m_dynamic_batch_size) ? + 1 : m_dynamic_batch_size; + return m_channel->flush(m_channel_timeout * dynamic_batch_size); +} + +hailo_status VdmaInputStreamBase::activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) +{ + auto status = set_dynamic_batch_size(dynamic_batch_size); + CHECK_SUCCESS(status); + + status = m_channel->activate(0, resume_pending_stream_transfers); + CHECK_SUCCESS(status); + + this->is_stream_activated = true; + + return HAILO_SUCCESS; +} + +hailo_status VdmaInputStreamBase::deactivate_stream() +{ + if (!is_stream_activated) { + return HAILO_SUCCESS; + } + + // Flush is best effort + auto status = m_channel->flush(VDMA_FLUSH_TIMEOUT); + if (HAILO_STREAM_ABORTED_BY_USER == status) { + LOGGER__INFO("Flush input_channel is not needed because channel was aborted. (channel {})", m_channel->get_channel_id()); + status = HAILO_SUCCESS; + } else if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to flush input_channel. (status {} channel {})", status, m_channel->get_channel_id()); + } + + status = m_channel->deactivate(); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to stop channel with status {}", status); + } + + this->is_stream_activated = false; + return status; +} + +uint16_t VdmaInputStreamBase::get_dynamic_batch_size() const +{ + return std::max(m_dynamic_batch_size, static_cast(1)); +} + +const char* VdmaInputStreamBase::get_dev_id() const +{ + return m_device->get_dev_id(); +} + +Expected VdmaInputStreamBase::get_buffer_state() +{ + return m_channel->get_buffer_state(); +} + +Expected VdmaInputStreamBase::get_buffer_frames_size() const +{ + return m_channel->get_transfers_count_in_buffer(m_stream_info.hw_frame_size); +} + +Expected VdmaInputStreamBase::get_pending_frames_count() const +{ + return m_channel->get_h2d_pending_frames_count(); +} + +hailo_status VdmaInputStreamBase::register_interrupt_callback(const vdma::ProcessingCompleteCallback &callback) +{ + return m_channel->register_interrupt_callback(callback); +} + +hailo_status VdmaInputStreamBase::set_dynamic_batch_size(uint16_t dynamic_batch_size) +{ + // TODO: use std::max in the configure stage + if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == m_max_batch_size) { + LOGGER__TRACE("max_batch_size is CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE; " + "Ignoring value of dynamic_batch_size {}", m_dynamic_batch_size); + return HAILO_SUCCESS; + } + + CHECK(dynamic_batch_size <= m_max_batch_size, HAILO_INVALID_ARGUMENT, + "Dynamic batch size ({}) must be <= than the configured batch size ({})", + dynamic_batch_size, m_max_batch_size); + + if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == dynamic_batch_size) { + LOGGER__TRACE("Received CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == dynamic_batch_size; " + "Leaving previously set value of {}", m_dynamic_batch_size); + } else { + LOGGER__TRACE("Setting stream's dynamic_batch_size to {}", dynamic_batch_size); + m_dynamic_batch_size = dynamic_batch_size; + + const auto status = m_channel->set_transfers_per_axi_intr(m_dynamic_batch_size); + CHECK_SUCCESS(status); + } + + return HAILO_SUCCESS; +} + +/** Output stream **/ +Expected> VdmaOutputStreamBase::create(hailo_stream_interface_t interface, + VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, uint16_t batch_size, + const hailo_stream_parameters_t &stream_params, EventPtr core_op_activated_event) +{ + CHECK_AS_EXPECTED(validate_device_interface_compatibility(interface, device.get_type()), HAILO_INTERNAL_FAILURE); + + if ((stream_params.flags & HAILO_STREAM_FLAGS_ASYNC) != 0) { + CHECK_AS_EXPECTED(channel->type() == vdma::BoundaryChannel::Type::ASYNC, HAILO_INVALID_ARGUMENT, + "Can't create a async vdma stream with a non async channel. Received channel type {}", channel->type()); + + hailo_status status = HAILO_UNINITIALIZED; + auto result = make_shared_nothrow(device, channel, edge_layer, core_op_activated_event, + batch_size, DEFAULT_TRANSFER_TIMEOUT, interface, status); + CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + + return std::static_pointer_cast(result); + } else { + CHECK_AS_EXPECTED(channel->type() == vdma::BoundaryChannel::Type::BUFFERED, HAILO_INVALID_ARGUMENT, + "Can't create a vdma stream with a non buffered channel. Received channel type {}", channel->type()); + + hailo_status status = HAILO_UNINITIALIZED; + auto result = make_shared_nothrow(device, channel, edge_layer, core_op_activated_event, + batch_size, DEFAULT_TRANSFER_TIMEOUT, interface, status); + CHECK_SUCCESS_AS_EXPECTED(status); + CHECK_NOT_NULL_AS_EXPECTED(result, HAILO_OUT_OF_HOST_MEMORY); + + return std::static_pointer_cast(result); + } +} + +VdmaOutputStreamBase::VdmaOutputStreamBase(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t interface, + hailo_status &status) : + OutputStreamBase(edge_layer, std::move(core_op_activated_event), status), + m_device(&device), + m_channel(std::move(channel)), + m_interface(interface), + is_stream_activated(false), + m_transfer_timeout(transfer_timeout), + m_max_batch_size(batch_size), + m_dynamic_batch_size(batch_size), + m_transfer_size(get_transfer_size(m_stream_info)) +{ + // Check status for base class c'tor + if (HAILO_SUCCESS != status) { + return; + } + + status = HAILO_SUCCESS; +} + +VdmaOutputStreamBase::~VdmaOutputStreamBase() +{ + // We want to stop the vdma channel before closing the stream in the firmware + // because sending data to a closed stream may terminate the dma engine + if (this->is_stream_activated) { + const auto status = VdmaOutputStreamBase::deactivate_stream(); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to deactivate stream with error status {}", status); + } + } +} + +hailo_stream_interface_t VdmaOutputStreamBase::get_interface() const +{ + return m_interface; +} + +hailo_status VdmaOutputStreamBase::set_timeout(std::chrono::milliseconds timeout) +{ + this->m_transfer_timeout = timeout; + return HAILO_SUCCESS; +} + +std::chrono::milliseconds VdmaOutputStreamBase::get_timeout() const +{ + return this->m_transfer_timeout; +} + +hailo_status VdmaOutputStreamBase::abort() +{ + return m_channel->abort(); +} + +hailo_status VdmaOutputStreamBase::clear_abort() +{ + return m_channel->clear_abort(); +} + +uint16_t VdmaOutputStreamBase::get_dynamic_batch_size() const +{ + return std::max(m_dynamic_batch_size, static_cast(1)); +} + +const char* VdmaOutputStreamBase::get_dev_id() const +{ + return m_device->get_dev_id(); +} + +Expected VdmaOutputStreamBase::get_buffer_state() +{ + return m_channel->get_buffer_state(); +} + +hailo_status VdmaOutputStreamBase::activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) +{ + auto status = set_dynamic_batch_size(dynamic_batch_size); + CHECK_SUCCESS(status); + + status = m_channel->activate(m_transfer_size, resume_pending_stream_transfers); + CHECK_SUCCESS(status); + + this->is_stream_activated = true; + + return HAILO_SUCCESS; +} + +hailo_status VdmaOutputStreamBase::register_interrupt_callback(const vdma::ProcessingCompleteCallback &callback) +{ + return m_channel->register_interrupt_callback(callback); +} + +hailo_status VdmaOutputStreamBase::deactivate_stream() +{ + if (!is_stream_activated) { + return HAILO_SUCCESS; + } + + auto status = m_channel->deactivate(); + if (HAILO_SUCCESS != status) { + LOGGER__ERROR("Failed to stop channel with status {}", status); + } + + this->is_stream_activated = false; + return HAILO_SUCCESS; +} + +uint32_t VdmaOutputStreamBase::get_transfer_size(const hailo_stream_info_t &stream_info) +{ + // The ppu outputs one bbox per vdma buffer in the case of nms + return (HAILO_FORMAT_ORDER_HAILO_NMS == stream_info.format.order) ? + stream_info.nms_info.bbox_size : stream_info.hw_frame_size; +} + +hailo_status VdmaOutputStreamBase::set_dynamic_batch_size(uint16_t dynamic_batch_size) +{ + // TODO: use std::max in the configure stage + if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == m_max_batch_size) { + LOGGER__TRACE("max_batch_size is CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE; " + "Ignoring value of dynamic_batch_size {}", m_dynamic_batch_size); + return HAILO_SUCCESS; + } + + CHECK(dynamic_batch_size <= m_max_batch_size, HAILO_INVALID_ARGUMENT, + "Dynamic batch size ({}) must be <= than the configured batch size ({})", + dynamic_batch_size, m_max_batch_size); + + if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == dynamic_batch_size) { + LOGGER__TRACE("Received CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == dynamic_batch_size; " + "Leaving previously set value of {}", m_dynamic_batch_size); + } else { + LOGGER__TRACE("Setting stream's dynamic_batch_size to {}", dynamic_batch_size); + m_dynamic_batch_size = dynamic_batch_size; + + const auto status = m_channel->set_transfers_per_axi_intr(m_dynamic_batch_size); + CHECK_SUCCESS(status); + } + + return HAILO_SUCCESS; +} + +Expected VdmaOutputStreamBase::get_buffer_frames_size() const +{ + if (HAILO_FORMAT_ORDER_HAILO_NMS == m_stream_info.format.order) { + // In NMS, each output frame has different size depending on the number of bboxes found for each class + // and m_stream_info.hw_frame_size is the max frame size. To know the actual frame size and + // calculate the number of frames we need to read the content of the buffer (and finding the delimiter for each class in each frame). + LOGGER__INFO("NMS is not supported in function get_buffer_frames_size()"); + return make_unexpected(HAILO_NOT_AVAILABLE); + } + + return m_channel->get_transfers_count_in_buffer(m_stream_info.hw_frame_size); +} + +Expected VdmaOutputStreamBase::get_pending_frames_count() const +{ + if (HAILO_FORMAT_ORDER_HAILO_NMS == m_stream_info.format.order) { + // In NMS, each output frame has different size depending on the number of bboxes found for each class + // and m_stream_info.hw_frame_size is the max frame size. To know the actual frame size and + // calculate the number of frames we need to read the content of the buffer (and finding the delimiter for each class in each frame). + LOGGER__INFO("NMS is not supported in function get_pending_frames_count()"); + return make_unexpected(HAILO_NOT_AVAILABLE); + } + + auto pending_descs_count = m_channel->get_d2h_pending_descs_count(); + CHECK_EXPECTED(pending_descs_count); + + auto channel_page_size = m_channel->get_page_size(); + uint32_t descs_per_frame = (0 == (m_stream_info.hw_frame_size % channel_page_size)) ? (m_stream_info.hw_frame_size / channel_page_size) : + ((m_stream_info.hw_frame_size / channel_page_size) + 1); + + return static_cast(std::floor(pending_descs_count.value() / descs_per_frame)); +} + +} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma/vdma_stream_base.hpp b/hailort/libhailort/src/vdma/vdma_stream_base.hpp new file mode 100644 index 0000000..a7cde98 --- /dev/null +++ b/hailort/libhailort/src/vdma/vdma_stream_base.hpp @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the MIT license (https://opensource.org/licenses/MIT) + **/ +/** + * @file vdma_stream_base.hpp + * @brief Base class for stream objects over vDMA channel + **/ + +#ifndef _HAILO_VDMA_STREAM_BASE_HPP_ +#define _HAILO_VDMA_STREAM_BASE_HPP_ + +#include "hailo/hailort.h" +#include "hailo/expected.hpp" + +#include "stream_common/stream_internal.hpp" +#include "vdma/vdma_device.hpp" +#include "vdma/channel/boundary_channel.hpp" + + +namespace hailort +{ +constexpr std::chrono::seconds VDMA_FLUSH_TIMEOUT(10); + +class VdmaInputStreamBase : public InputStreamBase { +public: + static Expected> create(hailo_stream_interface_t interface, + VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + const hailo_stream_parameters_t &stream_params, uint16_t batch_size, EventPtr core_op_activated_event); + + virtual ~VdmaInputStreamBase(); + + virtual hailo_stream_interface_t get_interface() const override; + virtual std::chrono::milliseconds get_timeout() const override; + virtual hailo_status set_timeout(std::chrono::milliseconds timeout) override; + virtual hailo_status abort() override; + virtual hailo_status clear_abort() override; + virtual hailo_status flush() override; + uint16_t get_dynamic_batch_size() const; + const char* get_dev_id() const; + Expected get_buffer_state(); + virtual Expected get_buffer_frames_size() const override; + virtual Expected get_pending_frames_count() const override; + virtual hailo_status register_interrupt_callback(const vdma::ProcessingCompleteCallback &callback) override; + +protected: + VdmaInputStreamBase(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t stream_interface, + hailo_status &status); + + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; + virtual hailo_status deactivate_stream() override; + hailo_status set_dynamic_batch_size(uint16_t dynamic_batch_size); + + VdmaDevice *m_device; + vdma::BoundaryChannelPtr m_channel; + const hailo_stream_interface_t m_interface; + bool is_stream_activated; + std::chrono::milliseconds m_channel_timeout; + const uint16_t m_max_batch_size; + uint16_t m_dynamic_batch_size; +}; + +class VdmaOutputStreamBase : public OutputStreamBase { +public: + static Expected> create(hailo_stream_interface_t interface, + VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, uint16_t batch_size, + const hailo_stream_parameters_t &stream_params, EventPtr core_op_activated_event); + + virtual ~VdmaOutputStreamBase(); + + virtual hailo_stream_interface_t get_interface() const override; + virtual std::chrono::milliseconds get_timeout() const override; + virtual hailo_status set_timeout(std::chrono::milliseconds timeout) override; + virtual hailo_status abort() override; + virtual hailo_status clear_abort() override; + uint16_t get_dynamic_batch_size() const; + const char* get_dev_id() const; + Expected get_buffer_state(); + virtual Expected get_buffer_frames_size() const override; + virtual Expected get_pending_frames_count() const override; + + virtual hailo_status register_interrupt_callback(const vdma::ProcessingCompleteCallback &callback); + +protected: + VdmaOutputStreamBase(VdmaDevice &device, vdma::BoundaryChannelPtr channel, const LayerInfo &edge_layer, + EventPtr core_op_activated_event, uint16_t batch_size, + std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t interface, + hailo_status &status); + + virtual hailo_status activate_stream(uint16_t dynamic_batch_size, bool resume_pending_stream_transfers) override; + virtual hailo_status deactivate_stream() override; + static uint32_t get_transfer_size(const hailo_stream_info_t &stream_info); + hailo_status set_dynamic_batch_size(uint16_t dynamic_batch_size); + + VdmaDevice *m_device; + vdma::BoundaryChannelPtr m_channel; + const hailo_stream_interface_t m_interface; + bool is_stream_activated; + std::chrono::milliseconds m_transfer_timeout; + const uint16_t m_max_batch_size; + uint16_t m_dynamic_batch_size; + const uint32_t m_transfer_size; + std::mutex m_read_mutex; +}; + + +} /* namespace hailort */ + +#endif /* _HAILO_VDMA_STREAM_BASE_HPP_ */ diff --git a/hailort/libhailort/src/vdma_channel.cpp b/hailort/libhailort/src/vdma_channel.cpp deleted file mode 100644 index 211afbb..0000000 --- a/hailort/libhailort/src/vdma_channel.cpp +++ /dev/null @@ -1,1283 +0,0 @@ -#include "vdma_channel.hpp" -#include "vdma_channel_regs.hpp" -#include "hw_consts.hpp" -#include "common/logger_macros.hpp" -#include "common/utils.hpp" -#include "vdma/sg_buffer.hpp" -#include "vdma_descriptor_list.hpp" - -#include "hailo/hailort_common.hpp" - -#include -#include -#include - -#include - -namespace hailort -{ - -#define FD_READ_SIZE (8) -#define MIN_TIMEOUT_DDR (1000) - -/* PLDA descriptor control */ -#define PCIE_DESCRIPTOR_CONTROL_CLR(src)\ - src = (src & (~(uint32_t)0xFF)) -#define PCIE_DESCRIPTOR_CONTROL_SET_DESC_STATUS_REQ(src)\ - src = ((src) | 0x01) -#define PCIE_DESCRIPTOR_CONTROL_SET_DESC_STATUS_REQ_ERR(src)\ - src = ((src) | 0x02) -#define PCIE_DESCRIPTOR_CONTROL_SET_DESC_SET_IRQ_ON_ERROR(src)\ - src = ((src) | 0x04) -#define PCIE_DESCRIPTOR_CONTROL_SET_DESC_SET_IRQ_ON_AXI_DOMAIN(src)\ - src = ((src) | 0x10) - - -void VdmaChannel::State::lock() -{ -#ifndef _MSC_VER - int err = pthread_mutex_lock(&m_state_lock); - if (0 != err) { - LOGGER__ERROR("Failed destory vdma channel mutex, errno {}", err); - assert(false); - } -#else - EnterCriticalSection(&m_state_lock); -#endif -} - -void VdmaChannel::State::unlock() -{ -#ifndef _MSC_VER - int err = pthread_mutex_unlock(&m_state_lock); - assert(0 == err); - (void)err; -#else - LeaveCriticalSection(&m_state_lock); -#endif -} - -Expected VdmaChannel::create(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, - uint16_t requested_desc_page_size, const std::string &stream_name, LatencyMeterPtr latency_meter, uint16_t transfers_per_axi_intr) -{ - CHECK_AS_EXPECTED(Direction::BOTH != direction, HAILO_INVALID_ARGUMENT); - - hailo_status status = HAILO_UNINITIALIZED; - auto desc_page_size_value = driver.calc_desc_page_size(requested_desc_page_size); - CHECK_AS_EXPECTED(is_powerof2(desc_page_size_value), HAILO_INVALID_ARGUMENT, - "Descriptor page_size must be a power of two."); - CHECK_AS_EXPECTED(channel_id.channel_index < VDMA_CHANNELS_PER_ENGINE, HAILO_INVALID_ARGUMENT, - "Invalid DMA channel index {}", channel_id.channel_index); - CHECK_AS_EXPECTED(channel_id.engine_index < driver.dma_engines_count(), HAILO_INVALID_ARGUMENT, - "Invalid DMA engine index {}, max {}", channel_id.engine_index, driver.dma_engines_count()); - - VdmaChannel object(channel_id, direction, driver, stream_name, latency_meter, desc_page_size_value, - transfers_per_axi_intr, status); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed creating VdmaChannel"); - return make_unexpected(status); - } - return object; -} - -VdmaChannel::VdmaChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, - const std::string &stream_name, LatencyMeterPtr latency_meter, uint16_t desc_page_size, uint16_t transfers_per_axi_intr, - hailo_status &status) - : m_d2h_callback_thread(nullptr), m_channel_id(channel_id), - m_direction(direction), m_driver(driver), - m_host_registers(driver, channel_id, direction), - m_device_registers(driver, channel_id, other_direction(direction)), - m_desc_page_size(desc_page_size), - m_stream_name(stream_name), m_latency_meter(latency_meter), m_channel_enabled(false), - m_transfers_per_axi_intr(transfers_per_axi_intr), m_pending_buffers_sizes(0), m_pending_num_avail_offset(0), m_is_waiting_for_channel_completion(false), - m_is_aborted_by_internal_source(false) -{ - if (m_transfers_per_axi_intr == 0) { - LOGGER__ERROR("Invalid transfers per axi interrupt"); - status = HAILO_INVALID_ARGUMENT; - return; - } - - auto channel_handle_memory = MmapBuffer::create_shared_memory(sizeof(HailoRTDriver::VdmaChannelHandle)); - if (!channel_handle_memory) { - LOGGER__ERROR("Failed allocating shared memory for channel, err = {}", channel_handle_memory.status()); - status = channel_handle_memory.status(); - return; - } - m_channel_handle = channel_handle_memory.release(); - *m_channel_handle = HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE; - - // The fw activates the channel (in ResourcesManager::enable_state_machine) - // The driver cleans the channel's state, in case the last shutdown wasn't successful. - m_channel_enabled = true; - - status = HAILO_SUCCESS; -} - -VdmaChannel::~VdmaChannel() -{ - if (m_channel_enabled) { - stop_channel(); - m_channel_enabled = false; - if (Direction::H2D == m_direction) { - m_can_write_buffer_cv.notify_all(); - } else { - m_can_read_buffer_cv.notify_all(); - } - } - - if (m_state) { -#ifndef _MSC_VER - int err = pthread_mutex_destroy(&m_state->m_state_lock); - if (0 != err) { - LOGGER__ERROR("Failed destory vdma channel mutex, errno {}", err); - } -#else - DeleteCriticalSection(&m_state->m_state_lock); -#endif - } -} - -VdmaChannel::VdmaChannel(VdmaChannel &&other) noexcept: - m_d2h_callback_thread(std::move(other.m_d2h_callback_thread)), - m_channel_id(std::move(other.m_channel_id)), - m_direction(other.m_direction), - m_driver(other.m_driver), - m_host_registers(std::move(other.m_host_registers)), - m_device_registers(std::move(other.m_device_registers)), - m_desc_page_size(other.m_desc_page_size), - m_buffer(std::move(other.m_buffer)), - m_stream_name(std::move(other.m_stream_name)), - m_latency_meter(std::move(other.m_latency_meter)), - m_state(std::move(other.m_state)), - m_channel_handle(std::move(other.m_channel_handle)), - m_channel_enabled(std::exchange(other.m_channel_enabled, false)), - m_transfers_per_axi_intr(std::move(other.m_transfers_per_axi_intr)), - m_pending_buffers_sizes(std::move(other.m_pending_buffers_sizes)), - m_pending_num_avail_offset(other.m_pending_num_avail_offset.exchange(0)), - m_is_waiting_for_channel_completion(other.m_is_waiting_for_channel_completion.exchange(false)), - m_is_aborted_by_internal_source(other.m_is_aborted_by_internal_source.exchange(false)) -{} - -hailo_status VdmaChannel::stop_channel() -{ - if (!m_state) { - const auto status = unregister_fw_controlled_channel(); - CHECK_SUCCESS(status, "Failed to disable channel {}", m_channel_id); - - } else { - std::unique_lock state_guard(*m_state); - const auto status = unregister_fw_controlled_channel(); - CHECK_SUCCESS(status, "Failed to disable channel {}", m_channel_id); - - if (Direction::D2H == m_direction) { - unregister_for_d2h_interrupts(state_guard); - } else { - if (m_state->m_should_reprogram_buffer || !m_pending_buffers_sizes.empty()) { - // If we've already reprogrammed the buffer or there are pending buffers, we'll set m_previous_tail - const auto curr_tail = CB_TAIL(m_state->m_descs); - m_state->m_previous_tail = (curr_tail + m_state->m_previous_tail) & m_state->m_descs.size_mask; - m_state->m_should_reprogram_buffer = true; - } - // For H2D channels we reset counters as we want to allow writes to the start of the buffer while the channel is stopped - reset_internal_counters(); - } - - } - return HAILO_SUCCESS; -} - -uint16_t VdmaChannel::get_page_size() -{ - return m_desc_page_size; -} - -Expected VdmaChannel::get_boundary_buffer_info(uint32_t transfer_size) -{ - CHECK_AS_EXPECTED(m_buffer, HAILO_INVALID_OPERATION, "Cannot get host buffer before buffer is allocated"); - return m_buffer->get_host_buffer_info(transfer_size); -} - -hailo_status VdmaChannel::abort() -{ - { - std::lock_guard state_guard(*m_state); - m_is_aborted_by_internal_source = true; - } - - if (Direction::H2D == m_direction) { - m_can_write_buffer_cv.notify_all(); - } else { - m_can_read_buffer_cv.notify_all(); - } - return m_driver.vdma_channel_abort(m_channel_id, *m_channel_handle); -} - -hailo_status VdmaChannel::clear_abort() -{ - auto status = m_driver.vdma_channel_clear_abort(m_channel_id, *m_channel_handle); - { - std::lock_guard state_guard(*m_state); - m_is_aborted_by_internal_source = false; - } - return status; -} - -size_t VdmaChannel::get_transfers_count_in_buffer(size_t transfer_size) -{ - const auto descs_in_transfer = m_buffer->descriptors_in_buffer(transfer_size); - const auto descs_count = CB_SIZE(m_state->m_descs); - return (descs_count - 1) / descs_in_transfer; -} - -size_t VdmaChannel::get_buffer_size() const -{ - assert(m_buffer); - return m_buffer->size(); -} - -Expected VdmaChannel::get_h2d_pending_frames_count() -{ - return m_pending_buffers_sizes.size(); -} - -Expected VdmaChannel::get_d2h_pending_descs_count() -{ - assert(m_state); - - std::lock_guard state_guard(*m_state); - - int num_proc = CB_TAIL(m_state->m_descs); - int desc_num_ready = CB_PROG(m_state->m_descs, num_proc, m_state->m_d2h_read_desc_index); - - return desc_num_ready; -} - -hailo_status VdmaChannel::prepare_d2h_pending_descriptors(uint32_t transfer_size) -{ - assert(m_buffer); - - auto transfers_count_in_buffer = get_transfers_count_in_buffer(transfer_size); - auto transfers_count = std::min(transfers_count_in_buffer, - static_cast(CB_SIZE(m_state->m_buffers) - 1)); - - // on D2H no need for interrupt of first descriptor - const auto first_desc_interrupts_domain = VdmaInterruptsDomain::NONE; - for (uint32_t i = 0; i < transfers_count; i++) { - /* Provide FW interrupt only in the end of the last transfer in the batch */ - auto last_desc_interrutps_domain = - (static_cast(m_transfers_per_axi_intr - 1) == (i % m_transfers_per_axi_intr)) ? - VdmaInterruptsDomain::BOTH : VdmaInterruptsDomain::HOST; - auto status = prepare_descriptors(transfer_size, first_desc_interrupts_domain, last_desc_interrutps_domain); - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__INFO("preparing descriptors failed because channel is not activated"); - return status; - } - CHECK_SUCCESS(status, "Failed prepare desc status={}", status); - } - - /* We assume each output transfer is in the same size */ - m_state->m_accumulated_transfers += ((m_state->m_accumulated_transfers + transfers_count) % m_transfers_per_axi_intr); - - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::allocate_resources(uint32_t descs_count) -{ - // TODO (HRT-3762) : Move channel's state to driver to avoid using shared memory - auto state = MmapBuffer::create_shared_memory(sizeof(VdmaChannel::State)); - CHECK_EXPECTED_AS_STATUS(state, "Failed to allocate channel's resources"); - -#ifndef _MSC_VER - // Make sharable mutex - pthread_mutexattr_t mutex_attrs{}; - int err = pthread_mutexattr_init(&mutex_attrs); - CHECK(0 == err, HAILO_INTERNAL_FAILURE, "pthread_mutexattr_init failed with {}", err); - - err = pthread_mutexattr_setpshared(&mutex_attrs, PTHREAD_PROCESS_SHARED); - if (0 != err) { - (void)pthread_mutexattr_destroy(&mutex_attrs); - LOGGER__ERROR("pthread_mutexattr_setpshared failed with {}", err); - return HAILO_INTERNAL_FAILURE; - } - - err = pthread_mutex_init(&state.value()->m_state_lock, &mutex_attrs); - if (0 != pthread_mutexattr_destroy(&mutex_attrs)) { - LOGGER__ERROR("pthread_mutexattr_destroy failed"); - // continue - } - CHECK(0 == err, HAILO_INTERNAL_FAILURE, "Mutex init failed with {}", err); -#else - InitializeCriticalSection(&state.value()->m_state_lock); -#endif - - m_state = state.release(); - m_pending_buffers_sizes = CircularArray(descs_count); - - // If measuring latency, max_active_transfer is limited to 16 (see hailort_driver.hpp doc for further information) - int pending_buffers_size = (nullptr == m_latency_meter) ? static_cast(m_state->m_pending_buffers.size()) : - (static_cast(m_state->m_pending_buffers.size()) / 2); - - if (MAX_DESCS_COUNT < descs_count) { - LOGGER__ERROR("Vdma channel descs_count mustn't be larger than {}", MAX_DESCS_COUNT); - return HAILO_INVALID_ARGUMENT; - } - - CB_INIT(m_state->m_descs, descs_count); - CB_INIT(m_state->m_buffers, pending_buffers_size); - m_state->m_previous_tail = 0; - m_state->m_should_reprogram_buffer = false; - - // Allocate descriptor list (host side) - auto status = allocate_buffer(descs_count * m_desc_page_size); - CHECK_SUCCESS(status, "Failed to allocate vDMA buffer for channel transfer! status={}", status); - - clear_descriptor_list(); - - return HAILO_SUCCESS; -} - -void VdmaChannel::reset_internal_counters() -{ - assert(m_state); - CB_RESET(m_state->m_descs); - CB_RESET(m_state->m_buffers); - m_state->m_d2h_read_desc_index = 0; - m_state->m_last_timestamp_num_processed = 0; - m_state->m_accumulated_transfers = 0; -} - -hailo_status VdmaChannel::complete_channel_activation(uint32_t transfer_size) -{ - /* descriptor buffer must be allocated */ - assert(m_buffer); - assert(m_state); - std::lock_guard state_guard(*m_state); - - CHECK(HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE != *m_channel_handle, - HAILO_INTERNAL_FAILURE, "Vdma channel must be registered before activation"); - - reset_internal_counters(); - - if ((Direction::D2H == m_direction) && (transfer_size != 0)) { - auto status = prepare_d2h_pending_descriptors(transfer_size); - if (HAILO_SUCCESS != status) { - stop_channel(); - } - return status; - } - - // We should have no active transfers now - if (m_state->m_should_reprogram_buffer) { - auto status = m_buffer->reprogram_buffer_offset(m_state->m_previous_tail * m_desc_page_size, m_channel_id.channel_index); - CHECK_SUCCESS(status); - } - - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::register_fw_controlled_channel() -{ - return register_channel_to_driver(); -} - -void VdmaChannel::notify_all() -{ - { - // Acquire mutex to make sure the notify_all will wake the blocking threads on the cv - std::lock_guard state_guard(*m_state); - } - m_can_write_buffer_cv.notify_all(); - m_can_read_buffer_cv.notify_all(); -} - -hailo_status VdmaChannel::register_for_d2h_interrupts(const std::function &callback) -{ - // This function has to be called after channel is started - assert(!((m_d2h_callback_thread) && m_d2h_callback_thread->joinable())); - m_d2h_callback_thread = make_unique_nothrow([this, callback]() { - wait_d2h_callback(callback); - }); - CHECK_NOT_NULL(m_d2h_callback_thread, HAILO_OUT_OF_HOST_MEMORY); - - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::unregister_for_d2h_interrupts(std::unique_lock &lock) -{ - // This function has to be called after channel is stopped (after unregister_fw_controlled_channel is called) - if ((m_d2h_callback_thread) && (m_d2h_callback_thread->joinable())) { - // Let the channel finish processing interrupts - lock.unlock(); - m_d2h_callback_thread->join(); - lock.lock(); - } - return HAILO_SUCCESS; -} - -void VdmaChannel::wait_d2h_callback(const std::function &callback) -{ - assert(Direction::D2H == m_direction); - if(!m_buffer) { - LOGGER__ERROR("Wait called without allocating buffers"); - return; - } - while (true) { - auto status = wait_for_channel_completion(HAILO_INFINITE_TIMEOUT, callback); - if (HAILO_SUCCESS == status || (HAILO_STREAM_ABORTED_BY_USER == status)) { - // Ignore HAILO_STREAM_ABORTED_BY_USER as we want to keep waiting for interrupts until channel is stopped - continue; - } else if (HAILO_STREAM_NOT_ACTIVATED == status) { - // Finish gracefully - return; - } else { - LOGGER__ERROR("wait_d2h_callback failed with status={}", status); - return; - } - } -} - -hailo_status VdmaChannel::wait(size_t buffer_size, std::chrono::milliseconds timeout) -{ - if (!m_buffer) { - LOGGER__ERROR("Wait called without allocating buffers"); - return HAILO_INVALID_OPERATION; - } - - CHECK(buffer_size < m_buffer->size(), HAILO_INVALID_ARGUMENT, - "Requested transfer size ({}) must be smaller than ({})", buffer_size, m_buffer->size()); - - if ((Direction::D2H == m_direction) && ((m_d2h_callback_thread) && (m_d2h_callback_thread->joinable()))) { - std::unique_lock state_guard(*m_state); - hailo_status status = HAILO_SUCCESS; // Best effort - bool was_successful = m_can_read_buffer_cv.wait_for(state_guard, timeout, [this, buffer_size, &status] () { - if ((!m_channel_enabled) || (m_is_aborted_by_internal_source)) { - status = HAILO_STREAM_ABORTED_BY_USER; - return true; // return true so that the wait will finish - } - return is_ready_for_transfer_d2h(buffer_size); - }); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("wait_for in d2h wait was aborted!"); - return HAILO_STREAM_ABORTED_BY_USER; - } - CHECK(was_successful, HAILO_TIMEOUT); - return HAILO_SUCCESS; - } - auto is_ready_for_transfer = (Direction::H2D == m_direction) ? - std::bind(&VdmaChannel::is_ready_for_transfer_h2d, this, buffer_size) : - std::bind(&VdmaChannel::is_ready_for_transfer_d2h, this, buffer_size); - return wait_for_condition(is_ready_for_transfer, timeout); -} - -hailo_status VdmaChannel::transfer(void *buf, size_t count) -{ - CHECK((nullptr != buf) && (0 < count), HAILO_INVALID_ARGUMENT); - CHECK(nullptr != m_buffer, HAILO_INVALID_OPERATION, "Transfer called without allocating buffers"); - - hailo_status status = HAILO_UNINITIALIZED; - assert(m_state); - std::lock_guard state_guard(*m_state); - - if (m_is_aborted_by_internal_source) { - LOGGER__INFO("Tried to write to aborted channel {}", m_channel_id); - return HAILO_STREAM_ABORTED_BY_USER; - } - - if (Direction::H2D == m_direction) { - status = transfer_h2d(buf, count); - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__INFO("Transfer failed because Channel {} is not activated", m_channel_id); - return HAILO_STREAM_NOT_ACTIVATED; - } - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Transfer failed for channel {}", m_channel_id); - return status; - } - return HAILO_SUCCESS; - } else { - status = transfer_d2h(buf, count); - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__INFO("Transfer failed because Channel {} is not activated", m_channel_id); - return HAILO_STREAM_NOT_ACTIVATED; - } - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Transfer failed for channel {} status {}", m_channel_id, status); - return status; - } - return HAILO_SUCCESS; - } - - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::write_buffer_impl(const MemoryView &buffer) -{ - CHECK(nullptr != m_buffer, HAILO_INVALID_OPERATION, "Transfer called without allocating buffers"); - - size_t desired_desc_num = m_buffer->descriptors_in_buffer(buffer.size()); - uint32_t desc_avail = (get_num_available() + m_pending_num_avail_offset) & m_state->m_descs.size_mask; - - assert(CB_AVAIL(m_state->m_descs, desc_avail, CB_TAIL(m_state->m_descs)) >= static_cast(desired_desc_num)); - - /* Copy buffer into the PLDA data struct */ - auto offset = ((desc_avail + m_state->m_previous_tail) & m_state->m_descs.size_mask) * m_desc_page_size; - auto status = m_buffer->write_cyclic(buffer.data(), buffer.size(), offset); - CHECK_SUCCESS(status); - - m_pending_num_avail_offset = static_cast(m_pending_num_avail_offset + desired_desc_num); - - CHECK(!m_pending_buffers_sizes.full(), HAILO_INVALID_OPERATION, "Cannot add more pending buffers!"); - m_pending_buffers_sizes.push_back(buffer.size()); - return HAILO_SUCCESS; -} - -Expected VdmaChannel::get_buffer_state() -{ - BufferState result; - result.num_avail = static_cast(CB_HEAD(m_state->m_descs)); - result.num_processed = static_cast(CB_TAIL(m_state->m_descs)); - auto hw_num_avail = m_host_registers.get_num_available(); - CHECK_EXPECTED(hw_num_avail); - result.hw_num_avail = hw_num_avail.release(); - auto hw_num_processed = get_hw_num_processed(); - CHECK_EXPECTED(hw_num_processed); - result.hw_num_processed = hw_num_processed.release(); - - // Get a snapshot of the buffer - auto vdma_buffer_copy = Buffer::create(m_buffer->size()); - CHECK_EXPECTED(vdma_buffer_copy); - // If this a D2H channel, we need to sync the vdma buffer so that we'll get an updated view of the buffer - const auto sync_needed = Direction::D2H == m_direction; - const auto status = m_buffer->read_cyclic(vdma_buffer_copy->data(), vdma_buffer_copy->size(), 0, sync_needed); - CHECK_SUCCESS_AS_EXPECTED(status); - - for (size_t offset = 0; offset < vdma_buffer_copy->size(); offset += m_desc_page_size) { - auto chunk = Buffer::create(vdma_buffer_copy->data() + offset, m_desc_page_size); - CHECK_EXPECTED(chunk); - const auto abs_index = offset / m_desc_page_size; - const auto desc_num = (abs_index >= static_cast(m_state->m_previous_tail)) ? - abs_index - m_state->m_previous_tail : - m_state->m_descs.size - m_state->m_previous_tail + abs_index; - result.desc_buffer_pairing.emplace_back(static_cast(desc_num), chunk.release()); - } - - return result; -} - -hailo_status VdmaChannel::write_buffer(const MemoryView &buffer, std::chrono::milliseconds timeout, - const std::function &should_cancel) -{ - assert(m_state); - std::unique_lock state_guard(*m_state); - - size_t desired_desc_num = m_buffer->descriptors_in_buffer(buffer.size()); - hailo_status channel_completion_status = HAILO_SUCCESS; - bool was_successful = m_can_write_buffer_cv.wait_for(state_guard, timeout, [this, desired_desc_num, timeout, &should_cancel, - &state_guard, &channel_completion_status] () { - if ((!m_channel_enabled) || (m_is_aborted_by_internal_source)) { - return true; - } - - if (should_cancel()) { - channel_completion_status = HAILO_STREAM_ABORTED_BY_USER; - return true; - } - - // Limit writes to not surpass size of m_buffers - int written_buffers_count = static_cast(m_pending_buffers_sizes.size()); - int sent_buffers_count = CB_PROG(m_state->m_buffers, CB_HEAD(m_state->m_buffers), CB_TAIL(m_state->m_buffers)); - if (written_buffers_count + sent_buffers_count >= CB_SIZE(m_state->m_buffers)) { - return false; - } - - // TODO (HRT-7252): Clean this code - while (true) { - int buffers_head = CB_HEAD(m_state->m_buffers); - int buffers_tail = CB_TAIL(m_state->m_buffers); - bool has_space_in_buffers = CB_AVAIL(m_state->m_buffers, buffers_head, buffers_tail); - - uint32_t desc_avail = (get_num_available() + m_pending_num_avail_offset) & m_state->m_descs.size_mask; - int num_free = CB_AVAIL(m_state->m_descs, desc_avail, CB_TAIL(m_state->m_descs)); - bool has_desc_space = (num_free >= static_cast(desired_desc_num)); - - if (has_space_in_buffers && has_desc_space) { - break; - } - - if (HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE == *m_channel_handle) { - return false; - } - - state_guard.unlock(); - channel_completion_status = wait_for_channel_completion(timeout); - state_guard.lock(); - if (HAILO_SUCCESS != channel_completion_status) { - LOGGER__INFO("wait_for_channel_completion failed with status={}", channel_completion_status); - return true; - } - } - - return true; - }); - if ((!m_channel_enabled) || (m_is_aborted_by_internal_source) || (HAILO_STREAM_ABORTED_BY_USER == channel_completion_status)) { - LOGGER__INFO("wait_for in write_buffer was aborted!"); - return HAILO_STREAM_ABORTED_BY_USER; - } - CHECK(was_successful, HAILO_TIMEOUT, "Waiting for descriptors in write_buffer has reached a timeout!"); - if (HAILO_STREAM_ABORTED_BY_USER == channel_completion_status) { - return HAILO_STREAM_ABORTED_BY_USER; - } - CHECK_SUCCESS(channel_completion_status); - - return write_buffer_impl(buffer); -} - -hailo_status VdmaChannel::send_pending_buffer_impl() -{ - CHECK(!m_pending_buffers_sizes.empty(), HAILO_INVALID_OPERATION, "There are no pending buffers to send!"); - assert(m_buffer); - - // For h2d, only the host need to get transfer done interrupts - VdmaInterruptsDomain last_desc_interrupts_domain = VdmaInterruptsDomain::HOST; - // If we measure latency, we need interrupt on the first descriptor - VdmaInterruptsDomain first_desc_interrupts_domain = (m_latency_meter != nullptr) ? - VdmaInterruptsDomain::HOST : VdmaInterruptsDomain::NONE; - - auto status = prepare_descriptors(m_pending_buffers_sizes.front(), first_desc_interrupts_domain, last_desc_interrupts_domain); - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__INFO("sending pending buffer failed because stream is not activated"); - // Stream was aborted during transfer - reset pending buffers - m_pending_num_avail_offset = 0; - while (m_pending_buffers_sizes.size() > 0) { - m_pending_buffers_sizes.pop_front(); - } - return status; - } - CHECK_SUCCESS(status); - m_state->m_accumulated_transfers = (m_state->m_accumulated_transfers + 1) % m_transfers_per_axi_intr; - - size_t desired_desc_num = m_buffer->descriptors_in_buffer(m_pending_buffers_sizes.front()); - m_pending_num_avail_offset = static_cast(m_pending_num_avail_offset - desired_desc_num); - - m_pending_buffers_sizes.pop_front(); - - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::send_pending_buffer() -{ - { - assert(m_state); - assert(m_buffer); - std::lock_guard state_guard(*m_state); - - auto status = send_pending_buffer_impl(); - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__INFO("stream is not activated"); - return HAILO_STREAM_NOT_ACTIVATED; - } else { - CHECK_SUCCESS(status); - } - } - m_can_write_buffer_cv.notify_one(); - - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::sync_state(std::chrono::milliseconds timeout) -{ - { - std::lock_guard state_guard(*m_state); - - // Make sure that only one thread is waiting for channel completion - if (m_is_waiting_for_channel_completion) { - return HAILO_SUCCESS; - } - } - return wait_for_channel_completion(timeout); -} - -hailo_status VdmaChannel::flush(const std::chrono::milliseconds &timeout) -{ - assert(m_state); - - if (Direction::D2H == m_direction) { - // We are not buffering user data - return HAILO_SUCCESS; - } - - if (!m_buffer) { - LOGGER__ERROR("VdmaChannel::flush is called on a channel without allocated resources"); - return HAILO_INVALID_OPERATION; - } - - return wait_for_condition([this] { return CB_HEAD(m_state->m_buffers) == CB_TAIL(m_state->m_buffers); }, timeout); -} - -hailo_status VdmaChannel::transfer_h2d(void *buf, size_t count) -{ - auto status = write_buffer_impl(MemoryView(buf, count)); - CHECK_SUCCESS(status); - - status = send_pending_buffer_impl(); - if (HAILO_STREAM_NOT_ACTIVATED == status) { - return status; - } else { - CHECK_SUCCESS(status); - } - - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::transfer_d2h(void *buf, size_t count) -{ - hailo_status status = HAILO_UNINITIALIZED; - /* Provide FW interrupt only in the end of the last transfer in the batch */ - VdmaInterruptsDomain first_desc_interrupts_domain = VdmaInterruptsDomain::NONE; - VdmaInterruptsDomain last_desc_interrupts_domain = (m_state->m_accumulated_transfers + 1 == m_transfers_per_axi_intr) ? - VdmaInterruptsDomain::BOTH : VdmaInterruptsDomain::HOST; - - assert(m_state); - assert(m_buffer); - - auto desired_desc_num = m_buffer->descriptors_in_buffer(count); - assert(desired_desc_num <= MAX_DESCS_COUNT); - int desc_num = static_cast(desired_desc_num); - - int num_processes = CB_TAIL(m_state->m_descs); - int num_ready = CB_PROG(m_state->m_descs, num_processes, m_state->m_d2h_read_desc_index); - if (num_ready < desc_num) { - return HAILO_OUT_OF_DESCRIPTORS; - } - - size_t offset = m_state->m_d2h_read_desc_index * m_desc_page_size; - status = m_buffer->read_cyclic(buf, count, offset); - if (status != HAILO_SUCCESS) { - return status; - } - - m_state->m_d2h_read_desc_index = (m_state->m_d2h_read_desc_index + desc_num) & m_state->m_descs.size_mask; - - // prepare descriptors for next recv - if (*m_channel_handle != HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE) { - status = prepare_descriptors(count, first_desc_interrupts_domain, last_desc_interrupts_domain); - if (HAILO_STREAM_NOT_ACTIVATED == status) { - LOGGER__INFO("transfer d2h failed because stream is not activated"); - return status; - } - CHECK_SUCCESS(status); - } - - m_state->m_accumulated_transfers = (m_state->m_accumulated_transfers + 1) % m_transfers_per_axi_intr; - - return HAILO_SUCCESS; -} - -uint16_t VdmaChannel::get_num_available() -{ - assert(m_state); - - uint16_t num_available = (uint16_t)CB_HEAD(m_state->m_descs); - -#ifndef NDEBUG - // Validate synchronization with HW - auto hw_num_avail = m_host_registers.get_num_available(); - assert(hw_num_avail); - // On case of channel aborted, the num_available is set to 0 (so we don't accept sync) - - auto is_aborted_exp = is_aborted(); - assert(is_aborted_exp); - - if ((HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE != *m_channel_handle) && !is_aborted_exp.value()) { - assert(hw_num_avail.value() == num_available); - } -#endif - return num_available; -} - -Expected VdmaChannel::get_hw_num_processed() -{ - assert(m_state); - - auto hw_num_processed = m_host_registers.get_num_processed(); - CHECK_EXPECTED(hw_num_processed, "Fail to read vdma num processed register"); - - // Although the hw_num_processed should be a number between 0 and m_descs.size-1, if - // m_desc.size < 0x10000 (the maximum desc size), the actual hw_num_processed is a number - // between 1 and m_descs.size. Therefore the value can be m_descs.size, in this case we change it - // to zero. - return static_cast(hw_num_processed.value() & m_state->m_descs.size_mask); -} - -hailo_status VdmaChannel::set_num_avail_value(uint16_t new_value) -{ - // TODO - HRT-7885 : add check in driver - CHECK(*m_channel_handle != HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE, HAILO_STREAM_NOT_ACTIVATED, - "Error, can't set num available when stream is not activated"); - - auto status = m_host_registers.set_num_available(new_value); - CHECK_SUCCESS(status, "Fail to write vdma num available register"); - -#ifndef NDEBUG - // Validate synchronization with HW - auto hw_num_avail = m_host_registers.get_num_available(); - assert(hw_num_avail); - assert(hw_num_avail.value() == new_value); -#endif - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::set_transfers_per_axi_intr(uint16_t transfers_per_axi_intr) -{ - CHECK(0 != transfers_per_axi_intr, HAILO_INVALID_ARGUMENT, "Invalid transfers per axi interrupt"); - m_transfers_per_axi_intr = transfers_per_axi_intr; - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::inc_num_available(uint16_t value) -{ - assert(m_state); - - //TODO: validate that count is added. - int num_available = get_num_available(); - int num_processed = CB_TAIL(m_state->m_descs); - int num_free = CB_AVAIL(m_state->m_descs, num_available, num_processed); - if (value > num_free) { - return HAILO_OUT_OF_DESCRIPTORS; - } - - CB_ENQUEUE(m_state->m_descs, value); - num_available = (num_available + value) & m_state->m_descs.size_mask; - return set_num_avail_value(static_cast(num_available)); -} - -void VdmaChannel::add_pending_buffer(uint32_t first_desc, uint32_t last_desc) -{ - assert(m_state); - - int head = CB_HEAD(m_state->m_buffers); - int tail = CB_TAIL(m_state->m_buffers); - if (!CB_AVAIL(m_state->m_buffers, head, tail)) { - LOGGER__ERROR("no avail space"); - } - m_state->m_pending_buffers[head].last_desc = last_desc; - m_state->m_pending_buffers[head].latency_measure_desc = (m_direction == Direction::H2D) ? first_desc : last_desc; - CB_ENQUEUE(m_state->m_buffers, 1); -} - -VdmaChannel::Direction VdmaChannel::other_direction(Direction direction) -{ - return (Direction::H2D == direction) ? Direction::D2H : Direction::H2D; -} - -hailo_status VdmaChannel::unregister_fw_controlled_channel() -{ - assert(m_channel_handle); - - if (m_state) { - // m_state is locked from stop_channel - m_state->m_channel_is_active = false; - } - - if (HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE != *m_channel_handle) { - auto status = m_driver.vdma_channel_disable(m_channel_id, *m_channel_handle); - *m_channel_handle = HailoRTDriver::INVALID_VDMA_CHANNEL_HANDLE; - CHECK_SUCCESS(status, "Failed to disable channel {}", m_channel_id); - } - - return HAILO_SUCCESS; -} - -hailo_status VdmaChannel::register_channel_to_driver() -{ - const bool measure_latency = (nullptr != m_latency_meter); - auto channel_handle = m_driver.vdma_channel_enable(m_channel_id, m_direction, measure_latency); - CHECK_EXPECTED_AS_STATUS(channel_handle, "Failed to enable channel {}", m_channel_id); - - *m_channel_handle = channel_handle.release(); - - if (m_state) { - std::lock_guard state_guard(*m_state); - m_state->m_channel_is_active = true; - } - - return HAILO_SUCCESS; -} - -// TODO - HRT-6984 - move function inside desc list class as part of the ctor -void VdmaChannel::clear_descriptor_list() -{ - assert(m_buffer); - - size_t desc_number = m_buffer->descs_count(); - size_t page_size = m_buffer->desc_page_size(); - auto desc_list = m_buffer->get_desc_list(); - - // Config Descriptors value in SG-List Host side - for (uint32_t j = 0; j < desc_number; j++) { - VdmaDescriptor &descInfo = (desc_list->get())[j]; - descInfo.PageSize_DescControl = static_cast((page_size << 8) + 0x2); - descInfo.RemainingPageSize_Status = 0x0; - } -} - -hailo_status VdmaChannel::allocate_buffer(const uint32_t buffer_size) -{ - assert((buffer_size % m_desc_page_size) == 0); - uint32_t desc_count = buffer_size / m_desc_page_size; - - if (m_buffer) { - LOGGER__ERROR("m_buffer is not NULL"); - return HAILO_INVALID_OPERATION; - } - - auto buffer = vdma::SgBuffer::create(m_driver, desc_count, m_desc_page_size, m_direction, - m_channel_id.channel_index); - CHECK_EXPECTED_AS_STATUS(buffer); - - m_buffer = make_unique_nothrow(buffer.release()); - CHECK_NOT_NULL(m_buffer, HAILO_OUT_OF_HOST_MEMORY); - - return HAILO_SUCCESS; -} - -uint32_t VdmaChannel::calculate_buffer_size(const HailoRTDriver &driver, uint32_t transfer_size, - uint32_t transfers_count, uint16_t requested_desc_page_size) { - auto desc_page_size = driver.calc_desc_page_size(requested_desc_page_size); - uint32_t descs_per_transfer = VdmaDescriptorList::descriptors_in_buffer(transfer_size, desc_page_size); - uint32_t descs_count = descs_per_transfer * transfers_count; - - if (descs_count > MAX_DESCS_COUNT) { - descs_count = MAX_DESCS_COUNT; - } - else if (descs_count < MIN_DESCS_COUNT) { - descs_count = MIN_DESCS_COUNT; - } - - return descs_count * desc_page_size; -} - -hailo_status VdmaChannel::trigger_channel_completion(uint16_t hw_num_processed, const std::function &callback) -{ - // NOTE: right now, we can retake the 'completion' descriptor for a new transfer before handling the interrupt. - // we should have our own pointers indicating whats free instead of reading from HW. - // TODO: consider calculating the last descriptor using the src_desc_avail and src_desc_proc instead of using - // status? - // TODO: we might free a pending buffer which we didn't get an interrupt for yet. we should still handle this - // situation correctly. - - assert(m_state); - assert(m_buffer); - std::lock_guard state_guard(*m_state); - - if (m_is_aborted_by_internal_source) { - return HAILO_STREAM_ABORTED_BY_USER; - } - - int processed_no = 0; - int head = CB_HEAD(m_state->m_buffers); - int tail = CB_TAIL(m_state->m_buffers); - int prog = CB_PROG(m_state->m_buffers, head, tail); - int last_tail = -1; - auto channel_error = m_host_registers.get_channel_error(); - CHECK_EXPECTED_AS_STATUS(channel_error, "Fail to read vdma channel error register"); - CHECK(0 == channel_error.value(), HAILO_INTERNAL_FAILURE, "Vdma channel {} in error state {}", m_channel_id, - channel_error.value()); - - uint16_t last_num_processed = static_cast(CB_TAIL(m_state->m_descs)); - - for (; prog > 0; prog--) { - uint16_t last_desc_index = static_cast(m_state->m_pending_buffers[tail].last_desc); - // Transfer is complete if its last descriptor is in [last_num_processed, hw_num_processed) or - // the the buffer is empty (hw_num_processed == get_num_available()) - bool is_complete = is_desc_between(last_num_processed, hw_num_processed, last_desc_index) || (hw_num_processed == get_num_available()); - -#ifndef NDEBUG - auto status = (m_buffer->get_desc_list()->get())[last_desc_index].RemainingPageSize_Status & 0xFF; - // Verify if a DMA Descriptor error occurred. - if (status & 0x2) { - LOGGER__ERROR("Error while processing descriptor {} of DMA {} on board {}.", last_desc_index, m_channel_id, - m_driver.dev_path()); - return HAILO_INTERNAL_FAILURE; - } - - // status is read after hw_num_processed, so we want is_complete -> (status == 1). - assert(!is_complete || ((status & 0x1) == 1)); -#endif - - if (!is_complete) { - break; - } - - processed_no++; - last_tail = tail; - tail = ((tail + 1) & m_state->m_buffers.size_mask); - } - - if (0 < processed_no) { - // TODO: use a different macro instead? - _CB_SET(m_state->m_descs.tail, (m_state->m_pending_buffers[last_tail].last_desc + 1) & m_state->m_descs.size_mask); - CB_DEQUEUE(m_state->m_buffers, processed_no); - - if (Direction::H2D == m_direction) { - m_can_write_buffer_cv.notify_one(); - } else { - m_can_read_buffer_cv.notify_one(); - } - callback(processed_no); - } - - m_is_waiting_for_channel_completion = false; - return HAILO_SUCCESS; -} - -bool VdmaChannel::is_ready_for_transfer_h2d(size_t buffer_size) -{ - assert(m_state); - assert(m_buffer); - - size_t desired_desc_num = m_buffer->descriptors_in_buffer(buffer_size); - assert(desired_desc_num <= MAX_DESCS_COUNT); - int desc_num = static_cast(desired_desc_num); - - int buffers_head = CB_HEAD(m_state->m_buffers); - int buffers_tail = CB_TAIL(m_state->m_buffers); - if (!CB_AVAIL(m_state->m_buffers, buffers_head, buffers_tail)) { - return false; - } - - int num_available = get_num_available(); - int num_processed = CB_TAIL(m_state->m_descs); - - if (desc_num == m_state->m_descs.size) { - // Special case when the checking if the buffer is empty - return num_available == num_processed; - } - - int num_free = CB_AVAIL(m_state->m_descs, num_available, num_processed); - if (num_free < desc_num) { - return false; - } - - return true; -} - -bool VdmaChannel::is_ready_for_transfer_d2h(size_t buffer_size) -{ - assert(m_state); - assert(m_buffer); - - size_t desired_desc_num = m_buffer->descriptors_in_buffer(buffer_size); - assert(desired_desc_num <= MAX_DESCS_COUNT); - int desc_num = static_cast(desired_desc_num); - - int buffers_head = CB_HEAD(m_state->m_buffers); - int buffers_tail = CB_TAIL(m_state->m_buffers); - if (!CB_AVAIL(m_state->m_buffers, buffers_head, buffers_tail)) { - return false; - } - - int num_processed = CB_TAIL(m_state->m_descs); - int num_ready = CB_PROG(m_state->m_descs, num_processed, m_state->m_d2h_read_desc_index); - if (num_ready < desc_num) { - return false; - } - return true; -} - -hailo_status VdmaChannel::prepare_descriptors(size_t transfer_size, VdmaInterruptsDomain first_desc_interrupts_domain, - VdmaInterruptsDomain last_desc_interrupts_domain) -{ - assert(m_buffer); - assert(m_state); - auto desc_info = m_buffer->get_desc_list(); - - /* calculate desired descriptors for the buffer */ - size_t desired_desc_num = m_buffer->descriptors_in_buffer(transfer_size); - assert(desired_desc_num <= MAX_DESCS_COUNT); - uint16_t desc_num = static_cast(desired_desc_num); - - int num_available = get_num_available(); - int num_processed = CB_TAIL(m_state->m_descs); - int num_free = CB_AVAIL(m_state->m_descs, num_available, num_processed); - if (num_free < desc_num) { - return HAILO_OUT_OF_DESCRIPTORS; - } - - auto actual_desc_count = desc_info->get().program_descriptors(transfer_size, first_desc_interrupts_domain, - last_desc_interrupts_domain, num_available, true); - if (!actual_desc_count) { - LOGGER__ERROR("Failed to program desc_list for channel {}", m_channel_id); - return actual_desc_count.status(); - } - assert (actual_desc_count.value() == desc_num); - int last_desc_avail = ((num_available + desc_num - 1) & m_state->m_descs.size_mask); - - add_pending_buffer(num_available, last_desc_avail); - return inc_num_available(desc_num); -} - -uint32_t VdmaChannel::calculate_descriptors_count(uint32_t buffer_size) -{ - return VdmaDescriptorList::calculate_descriptors_count(buffer_size, 1, m_desc_page_size); -} - -bool VdmaChannel::is_desc_between(uint16_t begin, uint16_t end, uint16_t desc) -{ - if (begin == end) { - // There is nothing between - return false; - } - if (begin < end) { - // desc needs to be in [begin, end) - return (begin <= desc) && (desc < end); - } - else { - // desc needs to be in [0, end) or [begin, m_state->m_descs.size()-1] - return (desc < end) || (begin <= desc); - } -} - -Expected VdmaChannel::is_aborted() -{ - // Checking if either src side or dst side of the channel are aborted - auto host_control = m_host_registers.get_control(); - CHECK_EXPECTED(host_control, "Fail to read vdma control register"); - if (vdma_channel_control_is_aborted(host_control.value()) || - vdma_channel_control_is_paused(host_control.value())) { - return true; - } - - auto device_control = m_device_registers.get_control(); - CHECK_EXPECTED(device_control, "Fail to read vdma control register"); - if (vdma_channel_control_is_aborted(device_control.value()) || - vdma_channel_control_is_paused(device_control.value())) { - return true; - } - - return false; -} - -hailo_status VdmaChannel::wait_for_condition(std::function condition, std::chrono::milliseconds timeout) -{ - auto start_time = std::chrono::steady_clock::now(); - std::chrono::milliseconds time_elapsed(0); - while (timeout > time_elapsed) { - if (condition()) { - return HAILO_SUCCESS; - } - - auto status = wait_for_channel_completion(timeout); - if (HAILO_SUCCESS != status) { - return status; - } - - time_elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - start_time); - } - - return condition() ? HAILO_SUCCESS : HAILO_TIMEOUT; -} - -hailo_status VdmaChannel::wait_for_channel_completion(std::chrono::milliseconds timeout, const std::function &callback) -{ - auto hw_num_processed = wait_interrupts(timeout); - if ((hw_num_processed.status() == HAILO_TIMEOUT) || - (hw_num_processed && hw_num_processed.value() == 0)) { - // We need to check for channel abort in this 2 cases: - // 1. TIMEOUT - maybe the timeout is a result of channel aborted. - // 2. hw_num_processed == 0 - In this case we receive an interrupt, but the channel may be - // aborted. When the channel is aborted, num processed is set to 0. - auto is_aborted_exp = is_aborted(); - CHECK_EXPECTED_AS_STATUS(is_aborted_exp); - if (is_aborted_exp.value()) { - assert(m_state); - std::lock_guard state_guard(*m_state); - if (!m_state->m_channel_is_active) { - return HAILO_STREAM_NOT_ACTIVATED; - } - - LOGGER__CRITICAL("Channel {} was aborted by an external source!", m_channel_id); - return HAILO_STREAM_ABORTED; - } - } - if ((HAILO_STREAM_ABORTED_BY_USER == hw_num_processed.status()) || - (HAILO_STREAM_NOT_ACTIVATED == hw_num_processed.status())) { - return hw_num_processed.status(); - } - CHECK_EXPECTED_AS_STATUS(hw_num_processed); - - auto status = trigger_channel_completion(hw_num_processed.value(), callback); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - return status; - } - CHECK_SUCCESS(status); - - return HAILO_SUCCESS; -} - -Expected VdmaChannel::wait_interrupts(std::chrono::milliseconds timeout) -{ - assert(m_state); - - auto irq_data = m_driver.wait_channel_interrupts(m_channel_id, *m_channel_handle, timeout); - if ((HAILO_STREAM_ABORTED_BY_USER == irq_data.status()) || - (HAILO_STREAM_NOT_ACTIVATED == irq_data.status())) { - LOGGER__INFO("Wait channel interrupts was aborted!"); - return make_unexpected(irq_data.status()); - } - CHECK_EXPECTED(irq_data); - - if (m_latency_meter == nullptr) { - return get_hw_num_processed(); - } - else { - // Fixing desc num_processed (it may be equal to m_state->m_descs.size, in this case we will make it zero) - for (size_t i = 0; i < irq_data->count; i++) { - irq_data->timestamp_list[i].desc_num_processed = static_cast( - irq_data->timestamp_list[i].desc_num_processed & m_state->m_descs.size_mask); - } - return update_latency_meter(irq_data.value()); - } -} - -Expected VdmaChannel::update_latency_meter(const ChannelInterruptTimestampList ×tamp_list) -{ - assert(m_state); - - uint16_t last_num_processed = m_state->m_last_timestamp_num_processed; - - if (timestamp_list.count == 0) { - // TODO: handle this in the driver level. - return last_num_processed; - } - - // TODO: now we have more iterations than we need. We know that the pending buffers + the timestamp list - // are ordered. If pending_buffer[i] is not in any of the timestamps_list[0, 1, ... k], then also pending_buffer[i+1,i+2,...] - // not in those timestamps - - int head = CB_HEAD(m_state->m_buffers); - int tail = CB_TAIL(m_state->m_buffers); - int prog = CB_PROG(m_state->m_buffers, head, tail); - - for (; prog > 0; prog--, tail = ((tail + 1) & m_state->m_buffers.size_mask)) { - uint16_t latency_desc = static_cast(m_state->m_pending_buffers[tail].latency_measure_desc); - for (size_t i = 0; i < timestamp_list.count; i++) { - const auto &irq_timestamp = timestamp_list.timestamp_list[i]; - if (is_desc_between(last_num_processed, irq_timestamp.desc_num_processed, latency_desc)) { - if (m_direction == Direction::H2D) { - m_latency_meter->add_start_sample(irq_timestamp.timestamp); - } - else { - m_latency_meter->add_end_sample(m_stream_name, irq_timestamp.timestamp); - } - break; - } - } - } - - m_state->m_last_timestamp_num_processed = timestamp_list.timestamp_list[timestamp_list.count-1].desc_num_processed; - return std::move(static_cast(m_state->m_last_timestamp_num_processed)); -} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma_channel.hpp b/hailort/libhailort/src/vdma_channel.hpp deleted file mode 100644 index ad65e92..0000000 --- a/hailort/libhailort/src/vdma_channel.hpp +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file vdma_channel.hpp - * @brief Cordinator of everything related to one channel of Vdma - * - * - **/ - -#ifndef _HAILO_VDMA_CHANNEL_HPP_ -#define _HAILO_VDMA_CHANNEL_HPP_ - -#include "hailo/hailort.h" -#include "common/circular_buffer.hpp" -#include "common/latency_meter.hpp" -#include "vdma_channel_regs.hpp" -#include "hailo/expected.hpp" -#include "os/hailort_driver.hpp" -#include "vdma/sg_buffer.hpp" -#include "vdma_descriptor_list.hpp" -#include "vdma/channel_id.hpp" -#include "hailo/buffer.hpp" - -#include -#include -#include - -namespace hailort -{ - -class VdmaChannel final -{ -public: - using Direction = HailoRTDriver::DmaDirection; - - static Expected create(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, - uint16_t requested_desc_page_size, const std::string &stream_name = "", LatencyMeterPtr latency_meter = nullptr, - uint16_t transfers_per_axi_intr = 1); - ~VdmaChannel(); - - /** - * Waits until the channel is ready for transfer `buffer_size` bytes. - * For now only supported in H2D stream. - * TODO: SDK-15831 support D2H - * - * @param[in] buffer_size - * @param[in] timeout - */ - hailo_status wait(size_t buffer_size, std::chrono::milliseconds timeout); - - hailo_status transfer(void *buf, size_t count); - // Either write_buffer + send_pending_buffer or transfer (h2d) should be used on a given channel, not both - hailo_status write_buffer(const MemoryView &buffer, std::chrono::milliseconds timeout, const std::function &should_cancel); - hailo_status send_pending_buffer(); - hailo_status trigger_channel_completion(uint16_t hw_num_processed, const std::function &callback); - hailo_status allocate_resources(uint32_t descs_count); - // Call for boundary channels, after the fw has activted them (via ResourcesManager::enable_state_machine) - hailo_status complete_channel_activation(uint32_t transfer_size); - // Libhailort registers the channels to the driver and the FW is responsible for opening and closing them - hailo_status register_fw_controlled_channel(); - hailo_status unregister_fw_controlled_channel(); - // For D2H channels, we don't buffer data - // Hence there's nothing to be "flushed" and the function will return with HAILO_SUCCESS - hailo_status flush(const std::chrono::milliseconds &timeout); - hailo_status set_num_avail_value(uint16_t new_value); - hailo_status set_transfers_per_axi_intr(uint16_t transfers_per_axi_intr); - hailo_status inc_num_available_for_ddr(uint16_t value, uint32_t size_mask); - Expected get_hw_num_processed_ddr(uint32_t size_mask); - - hailo_status stop_channel(); - uint16_t get_page_size(); - Expected get_boundary_buffer_info(uint32_t transfer_size); - - hailo_status abort(); - hailo_status clear_abort(); - - class BufferState { - public: - std::vector> desc_buffer_pairing; - uint16_t num_avail; - uint16_t num_processed; - uint16_t hw_num_avail; - uint16_t hw_num_processed; - }; - // Assumes that the channel is idle; doesn't block changes to the channel - // To be used for debugging purposes - Expected get_buffer_state(); - - // To be used for debugging purposes - hailo_status sync_state(std::chrono::milliseconds timeout); - - vdma::ChannelId get_channel_id() const - { - return m_channel_id; - } - - size_t get_transfers_count_in_buffer(size_t transfer_size); - size_t get_buffer_size() const; - Expected get_h2d_pending_frames_count(); - Expected get_d2h_pending_descs_count(); - - VdmaChannel(const VdmaChannel &other) = delete; - VdmaChannel &operator=(const VdmaChannel &other) = delete; - VdmaChannel(VdmaChannel &&other) noexcept; - VdmaChannel &operator=(VdmaChannel &&other) = delete; - - static uint32_t calculate_buffer_size(const HailoRTDriver &driver, uint32_t transfer_size, uint32_t transfers_count, - uint16_t requested_desc_page_size); - - hailo_status register_for_d2h_interrupts(const std::function &callback); - - void notify_all(); - -private: - struct PendingBuffer { - uint32_t last_desc; - uint32_t latency_measure_desc; - }; - - // TODO (HRT-3762) : Move channel's state to driver to avoid using shared memory - class State { - public: - - void lock(); - void unlock(); - -#ifndef _MSC_VER - pthread_mutex_t m_state_lock; -#else - CRITICAL_SECTION m_state_lock; -#endif - std::array m_pending_buffers; - circbuf_t m_buffers; - // TODO: describe why we must have our own num_available and num_proc. - // it's not just for efficiency but its critical to avoid a potential bug - see Avigail email. - // TODO: Consider C11 stdatomic - circbuf_t m_descs; - int m_d2h_read_desc_index; - // TODO: We want to refactor this class + VdmaChannel so that logic related to write_buffer + send_pending_buffer will - // be in another class. - // Points to the tail of the desc list when the channel is stopped (starts at zero) - // When calling VdmaChannel::write_buffer, buffers will be appended relative to this index (+ the current num_avail) - // We'll set it if there are pending buffers to be sent or if m_should_reprogram_buffer is set - int m_previous_tail; - bool m_should_reprogram_buffer; - // Contains the last num_processed of the last interrupt (only used on latency measurement) - uint16_t m_last_timestamp_num_processed; - size_t m_accumulated_transfers; - bool m_channel_is_active; - }; - - hailo_status register_channel_to_driver(); - hailo_status unregister_for_d2h_interrupts(std::unique_lock &lock); - - VdmaChannel(vdma::ChannelId channel_id, Direction direction, HailoRTDriver &driver, const std::string &stream_name, - LatencyMeterPtr latency_meter, uint16_t desc_page_size, uint16_t transfers_per_axi_intr, hailo_status &status); - - hailo_status allocate_buffer(const uint32_t buffer_size); - void clear_descriptor_list(); - hailo_status release_buffer(); - static Direction other_direction(const Direction direction); - hailo_status transfer_h2d(void *buf, size_t count); - hailo_status write_buffer_impl(const MemoryView &buffer); - hailo_status send_pending_buffer_impl(); - uint16_t get_num_available(); - Expected get_hw_num_processed(); - void add_pending_buffer(uint32_t first_desc, uint32_t last_desc); - hailo_status inc_num_available(uint16_t value); - hailo_status transfer_d2h(void *buf, size_t count); - bool is_ready_for_transfer_h2d(size_t buffer_size); - bool is_ready_for_transfer_d2h(size_t buffer_size); - hailo_status prepare_descriptors(size_t transfer_size, VdmaInterruptsDomain first_desc_interrupts_domain, - VdmaInterruptsDomain last_desc_interrupts_domain); - hailo_status prepare_d2h_pending_descriptors(uint32_t transfer_size); - void reset_internal_counters(); - hailo_status wait_for_channel_completion(std::chrono::milliseconds timeout, const std::function &callback = [](uint32_t) { return; }); - - uint32_t calculate_descriptors_count(uint32_t buffer_size); - - hailo_status wait_for_condition(std::function condition, std::chrono::milliseconds timeout); - - void wait_d2h_callback(const std::function &callback); - std::unique_ptr m_d2h_callback_thread; - - /** - * Returns the new hw num_processed of the irq - */ - Expected wait_interrupts(std::chrono::milliseconds timeout); - - /** - * Returns the new hw num processed. - */ - Expected update_latency_meter(const ChannelInterruptTimestampList ×tamp_list); - static bool is_desc_between(uint16_t begin, uint16_t end, uint16_t desc); - Expected is_aborted(); - - const vdma::ChannelId m_channel_id; - Direction m_direction; - HailoRTDriver &m_driver; - VdmaChannelRegs m_host_registers; - VdmaChannelRegs m_device_registers; - - // TODO: use m_descriptors_buffer.desc_page_size() - const uint16_t m_desc_page_size; - - // TODO: remove the unique_ptr, instead allocate the buffer in the ctor (needs to move ddr channel to - // other class) - std::unique_ptr m_buffer; - const std::string m_stream_name; - LatencyMeterPtr m_latency_meter; - - MmapBuffer m_state; - // Unique channel handle, may be changed between registration to driver. This object is shared - // because multiple processes can enable/disable vdma channel (which changes the channel) - MmapBuffer m_channel_handle; - - bool m_channel_enabled; - - uint16_t m_transfers_per_axi_intr; - // Using CircularArray because it won't allocate or free memory wile pushing and poping. The fact that it is circural is not relevant here - CircularArray m_pending_buffers_sizes; - std::atomic_uint16_t m_pending_num_avail_offset; - std::condition_variable_any m_can_write_buffer_cv; - std::condition_variable_any m_can_read_buffer_cv; - std::atomic_bool m_is_waiting_for_channel_completion; - std::atomic_bool m_is_aborted_by_internal_source; -}; - -} /* namespace hailort */ - -#endif // _HAILO_VDMA_CHANNEL_HPP_ \ No newline at end of file diff --git a/hailort/libhailort/src/vdma_stream.cpp b/hailort/libhailort/src/vdma_stream.cpp deleted file mode 100644 index ee31511..0000000 --- a/hailort/libhailort/src/vdma_stream.cpp +++ /dev/null @@ -1,498 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file vdma_stream.cpp - **/ - -#include "vdma_stream.hpp" -#include "pcie_stream.hpp" -#include "core_stream.hpp" - -namespace hailort -{ - -Expected> VdmaInputStream::create(VdmaDevice &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event) -{ - switch (device.get_type()) { - case Device::Type::PCIE: - { - auto local_stream = PcieInputStream::create(device, channel, edge_layer, batch_size, - network_group_activated_event); - CHECK_EXPECTED(local_stream); - return std::unique_ptr(local_stream.release()); - } - case Device::Type::CORE: - { - auto local_stream = CoreInputStream::create(device, channel, edge_layer, batch_size, - network_group_activated_event); - CHECK_EXPECTED(local_stream); - return std::unique_ptr(local_stream.release()); - } - default: - assert(false); - LOGGER__ERROR("Invalid device type {}", static_cast(device.get_type())); - return make_unexpected(HAILO_INTERNAL_FAILURE); - } -} - -VdmaInputStream::VdmaInputStream(VdmaDevice &device, std::shared_ptr channel, - const LayerInfo &edge_layer, EventPtr network_group_activated_event, uint16_t batch_size, - std::chrono::milliseconds transfer_timeout, - hailo_stream_interface_t stream_interface, hailo_status &status) : - InputStreamBase(edge_layer, stream_interface, std::move(network_group_activated_event), status), - m_device(&device), - m_channel(std::move(channel)), - is_stream_activated(false), - m_channel_timeout(transfer_timeout), - m_max_batch_size(batch_size), - m_dynamic_batch_size(batch_size) -{ - // Checking status for base class c'tor - if (HAILO_SUCCESS != status) { - return; - } - - status = HAILO_SUCCESS; -} - -VdmaInputStream::~VdmaInputStream() -{ - auto status = HAILO_UNINITIALIZED; - // We want to stop the vdma channel before closing the stream in the firmware - // because sending data to a closed stream may terminate the dma engine - if (this->is_stream_activated) { - status = VdmaInputStream::deactivate_stream(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to deactivate stream with error status {}", status); - } - } -} - -VdmaInputStream::VdmaInputStream(VdmaInputStream &&other) : - InputStreamBase(std::move(other)), - m_device(std::move(other.m_device)), - m_channel(std::move(other.m_channel)), - is_stream_activated(std::exchange(other.is_stream_activated, false)), - m_channel_timeout(std::move(other.m_channel_timeout)), - m_max_batch_size(other.m_max_batch_size), - m_dynamic_batch_size(other.m_dynamic_batch_size) -{} - -std::chrono::milliseconds VdmaInputStream::get_timeout() const -{ - return this->m_channel_timeout; -} - -hailo_status VdmaInputStream::set_timeout(std::chrono::milliseconds timeout) -{ - this->m_channel_timeout = timeout; - return HAILO_SUCCESS; -} - -hailo_status VdmaInputStream::abort() -{ - return m_channel->abort(); -} - -hailo_status VdmaInputStream::clear_abort() -{ - return m_channel->clear_abort(); -} - -hailo_status VdmaInputStream::flush() -{ - const auto dynamic_batch_size = (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == m_dynamic_batch_size) ? - 1 : m_dynamic_batch_size; - return m_channel->flush(m_channel_timeout * dynamic_batch_size); -} - -hailo_status VdmaInputStream::activate_stream(uint16_t dynamic_batch_size) -{ - auto status = set_dynamic_batch_size(dynamic_batch_size); - CHECK_SUCCESS(status); - - status = m_channel->complete_channel_activation(0); - CHECK_SUCCESS(status); - - this->is_stream_activated = true; - return HAILO_SUCCESS; -} - -hailo_status VdmaInputStream::deactivate_stream() -{ - if (!is_stream_activated) { - return HAILO_SUCCESS; - } - - // Flush is best effort - auto status = m_channel->flush(VDMA_FLUSH_TIMEOUT); - if (HAILO_STREAM_ABORTED_BY_USER == status) { - LOGGER__INFO("Flush input_channel is not needed because channel was aborted. (channel {})", m_channel->get_channel_id()); - status = HAILO_SUCCESS; - } else if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to flush input_channel. (status {} channel {})", status, m_channel->get_channel_id()); - } - - status = m_channel->stop_channel(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to stop channel with status {}", status); - } - - this->is_stream_activated = false; - return status; -} - -Expected VdmaInputStream::sync_write_raw_buffer(const MemoryView &buffer) -{ - hailo_status status = HAILO_UNINITIALIZED; - - status = m_channel->wait(buffer.size(), m_channel_timeout); - if ((status == HAILO_STREAM_ABORTED_BY_USER) || (status == HAILO_STREAM_NOT_ACTIVATED)) { - return make_unexpected(status); - } - CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_TIMEOUT, - "{} (H2D) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_channel_timeout.count()); - CHECK_SUCCESS_AS_EXPECTED(status); - - status = m_channel->transfer((void*)buffer.data(), buffer.size()); - if ((status == HAILO_STREAM_ABORTED_BY_USER) || (status == HAILO_STREAM_NOT_ACTIVATED)) { - return make_unexpected(status); - } - CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_TIMEOUT, - "{} (H2D) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_channel_timeout.count()); - CHECK_SUCCESS_AS_EXPECTED(status); - - return buffer.size(); -} - -hailo_status VdmaInputStream::write_buffer_only(const MemoryView &buffer, - const std::function &should_cancel) -{ - std::unique_lock lock(m_write_only_mutex); - return m_channel->write_buffer(buffer, m_channel_timeout, should_cancel); -} - -hailo_status VdmaInputStream::send_pending_buffer(size_t device_index) -{ - std::unique_lock lock(m_send_pending_mutex); - CHECK(0 == device_index, HAILO_INVALID_OPERATION); - hailo_status status = m_channel->wait(get_frame_size(), m_channel_timeout); - if ((HAILO_STREAM_ABORTED_BY_USER == status) || (HAILO_STREAM_NOT_ACTIVATED == status)) { - return status; - } - CHECK(HAILO_TIMEOUT != status, HAILO_TIMEOUT, - "{} (H2D) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_channel_timeout.count()); - CHECK_SUCCESS(status); - - return m_channel->send_pending_buffer(); -} - -uint16_t VdmaInputStream::get_dynamic_batch_size() const -{ - return std::max(m_dynamic_batch_size, static_cast(1)); -} - -const char* VdmaInputStream::get_dev_id() const -{ - return m_device->get_dev_id(); -} - -Expected VdmaInputStream::get_buffer_state() -{ - return m_channel->get_buffer_state(); -} - -hailo_status VdmaInputStream::sync_channel_state() -{ - return m_channel->sync_state(get_timeout()); -} - -Expected VdmaInputStream::get_buffer_frames_size() const -{ - return m_channel->get_transfers_count_in_buffer(m_stream_info.hw_frame_size); -} - -Expected VdmaInputStream::get_pending_frames_count() const -{ - return m_channel->get_h2d_pending_frames_count(); -} - -hailo_status VdmaInputStream::sync_write_all_raw_buffer_no_transform_impl(void *buffer, size_t offset, size_t size) -{ - ASSERT(NULL != buffer); - - return sync_write_raw_buffer(MemoryView(static_cast(buffer) + offset, size)).status(); -} - -hailo_status VdmaInputStream::set_dynamic_batch_size(uint16_t dynamic_batch_size) -{ - // TODO: use std::max in the configure stage - if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == m_max_batch_size) { - LOGGER__TRACE("max_batch_size is CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE; " - "Ignoring value of dynamic_batch_size {}", m_dynamic_batch_size); - return HAILO_SUCCESS; - } - - CHECK(dynamic_batch_size <= m_max_batch_size, HAILO_INVALID_ARGUMENT, - "Dynamic batch size ({}) must be <= than the configured batch size ({})", - dynamic_batch_size, m_max_batch_size); - - if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == dynamic_batch_size) { - LOGGER__TRACE("Received CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == dynamic_batch_size; " - "Leaving previously set value of {}", m_dynamic_batch_size); - } else { - LOGGER__TRACE("Setting stream's dynamic_batch_size to {}", dynamic_batch_size); - m_dynamic_batch_size = dynamic_batch_size; - - const auto status = m_channel->set_transfers_per_axi_intr(m_dynamic_batch_size); - CHECK_SUCCESS(status); - } - - return HAILO_SUCCESS; -} - -/** Output stream **/ -Expected> VdmaOutputStream::create(VdmaDevice &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event) -{ - switch (device.get_type()) { - case Device::Type::PCIE: - { - auto local_stream = PcieOutputStream::create(device, channel, edge_layer, batch_size, - network_group_activated_event); - CHECK_EXPECTED(local_stream); - return std::unique_ptr(local_stream.release()); - } - case Device::Type::CORE: - { - auto local_stream = CoreOutputStream::create(device, channel, edge_layer, batch_size, - network_group_activated_event); - CHECK_EXPECTED(local_stream); - return std::unique_ptr(local_stream.release()); - } - default: - assert(false); - LOGGER__ERROR("Invalid device type {}", static_cast(device.get_type())); - return make_unexpected(HAILO_INTERNAL_FAILURE); - } -} - -VdmaOutputStream::VdmaOutputStream(VdmaDevice &device, std::shared_ptr channel, - const LayerInfo &edge_layer, EventPtr network_group_activated_event, uint16_t batch_size, - std::chrono::milliseconds transfer_timeout, hailo_status &status) : - OutputStreamBase(edge_layer, std::move(network_group_activated_event), status), - m_device(&device), - m_channel(std::move(channel)), - is_stream_activated(false), - m_transfer_timeout(transfer_timeout), - m_max_batch_size(batch_size), - m_dynamic_batch_size(batch_size), - m_transfer_size(get_transfer_size(m_stream_info)) -{ - // Check status for base class c'tor - if (HAILO_SUCCESS != status) { - return; - } - - status = HAILO_SUCCESS; -} - -VdmaOutputStream::VdmaOutputStream(VdmaOutputStream &&other) : - OutputStreamBase(std::move(other)), - m_device(std::move(other.m_device)), - m_channel(std::move(other.m_channel)), - is_stream_activated(std::exchange(other.is_stream_activated, false)), - m_transfer_timeout(std::move(other.m_transfer_timeout)), - m_max_batch_size(other.m_max_batch_size), - m_dynamic_batch_size(other.m_dynamic_batch_size), - m_transfer_size(other.m_transfer_size) -{} - -VdmaOutputStream::~VdmaOutputStream() -{ - // We want to stop the vdma channel before closing the stream in the firmware - // because sending data to a closed stream may terminate the dma engine - auto status = HAILO_UNINITIALIZED; - - if (this->is_stream_activated) { - status = VdmaOutputStream::deactivate_stream(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to deactivate stream with error status {}", status); - } - } -} - -hailo_status VdmaOutputStream::set_timeout(std::chrono::milliseconds timeout) -{ - this->m_transfer_timeout = timeout; - return HAILO_SUCCESS; -} - -std::chrono::milliseconds VdmaOutputStream::get_timeout() const -{ - return this->m_transfer_timeout; -} - -hailo_status VdmaOutputStream::abort() -{ - return m_channel->abort(); -} - -hailo_status VdmaOutputStream::clear_abort() -{ - return m_channel->clear_abort(); -} - -uint16_t VdmaOutputStream::get_dynamic_batch_size() const -{ - return std::max(m_dynamic_batch_size, static_cast(1)); -} - -const char* VdmaOutputStream::get_dev_id() const -{ - return m_device->get_dev_id(); -} - -Expected VdmaOutputStream::get_buffer_state() -{ - return m_channel->get_buffer_state(); -} - -hailo_status VdmaOutputStream::activate_stream(uint16_t dynamic_batch_size) -{ - auto status = set_dynamic_batch_size(dynamic_batch_size); - CHECK_SUCCESS(status); - - status = m_channel->complete_channel_activation(m_transfer_size); - CHECK_SUCCESS(status); - - this->is_stream_activated = true; - - return HAILO_SUCCESS; -} - -hailo_status VdmaOutputStream::register_for_d2h_interrupts(const std::function &callback) -{ - return m_channel->register_for_d2h_interrupts(callback); -} - -hailo_status VdmaOutputStream::deactivate_stream() -{ - if (!is_stream_activated) { - return HAILO_SUCCESS; - } - - auto status = m_channel->stop_channel(); - if (HAILO_SUCCESS != status) { - LOGGER__ERROR("Failed to stop channel with status {}", status); - } - - this->is_stream_activated = false; - return HAILO_SUCCESS; -} - -Expected VdmaOutputStream::sync_read_raw_buffer(MemoryView &buffer) -{ - hailo_status status = HAILO_UNINITIALIZED; - - status = m_channel->wait(buffer.size(), m_transfer_timeout); - if ((status == HAILO_STREAM_ABORTED_BY_USER) || (status == HAILO_STREAM_NOT_ACTIVATED)) { - return make_unexpected(status); - } - CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_TIMEOUT, - "{} (D2H) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_transfer_timeout.count()); - CHECK_SUCCESS_AS_EXPECTED(status); - - status = m_channel->transfer(buffer.data(), buffer.size()); - if ((status == HAILO_STREAM_NOT_ACTIVATED) || (status == HAILO_STREAM_ABORTED_BY_USER)) { - return make_unexpected(status); - } - CHECK_AS_EXPECTED(HAILO_TIMEOUT != status, HAILO_TIMEOUT, - "{} (D2H) failed with status={} (timeout={}ms)", name(), HAILO_TIMEOUT, m_transfer_timeout.count()); - CHECK_SUCCESS_AS_EXPECTED(status); - - return buffer.size(); -} - -hailo_status VdmaOutputStream::read_all(MemoryView &buffer) -{ - std::unique_lock lock(m_read_mutex); - CHECK((buffer.size() % HailoRTCommon::HW_DATA_ALIGNMENT) == 0, HAILO_INVALID_ARGUMENT, - "Size must be aligned to {} (got {})", HailoRTCommon::HW_DATA_ALIGNMENT, buffer.size()); - - return sync_read_raw_buffer(buffer).status(); -} - -uint32_t VdmaOutputStream::get_transfer_size(const hailo_stream_info_t &stream_info) -{ - // The ppu outputs one bbox per vdma buffer in the case of nms - return (HAILO_FORMAT_ORDER_HAILO_NMS == stream_info.format.order) ? - stream_info.nms_info.bbox_size : stream_info.hw_frame_size; -} - -hailo_status VdmaOutputStream::set_dynamic_batch_size(uint16_t dynamic_batch_size) -{ - // TODO: use std::max in the configure stage - if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == m_max_batch_size) { - LOGGER__TRACE("max_batch_size is CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE; " - "Ignoring value of dynamic_batch_size {}", m_dynamic_batch_size); - return HAILO_SUCCESS; - } - - CHECK(dynamic_batch_size <= m_max_batch_size, HAILO_INVALID_ARGUMENT, - "Dynamic batch size ({}) must be <= than the configured batch size ({})", - dynamic_batch_size, m_max_batch_size); - - if (CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == dynamic_batch_size) { - LOGGER__TRACE("Received CONTROL_PROTOCOL__IGNORE_DYNAMIC_BATCH_SIZE == dynamic_batch_size; " - "Leaving previously set value of {}", m_dynamic_batch_size); - } else { - LOGGER__TRACE("Setting stream's dynamic_batch_size to {}", dynamic_batch_size); - m_dynamic_batch_size = dynamic_batch_size; - - const auto status = m_channel->set_transfers_per_axi_intr(m_dynamic_batch_size); - CHECK_SUCCESS(status); - } - - return HAILO_SUCCESS; -} - -Expected VdmaOutputStream::get_buffer_frames_size() const -{ - if (HAILO_FORMAT_ORDER_HAILO_NMS == m_stream_info.format.order) { - // In NMS, each output frame has different size depending on the number of bboxes found for each class - // and m_stream_info.hw_frame_size is the max frame size. To know the actual frame size and - // calculate the number of frames we need to read the content of the buffer (and finding the delimiter for each class in each frame). - LOGGER__INFO("NMS is not supported in function get_buffer_frames_size()"); - return make_unexpected(HAILO_NOT_AVAILABLE); - } - - return m_channel->get_transfers_count_in_buffer(m_stream_info.hw_frame_size); -} - -Expected VdmaOutputStream::get_pending_frames_count() const -{ - if (HAILO_FORMAT_ORDER_HAILO_NMS == m_stream_info.format.order) { - // In NMS, each output frame has different size depending on the number of bboxes found for each class - // and m_stream_info.hw_frame_size is the max frame size. To know the actual frame size and - // calculate the number of frames we need to read the content of the buffer (and finding the delimiter for each class in each frame). - LOGGER__INFO("NMS is not supported in function get_pending_frames_count()"); - return make_unexpected(HAILO_NOT_AVAILABLE); - } - - auto pending_descs_count = m_channel->get_d2h_pending_descs_count(); - CHECK_EXPECTED(pending_descs_count); - - auto channel_page_size = m_channel->get_page_size(); - uint32_t descs_per_frame = (0 == (m_stream_info.hw_frame_size % channel_page_size)) ? (m_stream_info.hw_frame_size / channel_page_size) : - ((m_stream_info.hw_frame_size / channel_page_size) + 1); - - return static_cast(std::floor(pending_descs_count.value() / descs_per_frame)); -} - -} /* namespace hailort */ diff --git a/hailort/libhailort/src/vdma_stream.hpp b/hailort/libhailort/src/vdma_stream.hpp deleted file mode 100644 index f39ed34..0000000 --- a/hailort/libhailort/src/vdma_stream.hpp +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved. - * Distributed under the MIT license (https://opensource.org/licenses/MIT) - **/ -/** - * @file vdma_stream.hpp - * @brief Stream object over vDMA channel - **/ - -#ifndef _HAILO_VDMA_STREAM_HPP_ -#define _HAILO_VDMA_STREAM_HPP_ - -#include "stream_internal.hpp" -#include "vdma_device.hpp" -#include "vdma_channel.hpp" -#include "hailo/hailort.h" -#include "hailo/expected.hpp" - -namespace hailort -{ -constexpr std::chrono::seconds VDMA_FLUSH_TIMEOUT(10); - -class VdmaInputStream : public InputStreamBase { -public: - static Expected> create(VdmaDevice &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event); - - VdmaInputStream(VdmaInputStream &&other); - virtual ~VdmaInputStream(); - - virtual std::chrono::milliseconds get_timeout() const override; - virtual hailo_status set_timeout(std::chrono::milliseconds timeout) override; - virtual hailo_status abort() override; - virtual hailo_status clear_abort() override; - virtual hailo_status flush() override; - hailo_status write_buffer_only(const MemoryView &buffer, const std::function &should_cancel = []() { return false; }); - hailo_status send_pending_buffer(size_t device_index = 0); - uint16_t get_dynamic_batch_size() const; - const char* get_dev_id() const; - Expected get_buffer_state(); - virtual Expected get_buffer_frames_size() const override; - virtual Expected get_pending_frames_count() const override; - - // To be used for debugging purposes - hailo_status sync_channel_state(); - - void notify_all() - { - return m_channel->notify_all(); - } - -protected: - VdmaInputStream(VdmaDevice &device, std::shared_ptr channel, const LayerInfo &edge_layer, - EventPtr network_group_activated_event, uint16_t batch_size, - std::chrono::milliseconds transfer_timeout, hailo_stream_interface_t stream_interface, - hailo_status &status); - - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; - virtual hailo_status deactivate_stream() override; - virtual Expected sync_write_raw_buffer(const MemoryView &buffer) override; - virtual hailo_status sync_write_all_raw_buffer_no_transform_impl(void *buffer, size_t offset, size_t size) override; - - VdmaDevice *m_device; - std::shared_ptr m_channel; - -private: - hailo_status set_dynamic_batch_size(uint16_t dynamic_batch_size); - - bool is_stream_activated; - std::chrono::milliseconds m_channel_timeout; - const uint16_t m_max_batch_size; - uint16_t m_dynamic_batch_size; - std::mutex m_write_only_mutex; - std::mutex m_send_pending_mutex; - - friend class InputVDeviceBaseStream; - friend class InputVDeviceNativeStream; -}; - -class VdmaOutputStream : public OutputStreamBase { -public: - static Expected> create(VdmaDevice &device, - std::shared_ptr channel, const LayerInfo &edge_layer, uint16_t batch_size, - EventPtr network_group_activated_event); - - VdmaOutputStream(VdmaOutputStream &&other); - virtual ~VdmaOutputStream(); - - virtual std::chrono::milliseconds get_timeout() const override; - virtual hailo_status set_timeout(std::chrono::milliseconds timeout) override; - virtual hailo_status abort() override; - virtual hailo_status clear_abort() override; - uint16_t get_dynamic_batch_size() const; - const char* get_dev_id() const; - Expected get_buffer_state(); - virtual Expected get_buffer_frames_size() const override; - virtual Expected get_pending_frames_count() const override; - - virtual hailo_status register_for_d2h_interrupts(const std::function &callback); - -protected: - VdmaOutputStream(VdmaDevice &device, std::shared_ptr channel, const LayerInfo &edge_layer, - EventPtr network_group_activated_event, uint16_t batch_size, - std::chrono::milliseconds transfer_timeout, hailo_status &status); - - virtual hailo_status activate_stream(uint16_t dynamic_batch_size) override; - virtual hailo_status deactivate_stream() override; - virtual Expected sync_read_raw_buffer(MemoryView &buffer); - - VdmaDevice *m_device; - std::shared_ptr m_channel; - -private: - hailo_status read_all(MemoryView &buffer) override; - static uint32_t get_transfer_size(const hailo_stream_info_t &stream_info); - hailo_status set_dynamic_batch_size(uint16_t dynamic_batch_size); - - bool is_stream_activated; - std::chrono::milliseconds m_transfer_timeout; - const uint16_t m_max_batch_size; - uint16_t m_dynamic_batch_size; - const uint32_t m_transfer_size; - std::mutex m_read_mutex; - - friend class OutputVDeviceBaseStream; -}; - - -} /* namespace hailort */ - -#endif /* _HAILO_VDMA_STREAM_HPP_ */ diff --git a/hailort/rpc/hailort_rpc.proto b/hailort/rpc/hailort_rpc.proto index ac76f2b..3df851f 100644 --- a/hailort/rpc/hailort_rpc.proto +++ b/hailort/rpc/hailort_rpc.proto @@ -5,12 +5,15 @@ option optimize_for = LITE_RUNTIME; service ProtoHailoRtRpc { rpc client_keep_alive (keepalive_Request) returns (empty) {} rpc get_service_version (get_service_version_Request) returns (get_service_version_Reply) {} + rpc VDevice_create (VDevice_create_Request) returns (VDevice_create_Reply) {} + rpc VDevice_dup_handle (dup_handle_Request) returns (dup_handle_Reply) {} rpc VDevice_release (Release_Request) returns (Release_Reply) {} rpc VDevice_configure (VDevice_configure_Request) returns (VDevice_configure_Reply) {} rpc VDevice_get_physical_devices_ids (VDevice_get_physical_devices_ids_Request) returns (VDevice_get_physical_devices_ids_Reply) {} rpc VDevice_get_default_streams_interface (VDevice_get_default_streams_interface_Request) returns (VDevice_get_default_streams_interface_Reply) {} + rpc ConfiguredNetworkGroup_dup_handle (dup_handle_Request) returns (dup_handle_Reply) {} rpc ConfiguredNetworkGroup_release (Release_Request) returns (Release_Reply) {} rpc ConfiguredNetworkGroup_make_input_vstream_params (ConfiguredNetworkGroup_make_input_vstream_params_Request) returns (ConfiguredNetworkGroup_make_input_vstream_params_Reply) {} rpc ConfiguredNetworkGroup_make_output_vstream_params (ConfiguredNetworkGroup_make_output_vstream_params_Request) returns (ConfiguredNetworkGroup_make_output_vstream_params_Reply) {} @@ -23,13 +26,17 @@ service ProtoHailoRtRpc { rpc ConfiguredNetworkGroup_get_input_vstream_infos (ConfiguredNetworkGroup_get_vstream_infos_Request) returns (ConfiguredNetworkGroup_get_vstream_infos_Reply) {} rpc ConfiguredNetworkGroup_get_output_vstream_infos (ConfiguredNetworkGroup_get_vstream_infos_Request) returns (ConfiguredNetworkGroup_get_vstream_infos_Reply) {} rpc ConfiguredNetworkGroup_get_all_vstream_infos (ConfiguredNetworkGroup_get_vstream_infos_Request) returns (ConfiguredNetworkGroup_get_vstream_infos_Reply) {} + rpc ConfiguredNetworkGroup_is_scheduled (ConfiguredNetworkGroup_is_scheduled_Request) returns (ConfiguredNetworkGroup_is_scheduled_Reply) {} rpc ConfiguredNetworkGroup_set_scheduler_timeout (ConfiguredNetworkGroup_set_scheduler_timeout_Request) returns (ConfiguredNetworkGroup_set_scheduler_timeout_Reply) {} rpc ConfiguredNetworkGroup_set_scheduler_threshold (ConfiguredNetworkGroup_set_scheduler_threshold_Request) returns (ConfiguredNetworkGroup_set_scheduler_threshold_Reply) {} + rpc ConfiguredNetworkGroup_set_scheduler_priority (ConfiguredNetworkGroup_set_scheduler_priority_Request) returns (ConfiguredNetworkGroup_set_scheduler_priority_Reply) {} rpc ConfiguredNetworkGroup_get_latency_measurement (ConfiguredNetworkGroup_get_latency_measurement_Request) returns (ConfiguredNetworkGroup_get_latency_measurement_Reply) {} rpc ConfiguredNetworkGroup_is_multi_context (ConfiguredNetworkGroup_is_multi_context_Request) returns (ConfiguredNetworkGroup_is_multi_context_Reply) {} rpc ConfiguredNetworkGroup_get_config_params(ConfiguredNetworkGroup_get_config_params_Request) returns (ConfiguredNetworkGroup_get_config_params_Reply) {} rpc InputVStreams_create (VStream_create_Request) returns (VStreams_create_Reply) {} + rpc InputVStream_dup_handle (dup_handle_Request) returns (dup_handle_Reply) {} + rpc OutputVStream_dup_handle (dup_handle_Request) returns (dup_handle_Reply) {} rpc InputVStream_release (Release_Request) returns (Release_Reply) {} rpc OutputVStreams_create (VStream_create_Request) returns (VStreams_create_Reply) {} rpc OutputVStream_release (Release_Request) returns (Release_Reply) {} @@ -55,7 +62,7 @@ service ProtoHailoRtRpc { message empty {} message keepalive_Request { - uint32 process_id = 1; + uint32 pid = 1; } message ProtoVDeviceParams { @@ -79,6 +86,15 @@ message get_service_version_Reply { ProtoHailoVersion hailo_version = 2; } +message dup_handle_Request { + uint32 pid = 1; + uint32 handle = 2; +} + +message dup_handle_Reply { + uint32 handle = 1; +} + message VDevice_create_Request { ProtoVDeviceParams hailo_vdevice_params = 1; uint32 pid = 2; @@ -175,6 +191,7 @@ message ProtoStreamInfo { message ProtoStreamsParams { uint32 stream_interface = 1; uint32 direction = 2; + uint32 flags = 3; } message ProtoNamedStreamParams { @@ -359,6 +376,15 @@ message ConfiguredNetworkGroup_get_vstream_infos_Reply { repeated ProtoVStreamInfo vstream_infos = 2; } +message ConfiguredNetworkGroup_is_scheduled_Request { + uint32 handle = 1; +} + +message ConfiguredNetworkGroup_is_scheduled_Reply { + uint32 status = 1; + bool is_scheduled = 2; +} + message ConfiguredNetworkGroup_set_scheduler_timeout_Request { uint32 handle = 1; uint32 timeout_ms = 2; @@ -379,6 +405,16 @@ message ConfiguredNetworkGroup_set_scheduler_threshold_Reply { uint32 status = 1; } +message ConfiguredNetworkGroup_set_scheduler_priority_Request { + uint32 handle = 1; + uint32 priority = 2; + string network_name = 3; +} + +message ConfiguredNetworkGroup_set_scheduler_priority_Reply { + uint32 status = 1; +} + message ConfiguredNetworkGroup_get_latency_measurement_Reply { uint32 status = 1; uint32 avg_hw_latency = 2; diff --git a/hailort/rpc/rpc_definitions.hpp b/hailort/rpc/rpc_definitions.hpp index 7d18326..0000c4c 100644 --- a/hailort/rpc/rpc_definitions.hpp +++ b/hailort/rpc/rpc_definitions.hpp @@ -13,10 +13,14 @@ namespace hailort { +#ifdef _WIN32 +static const std::string HAILORT_SERVICE_DEFAULT_ADDR = "127.0.0.1:50051"; +#else static const std::string HAILO_UDS_PREFIX = "unix://"; static const std::string HAILO_DEFAULT_SERVICE_ADDR = "/tmp/hailort_uds.sock"; -static const std::string HAILO_DEFAULT_UDS_ADDR = HAILO_UDS_PREFIX + HAILO_DEFAULT_SERVICE_ADDR; -static const uint32_t HAILO_KEEPALIVE_INTERVAL_SEC = 2; +static const std::string HAILORT_SERVICE_DEFAULT_ADDR = HAILO_UDS_PREFIX + HAILO_DEFAULT_SERVICE_ADDR; +#endif +static const std::chrono::seconds HAILO_KEEPALIVE_INTERVAL(2); } diff --git a/hailort/scripts/download_firmware_eth.cmd b/hailort/scripts/download_firmware_eth.cmd index 16e6333..043b312 100644 --- a/hailort/scripts/download_firmware_eth.cmd +++ b/hailort/scripts/download_firmware_eth.cmd @@ -2,7 +2,7 @@ @ECHO OFF set BASE_URI=https://hailo-hailort.s3.eu-west-2.amazonaws.com -set HRT_VERSION=4.12.1 +set HRT_VERSION=4.13.0 set FW_DIR=Hailo8/%HRT_VERSION%/FW set FW=hailo8_fw.%HRT_VERSION%_eth.bin diff --git a/hailort/scripts/download_firmware_eth.sh b/hailort/scripts/download_firmware_eth.sh index 2f145c8..70c8886 100755 --- a/hailort/scripts/download_firmware_eth.sh +++ b/hailort/scripts/download_firmware_eth.sh @@ -2,7 +2,7 @@ set -e readonly BASE_URI="https://hailo-hailort.s3.eu-west-2.amazonaws.com" -readonly HRT_VERSION=4.12.1 +readonly HRT_VERSION=4.13.0 readonly FW_AWS_DIR="Hailo8/${HRT_VERSION}/FW" readonly FW="hailo8_fw.${HRT_VERSION}_eth.bin" diff --git a/hailort/scripts/download_hefs.cmd b/hailort/scripts/download_hefs.cmd index 78ec220..332c9f1 100644 --- a/hailort/scripts/download_hefs.cmd +++ b/hailort/scripts/download_hefs.cmd @@ -1,7 +1,7 @@ :: cmd @ECHO OFF set BASE_URI=https://hailo-hailort.s3.eu-west-2.amazonaws.com -set HRT_VERSION=4.12.1 +set HRT_VERSION=4.13.0 set REMOTE_HEF_DIR=Hailo8/%HRT_VERSION%/HEFS set LOCAL_EXAMPLES_HEF_DIR=..\libhailort\examples\hefs set LOCAL_TUTORIALS_HEF_DIR=..\libhailort\bindings\python\platform\tutorials\hefs diff --git a/hailort/scripts/download_hefs.sh b/hailort/scripts/download_hefs.sh index 7325811..489367f 100755 --- a/hailort/scripts/download_hefs.sh +++ b/hailort/scripts/download_hefs.sh @@ -2,7 +2,7 @@ set -e readonly BASE_URI="https://hailo-hailort.s3.eu-west-2.amazonaws.com" -readonly HRT_VERSION=4.12.1 +readonly HRT_VERSION=4.13.0 readonly REMOTE_HEF_DIR="Hailo8/${HRT_VERSION}/HEFS" readonly LOCAL_EXAMPLES_HEF_DIR="../libhailort/examples/hefs" readonly LOCAL_TUTORIALS_HEF_DIR="../libhailort/bindings/python/platform/tutorials/hefs/" diff --git a/hailort/tools/hw_debug/CMakeLists.txt b/hailort/tools/hw_debug/CMakeLists.txt index 15217cc..5fd1d77 100644 --- a/hailort/tools/hw_debug/CMakeLists.txt +++ b/hailort/tools/hw_debug/CMakeLists.txt @@ -6,7 +6,7 @@ set(FILES readline_wrapper.cpp driver_memory.cpp memory_commands.cpp - mercury_fields.cpp + hailo15_fields.cpp # Depends on hailort_driver and its dependencies ${HAILO_OS_DIR}/hailort_driver.cpp diff --git a/hailort/tools/hw_debug/driver_memory.cpp b/hailort/tools/hw_debug/driver_memory.cpp index 7a1270e..83d3439 100644 --- a/hailort/tools/hw_debug/driver_memory.cpp +++ b/hailort/tools/hw_debug/driver_memory.cpp @@ -4,7 +4,7 @@ */ #include "driver_memory.hpp" -#include "mercury_fields.hpp" +#include "hailo15_fields.hpp" DriverMemorySource::DriverMemorySource(std::shared_ptr driver, HailoRTDriver::MemoryType memory_type) : m_driver(driver), @@ -29,6 +29,7 @@ size_t DriverMemorySource::total_size() const static constexpr size_t VDMA_CHANNELS_COUNT = 32; +static constexpr size_t VDMA_H2D_CHANNELS_COUNT = 16; #pragma pack(push, 1) struct VdmaDataPerDirection { @@ -57,8 +58,8 @@ struct VdmaDataPerDirection { static_assert(0x10 == sizeof(VdmaDataPerDirection), "Invalid VdmaDataPerDirection size"); struct VdmaChannelData { - VdmaDataPerDirection h2d; - VdmaDataPerDirection d2h; + VdmaDataPerDirection src; + VdmaDataPerDirection dest; }; #pragma pack(pop) @@ -82,12 +83,39 @@ public: throw std::runtime_error(fmt::format("Failed reading memory, status {}", status)); } - return fmt::format("channel[{}] (offset=0x{:X} size=0x{:X}):\n", index, index * sizeof(data), sizeof(data)) + - fmt::format(" host: {}\n", print_direction(data.h2d)) + - fmt::format(" device: {}\n", print_direction(data.d2h)); + return fmt::format("channel[{}] (offset=0x{:X} size=0x{:X} type= {}):\n", index, index * sizeof(data), sizeof(data), + index < VDMA_H2D_CHANNELS_COUNT ? "H2D" : "D2H") + + fmt::format(" Src status: {}\n", print_src_status(data.src)) + + fmt::format(" Dest status: {}\n", print_dest_status(data.dest)) + + fmt::format(" Src: {}\n", print_direction(data.src)) + + fmt::format(" Dest: {}\n", print_direction(data.dest)); } private: + static std::string print_src_status(const VdmaDataPerDirection &data) { + auto max_desc_mask = static_cast((1 << data.depth) - 1); + std::string status = + data.error ? "CHANNEL ERROR" : + !data.start_abort ? "ABORTED" : + data.pause_resume ? "PAUSED" : + (data.num_ongoing & max_desc_mask) != (data.num_processed & max_desc_mask) ? "DURING TRANSFER" : + (data.num_available & max_desc_mask) != (data.num_processed & max_desc_mask) ? "WAITING TO SEND" : + "IDLE"; + return status; + } + + static std::string print_dest_status(const VdmaDataPerDirection &data) { + auto max_desc_mask = static_cast((1 << data.depth) - 1); + std::string status = + data.error ? "CHANNEL ERROR" : + !data.start_abort ? "ABORTED" : + data.pause_resume ? "PAUSED" : + (data.num_ongoing & max_desc_mask) != (data.num_processed & max_desc_mask) ? "DURING TRANSFER" : + (data.num_available & max_desc_mask) != (data.num_processed & max_desc_mask) ? "WAITING TO RECEIVE" : + "IDLE"; + return status; + } + static std::string print_direction(const VdmaDataPerDirection &data) { return fmt::format( diff --git a/hailort/tools/hw_debug/mercury_fields.cpp b/hailort/tools/hw_debug/hailo15_fields.cpp similarity index 96% rename from hailort/tools/hw_debug/mercury_fields.cpp rename to hailort/tools/hw_debug/hailo15_fields.cpp index 07561e2..d4df9dd 100644 --- a/hailort/tools/hw_debug/mercury_fields.cpp +++ b/hailort/tools/hw_debug/hailo15_fields.cpp @@ -1,10 +1,10 @@ /** - * @file mercury_fields.cpp - * @brief Contains all memory fields related to mercury + * @file hailo15_fields.cpp + * @brief Contains all memory fields related to hailo15 */ -#include "mercury_fields.hpp" -#include "hw_consts/mercury/dram_dma_engine_config_regs.h" +#include "hailo15_fields.hpp" +#include "hw_consts/hailo15/dram_dma_engine_config_regs.h" // Implement our own offsetof to allow access to array #define my_offsetof(type,field) ((size_t)(&(((type*)(0))->field))) diff --git a/hailort/tools/hw_debug/mercury_fields.hpp b/hailort/tools/hw_debug/hailo15_fields.hpp similarity index 90% rename from hailort/tools/hw_debug/mercury_fields.hpp rename to hailort/tools/hw_debug/hailo15_fields.hpp index 29709d7..a159f01 100644 --- a/hailort/tools/hw_debug/mercury_fields.hpp +++ b/hailort/tools/hw_debug/hailo15_fields.hpp @@ -1,10 +1,10 @@ /** - * @file mercury_fields.hpp - * @brief Contains all memory fields related to mercury + * @file hailo15_fields.hpp + * @brief Contains all memory fields related to hailo15 */ -#ifndef _HW_DEBUG_MERCURY_FIELDS_H_ -#define _HW_DEBUG_MERCURY_FIELDS_H_ +#ifndef _HW_DEBUG_HAILO15_FIELDS_H_ +#define _HW_DEBUG_HAILO15_FIELDS_H_ #include "memory_commands.hpp" @@ -63,4 +63,4 @@ private: std::string mode(MemorySource &memory, size_t index) const; }; -#endif /* _HW_DEBUG_MERCURY_FIELDS_H_ */ +#endif /* _HW_DEBUG_HAILO15_FIELDS_H_ */ diff --git a/hailort/tools/hw_debug/hw_consts/mercury/dram_dma_engine_config_macros.h b/hailort/tools/hw_debug/hw_consts/hailo15/dram_dma_engine_config_macros.h similarity index 100% rename from hailort/tools/hw_debug/hw_consts/mercury/dram_dma_engine_config_macros.h rename to hailort/tools/hw_debug/hw_consts/hailo15/dram_dma_engine_config_macros.h diff --git a/hailort/tools/hw_debug/hw_consts/mercury/dram_dma_engine_config_regs.h b/hailort/tools/hw_debug/hw_consts/hailo15/dram_dma_engine_config_regs.h similarity index 100% rename from hailort/tools/hw_debug/hw_consts/mercury/dram_dma_engine_config_regs.h rename to hailort/tools/hw_debug/hw_consts/hailo15/dram_dma_engine_config_regs.h diff --git a/hailort/tools/hw_debug/hw_consts/mercury/dram_dma_package_macros.h b/hailort/tools/hw_debug/hw_consts/hailo15/dram_dma_package_macros.h similarity index 100% rename from hailort/tools/hw_debug/hw_consts/mercury/dram_dma_package_macros.h rename to hailort/tools/hw_debug/hw_consts/hailo15/dram_dma_package_macros.h -- 2.34.1