From 92389d6931cab8b115385e1291bab86b9b571f86 Mon Sep 17 00:00:00 2001 From: Jaeyun Jung Date: Mon, 13 Nov 2023 14:31:40 +0900 Subject: [PATCH] [Edge] util to handle tensor info Code clean, fix indent and use util function if current caps is tensor stream. Signed-off-by: Jaeyun Jung --- gst/edge/edge_sink.c | 37 +++++++++++++++++++++++++++++++------ gst/edge/edge_sink.h | 3 ++- gst/edge/edge_src.c | 51 +++++++++++++++++++++++++++++++++++++++++---------- gst/edge/edge_src.h | 3 ++- gst/edge/meson.build | 2 +- gst/meson.build | 9 ++++----- 6 files changed, 81 insertions(+), 24 deletions(-) diff --git a/gst/edge/edge_sink.c b/gst/edge/edge_sink.c index 7bf77c2..eae004f 100644 --- a/gst/edge/edge_sink.c +++ b/gst/edge/edge_sink.c @@ -363,6 +363,9 @@ static GstFlowReturn gst_edgesink_render (GstBaseSink * basesink, GstBuffer * buffer) { GstEdgeSink *self = GST_EDGESINK (basesink); + GstCaps *caps; + GstStructure *structure; + gboolean is_tensor; nns_edge_data_h data_h; guint i, num_mems; int ret; @@ -375,19 +378,40 @@ gst_edgesink_render (GstBaseSink * basesink, GstBuffer * buffer) return GST_FLOW_ERROR; } - num_mems = gst_buffer_n_memory (buffer); + caps = gst_pad_get_current_caps (GST_BASE_SINK_PAD (basesink)); + structure = gst_caps_get_structure (caps, 0); + is_tensor = gst_structure_is_tensor_stream (structure); + gst_caps_unref (caps); + + if (is_tensor) + num_mems = gst_tensor_buffer_get_count (buffer); + else + num_mems = gst_buffer_n_memory (buffer); + for (i = 0; i < num_mems; i++) { - mem[i] = gst_buffer_peek_memory (buffer, i); + if (is_tensor) + mem[i] = gst_tensor_buffer_get_nth_memory (buffer, i); + else + mem[i] = gst_buffer_get_memory (buffer, i); + if (!gst_memory_map (mem[i], &map[i], GST_MAP_READ)) { - nns_loge ("Cannot map the %uth memory in gst-buffer", i); + nns_loge ("Cannot map the %uth memory in gst-buffer.", i); + gst_memory_unref (mem[i]); num_mems = i; goto done; } - nns_edge_data_add (data_h, map[i].data, map[i].size, NULL); + + ret = nns_edge_data_add (data_h, map[i].data, map[i].size, NULL); + if (ret != NNS_EDGE_ERROR_NONE) { + nns_loge ("Failed to append %u-th memory into edge data.", i); + num_mems = i + 1; + goto done; + } } - nns_edge_send (self->edge_h, data_h); - goto done; + ret = nns_edge_send (self->edge_h, data_h); + if (ret != NNS_EDGE_ERROR_NONE) + nns_loge ("Failed to send edge data, connection lost or internal error."); done: if (data_h) @@ -395,6 +419,7 @@ done: for (i = 0; i < num_mems; i++) { gst_memory_unmap (mem[i], &map[i]); + gst_memory_unref (mem[i]); } return GST_FLOW_OK; diff --git a/gst/edge/edge_sink.h b/gst/edge/edge_sink.h index babe9c6..953b793 100644 --- a/gst/edge/edge_sink.h +++ b/gst/edge/edge_sink.h @@ -17,7 +17,8 @@ #include #include "edge_common.h" #include -#include "../nnstreamer/nnstreamer_log.h" +#include "nnstreamer_log.h" +#include "nnstreamer_plugin_api.h" #include "tensor_typedef.h" G_BEGIN_DECLS diff --git a/gst/edge/edge_src.c b/gst/edge/edge_src.c index 24c9e0c..24b2b7c 100644 --- a/gst/edge/edge_src.c +++ b/gst/edge/edge_src.c @@ -65,8 +65,8 @@ static void gst_edgesrc_set_dest_port (GstEdgeSrc * self, static nns_edge_connect_type_e gst_edgesrc_get_connect_type (GstEdgeSrc * self); static void gst_edgesrc_set_connect_type (GstEdgeSrc * self, const nns_edge_connect_type_e connect_type); -static GstStateChangeReturn gst_edgesrc_change_state (GstElement * - element, GstStateChange transition); +static GstStateChangeReturn gst_edgesrc_change_state (GstElement * element, + GstStateChange transition); /** * @brief initialize the class @@ -248,7 +248,6 @@ gst_edgesrc_class_finalize (GObject * object) G_OBJECT_CLASS (parent_class)->finalize (object); } - /** * @brief Change state of edgesrc. */ @@ -268,16 +267,16 @@ gst_edgesrc_change_state (GstElement * element, GstStateChange transition) } ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + switch (transition) { case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - { GST_INFO_OBJECT (self, "State changed from PLAYING to PAUSED."); self->playing = FALSE; break; - } default: break; } + return ret; } @@ -380,11 +379,18 @@ gst_edgesrc_create (GstBaseSrc * basesrc, guint64 offset, guint size, GstEdgeSrc *self = GST_EDGESRC (basesrc); nns_edge_data_h data_h = NULL; GstBuffer *buffer = NULL; - guint i, num_data; + GstMemory *mem; + GstCaps *caps; + GstStructure *structure; + GstTensorsConfig config; + GstTensorInfo *_info; + gboolean is_tensor; + guint i, num_data, max_mems; int ret; UNUSED (offset); UNUSED (size); + gst_tensors_config_init (&config); while (self->playing && !data_h) { data_h = g_async_queue_timeout_pop (self->msg_queue, G_USEC_PER_SEC); @@ -401,6 +407,24 @@ gst_edgesrc_create (GstBaseSrc * basesrc, guint64 offset, guint size, goto done; } + /* Check current caps and max memory. */ + caps = gst_pad_get_current_caps (GST_BASE_SRC_PAD (basesrc)); + structure = gst_caps_get_structure (caps, 0); + is_tensor = gst_structure_is_tensor_stream (structure); + + if (is_tensor) + gst_tensors_config_from_structure (&config, structure); + + gst_caps_unref (caps); + + max_mems = is_tensor ? NNS_TENSOR_SIZE_LIMIT : gst_buffer_get_max_memory (); + if (num_data > max_mems) { + nns_loge + ("Cannot create new buffer. The edge-data has %u memories, but allowed memories is %u.", + num_data, max_mems); + goto done; + } + buffer = gst_buffer_new (); for (i = 0; i < num_data; i++) { void *data = NULL; @@ -409,16 +433,23 @@ gst_edgesrc_create (GstBaseSrc * basesrc, guint64 offset, guint size, nns_edge_data_get (data_h, i, &data, &data_len); new_data = _g_memdup (data, data_len); - - gst_buffer_append_memory (buffer, - gst_memory_new_wrapped (0, new_data, data_len, 0, data_len, new_data, - g_free)); + mem = gst_memory_new_wrapped (0, new_data, data_len, 0, data_len, + new_data, g_free); + + if (is_tensor) { + _info = gst_tensors_info_get_nth_info (&config.info, i); + gst_tensor_buffer_append_memory (buffer, mem, _info); + } else { + gst_buffer_append_memory (buffer, mem); + } } done: if (data_h) nns_edge_data_destroy (data_h); + gst_tensors_config_free (&config); + if (buffer == NULL) { nns_loge ("Failed to get buffer to push to the edgesrc."); return GST_FLOW_ERROR; diff --git a/gst/edge/edge_src.h b/gst/edge/edge_src.h index 40671bc..a93857a 100644 --- a/gst/edge/edge_src.h +++ b/gst/edge/edge_src.h @@ -17,8 +17,9 @@ #include #include "edge_common.h" #include +#include "nnstreamer_log.h" +#include "nnstreamer_plugin_api.h" #include "nnstreamer_util.h" -#include "../nnstreamer/nnstreamer_log.h" G_BEGIN_DECLS #define GST_TYPE_EDGESRC \ diff --git a/gst/edge/meson.build b/gst/edge/meson.build index b56a994..38cee04 100644 --- a/gst/edge/meson.build +++ b/gst/edge/meson.build @@ -9,6 +9,7 @@ edge_deps = [ glib_dep, gst_base_dep, gst_dep, + nnstreamer_dep, nnstreamer_edge_dep, ] @@ -22,7 +23,6 @@ gstedge_shared = shared_library('gstedge', edge_srcs, dependencies: edge_deps, install: true, - include_directories: include_directories('../nnstreamer/include'), install_dir: plugins_install_dir ) diff --git a/gst/meson.build b/gst/meson.build index 6e4edc8..24a2fc4 100644 --- a/gst/meson.build +++ b/gst/meson.build @@ -1,12 +1,11 @@ -if nnstreamer_edge_support_is_available - subdir('edge') -endif subdir('join') +subdir('nnstreamer') if mqtt_support_is_available subdir('mqtt') endif -subdir('nnstreamer') - +if nnstreamer_edge_support_is_available + subdir('edge') +endif if datarepo_support_is_available subdir('datarepo') endif -- 2.7.4