From d325462e1f1a731379a436fe2f655af36b513e8c Mon Sep 17 00:00:00 2001 From: gichan-jang Date: Mon, 18 May 2020 10:37:16 +0900 Subject: [PATCH] [Converter] Prepare to register sub-plugin Preprare to register sub-plugin for flatbuffers and protobuf Signed-off-by: gichan-jang --- .../include/nnstreamer_plugin_api_converter.h | 60 ++++---- gst/nnstreamer/include/tensor_typedef.h | 8 ++ gst/nnstreamer/nnstreamer_conf.c | 2 +- gst/nnstreamer/nnstreamer_subplugin.c | 2 +- gst/nnstreamer/tensor_converter/tensor_converter.c | 158 +++++++++++++++------ gst/nnstreamer/tensor_converter/tensor_converter.h | 5 +- meson.build | 1 + 7 files changed, 154 insertions(+), 82 deletions(-) diff --git a/gst/nnstreamer/include/nnstreamer_plugin_api_converter.h b/gst/nnstreamer/include/nnstreamer_plugin_api_converter.h index ba5a698..5358828 100644 --- a/gst/nnstreamer/include/nnstreamer_plugin_api_converter.h +++ b/gst/nnstreamer/include/nnstreamer_plugin_api_converter.h @@ -38,54 +38,50 @@ extern "C" { /** * @brief Converter's subplugin implementation. */ -struct _NNStreamerExternalConverter { +typedef struct _NNStreamerExternalConverter +{ const char *media_type_name; /** 1. chain func, data handling. */ - GstBuffer * (*convert) (GstTensorConverter * self, - const GstBuffer * buf, gsize * frame_size, guint * frames_in); - /**< Convert the given input stream to tensor/tensors stream. - * @param[in/out] self A pointer designating "this". - * @param[in] buf The input stream buffer - * @param[out] frame_size The size of each frame (output buffer) - * @param[out] frames_in The number of frames in the given input buffer. - * @retval Return inbuf if the data is to be kept untouched. - * @retval Retrun a new GstBuf if the data is to be modified. - */ + GstBuffer *(*convert) (GstBuffer * buf, gsize * frame_size, + guint * frames_in, GstTensorsConfig * config); + /**< Convert the given input stream to tensor/tensors stream. + * @param[in] buf The input stream buffer + * @param[out] frame_size The size of each frame (output buffer) + * @param[out] frames_in The number of frames in the given input buffer. + * @param[out] config tensors config structure to be filled + * @retval Return inbuf if the data is to be kept untouched. + * @retval Retrun a new GstBuf if the data is to be modified. + */ - /** 2. parse_caps (type conv, input(media) to output(tensor)) */ - gboolean (*get_caps) (GstTensorConverter * self, const GstStructure * st, - GstTensorConfig * config); - /**< Set the tensor config structure from the given stream frame - * @param[in/out] self A pointer designating "this". - * @param[in] st The input (original/media data) stream's metadata - * @param[out] config The output (tensor/tensors) emtadata - */ - /** 3. query_cap (tpye conf, output(tensor) to input(media)) */ - gboolean (*query_caps) (GstTensorConverter * self, - const GstTensorConfig *config, GstStructure *st); - /**< Filters (narrows down) the GstCap (st) with the given config. - * @param[in/out] self A pointer designating "this". - * @param[in] config The config of output tensor/tensors - * @param[in/out] st The gstcap of input to be filtered with config. - */ -}; + /** 2. get_caps (type conv, input(media) to output(tensor)) */ + gboolean (*get_caps) (const GstStructure * st, GstTensorConfig * config); + /**< Set the tensor config structure from the given stream frame + * @param[in] st The input (original/media data) stream's metadata + * @param[out] config The output (tensor/tensors) metadata + */ + /** 3. query_cap (tpye conf, output(tensor) to input(media)) */ + GstCaps * (*query_caps) (const GstTensorConfig * config, + GstStructure * st); + /**< Filters (narrows down) the GstCap (st) with the given config. + * @param[in] config The config of output tensor/tensors + * @param[in/out] st The gstcap of input to be filtered with config. + */ + } NNStreamerExternalConverter; /** * @brief Converter's sub-plugin should call this function to register itself. * @param[in] ex Converter sub-plugin to be registered. * @return TRUE if registered. FALSE is failed or duplicated. */ -extern int -registerExternalConverter (NNStreamerExternalConverter *ex); +extern int registerExternalConverter (NNStreamerExternalConverter * ex); /** * @brief Converter's sub-plugin may call this to unregister itself. * @param[in] prefix The name of converter sub-plugin. */ -extern void -unregisterExternalConverter (const char *prefix); +extern void unregisterExternalConverter (const char *prefix); #ifdef __cplusplus } diff --git a/gst/nnstreamer/include/tensor_typedef.h b/gst/nnstreamer/include/tensor_typedef.h index 96bab2c..44d55b8 100644 --- a/gst/nnstreamer/include/tensor_typedef.h +++ b/gst/nnstreamer/include/tensor_typedef.h @@ -87,6 +87,14 @@ #define GST_PROTOBUF_TENSOR_CAP_DEFAULT \ "other/protobuf-tensor, " \ "framerate = " GST_TENSOR_RATE_RANGE +/** + * @brief Default static capibility for flatbuffers + * Flatbuf converter will convert this capability to other/tensor(s) + * @todo Move this definition to proper header file + */ +#define GST_FLATBUF_TENSOR_CAP_DEFAULT \ + "other/flatbuf-tensor, " \ + "framerate = " GST_TENSOR_RATE_RANGE /** * @brief Possible data element types of other/tensor. diff --git a/gst/nnstreamer/nnstreamer_conf.c b/gst/nnstreamer/nnstreamer_conf.c index 48dc022..076eeb2 100644 --- a/gst/nnstreamer/nnstreamer_conf.c +++ b/gst/nnstreamer/nnstreamer_conf.c @@ -37,7 +37,7 @@ #define NNSTREAMER_PREFIX_DECODER "libnnstreamer_decoder_" #define NNSTREAMER_PREFIX_FILTER "libnnstreamer_filter_" #define NNSTREAMER_PREFIX_CUSTOMFILTERS "" -#define NNSTREAMER_PREFIX_CONVERTER "" +#define NNSTREAMER_PREFIX_CONVERTER "libnnstreamer_converter_" /* Custom filter does not have prefix */ /* Env-var names */ diff --git a/gst/nnstreamer/nnstreamer_subplugin.c b/gst/nnstreamer/nnstreamer_subplugin.c index 4d8a068..54c2ae0 100644 --- a/gst/nnstreamer/nnstreamer_subplugin.c +++ b/gst/nnstreamer/nnstreamer_subplugin.c @@ -67,7 +67,7 @@ static subpluginSearchLogic searchAlgorithm[] = { [NNS_SUBPLUGIN_FILTER] = NNS_SEARCH_FILENAME, [NNS_SUBPLUGIN_DECODER] = NNS_SEARCH_FILENAME, [NNS_EASY_CUSTOM_FILTER] = NNS_SEARCH_FILENAME, - [NNS_SUBPLUGIN_CONVERTER] = NNS_SEARCH_GETALL, + [NNS_SUBPLUGIN_CONVERTER] = NNS_SEARCH_FILENAME, [NNS_SUBPLUGIN_END] = NNS_SEARCH_NO_OP, }; diff --git a/gst/nnstreamer/tensor_converter/tensor_converter.c b/gst/nnstreamer/tensor_converter/tensor_converter.c index 4e58ec2..3779c38 100644 --- a/gst/nnstreamer/tensor_converter/tensor_converter.c +++ b/gst/nnstreamer/tensor_converter/tensor_converter.c @@ -60,7 +60,6 @@ #endif #include #include -#include /** * @brief Caps string for text input @@ -180,6 +179,7 @@ static gboolean gst_tensor_converter_parse_caps (GstTensorConverter * self, static const NNStreamerExternalConverter *findExternalConverter (const char *media_type_name); +static const gchar *getExternalConverterName (const char *name); /** * @brief Initialize the tensor_converter's class. @@ -261,7 +261,9 @@ gst_tensor_converter_class_init (GstTensorConverterClass * klass) DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /* set src pad template */ - pad_caps = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT); + pad_caps = + gst_caps_from_string (GST_TENSOR_CAP_DEFAULT "; " + GST_TENSORS_CAP_DEFAULT); pad_template = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, pad_caps); @@ -270,13 +272,7 @@ gst_tensor_converter_class_init (GstTensorConverterClass * klass) gst_caps_unref (pad_caps); /* set sink pad template */ - pad_caps = gst_caps_new_empty (); - - /* append caps string for all media types */ - append_video_caps_template (pad_caps); - append_audio_caps_template (pad_caps); - append_text_caps_template (pad_caps); - append_octet_caps_template (pad_caps); + pad_caps = gst_caps_new_any (); pad_template = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, pad_caps); @@ -329,6 +325,7 @@ gst_tensor_converter_init (GstTensorConverter * self) self->in_media_type = _NNS_MEDIA_INVALID; self->frame_size = 0; self->remove_padding = FALSE; + self->externalConverter = NULL; gst_tensor_info_init (&self->tensor_info); self->adapter = gst_adapter_new (); @@ -686,6 +683,7 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) { GstTensorConverter *self; GstTensorConfig *config; + GstTensorsConfig tensors_config; GstAdapter *adapter; GstBuffer *inbuf; gsize avail, buf_size, frame_size, out_size; @@ -693,6 +691,7 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) GstFlowReturn ret = GST_FLOW_OK; GstClockTime pts, dts, duration; gboolean have_framerate; + GstCaps *curr_caps, *peer_caps, *out_caps = NULL; buf_size = gst_buffer_get_size (buf); g_return_val_if_fail (buf_size > 0, GST_FLOW_ERROR); @@ -827,12 +826,45 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) if (self->externalConverter == NULL || self->externalConverter->convert == NULL) return GST_FLOW_NOT_SUPPORTED; - inbuf = self->externalConverter->convert (self, buf, &frame_size, - &frames_in); + inbuf = + self->externalConverter->convert (buf, &frame_size, &frames_in, + &tensors_config); + + self->tensors_config = tensors_config; + if (tensors_config.info.num_tensors == 1) { + GstStructure *st; + peer_caps = gst_pad_peer_query_caps (self->srcpad, NULL); + silent_debug_caps (peer_caps, "peer caps"); + + st = gst_caps_get_structure (peer_caps, 0); + + if (g_strcmp0 (gst_structure_get_name (st), "other/tensor") == 0) { + self->tensor_config.info = tensors_config.info.info[0]; + self->tensor_config.rate_n = tensors_config.rate_n; + self->tensor_config.rate_d = tensors_config.rate_d; + out_caps = gst_tensor_caps_from_config (&self->tensor_config); + } + } + + /* caps for tensors */ + if (out_caps == NULL) + out_caps = gst_tensors_caps_from_config (&self->tensors_config); + + silent_debug_caps (out_caps, "out-caps"); + + /* Update source pad caps. If it is different */ + curr_caps = gst_pad_get_current_caps (self->srcpad); + if (gst_caps_is_equal (curr_caps, out_caps) != TRUE) { + gst_pad_set_caps (self->srcpad, out_caps); + } g_assert (inbuf != NULL); g_assert (frame_size > 0); - g_assert ((buf_size % frame_size) == 0); + + gst_caps_unref (curr_caps); + gst_caps_unref (out_caps); + gst_buffer_unref (buf); + break; } default: @@ -1251,6 +1283,48 @@ gst_tensor_converter_parse_octet (GstTensorConverter * self, } /** + * @brief Get sink pad template caps + */ +static GstCaps * +gst_tensor_converter_get_template_caps (GstTensorConverter * self) +{ + GstCaps *caps = gst_caps_new_empty (); + const NNStreamerExternalConverter *ex; + gchar *plugin_name; + guint total, i; + subplugin_info_s info; + const gchar *prefix_str; + gsize prefix_len, extension_len, len; + + /* append default caps */ + append_video_caps_template (caps); + append_audio_caps_template (caps); + append_text_caps_template (caps); + append_octet_caps_template (caps); + + /* get template caps from sub-plugins */ + total = nnsconf_get_subplugin_info (NNSCONF_PATH_CONVERTERS, &info); + if (total > 0) { + prefix_str = nnsconf_get_subplugin_name_prefix (NNSCONF_PATH_CONVERTERS); + prefix_len = strlen (prefix_str); + extension_len = strlen (NNSTREAMER_SO_FILE_EXTENSION); + + for (i = 0; i < total; i++) { + len = strlen (info.names[i]) - prefix_len - extension_len; + plugin_name = g_strndup (info.names[i] + prefix_len, len); + + ex = findExternalConverter (plugin_name); + + if (ex && ex->query_caps) { + gst_caps_append (caps, ex->query_caps (NULL, NULL)); + } + g_free (plugin_name); + } + } + return caps; +} + +/** * @brief Get possible media-caps from downstream element. */ static GstCaps * @@ -1259,6 +1333,8 @@ gst_tensor_converter_get_possible_media_caps (GstTensorConverter * self) GstCaps *media_caps = NULL; GstCaps *peer_caps; + media_caps = gst_tensor_converter_get_template_caps (self); + /* get possible caps from downstream element */ peer_caps = gst_pad_peer_query_caps (self->srcpad, NULL); @@ -1276,8 +1352,6 @@ gst_tensor_converter_get_possible_media_caps (GstTensorConverter * self) gst_tensor_config_from_structure (&config, st); /* convert peer caps to possible media caps */ - media_caps = gst_pad_get_pad_template_caps (self->sinkpad); - media_caps = gst_caps_make_writable (media_caps); caps_len = gst_caps_get_size (media_caps); for (i = 0; i < caps_len; ++i) { @@ -1382,32 +1456,6 @@ gst_tensor_converter_get_possible_media_caps (GstTensorConverter * self) } } break; - case _NNS_MEDIA_INVALID: /* this could be MEDIA_PLUGIN */ - { - const gchar *name = gst_structure_get_name (st); - const NNStreamerExternalConverter *ex; - - if (name == NULL) - break; - ex = findExternalConverter (name); - if (ex == NULL) - break; - - /** @todo What if this is inconsistent with self->ex? */ - - if (NULL == ex->query_caps) { - ml_logf - ("The given conveter subplugin, \"%s\", does not have a valid query_caps callback.\n", - name); - break; - } - if (ex->query_caps (self, &config, st) == FALSE) { - GST_ERROR_OBJECT (self, - "Failed to filter GstCap structure with the given config"); - } - - break; - } default: /* do nothing for text and octet stream */ break; @@ -1484,7 +1532,6 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self, structure = gst_caps_get_structure (caps, 0); in_type = gst_tensor_media_type_from_structure (structure); - self->externalConverter = NULL; switch (in_type) { case _NNS_VIDEO: @@ -1597,17 +1644,24 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self, default: { /* if found, configure in_mdeia_type = _NNS_MEDIA_PLUGINS */ - const gchar *name = gst_structure_get_name (structure); - if (name != NULL) { - const NNStreamerExternalConverter *ex = findExternalConverter (name); + const gchar *struct_name, *ext_conv_name; + + struct_name = gst_structure_get_name (structure); + + if (struct_name != NULL) { + const NNStreamerExternalConverter *ex; + + ext_conv_name = getExternalConverterName (struct_name); + ex = findExternalConverter (ext_conv_name); + if (ex != NULL) { in_type = _NNS_MEDIA_PLUGINS; self->externalConverter = ex; - if (NULL == ex->get_caps || !ex->get_caps (self, structure, &config)) { + if (NULL == ex->get_caps || !ex->get_caps (structure, &config)) { GST_ERROR_OBJECT (self, "Failed to get tensor info from %s. Check the given options.", - name); + ext_conv_name); ml_loge ("Please set the options property correctly.\n"); self->externalConverter = NULL; return FALSE; @@ -1646,6 +1700,18 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self, } /** + * @brief Get converter's subplugins name + */ +static const gchar * +getExternalConverterName (const char *name) +{ + if (g_strcmp0 (name, "other/flatbuf-tensor") == 0) { + return "flatbuf"; + } + return "invalid_media_type"; +} + +/** * @brief Converter's external subplugins should call this at init. */ int diff --git a/gst/nnstreamer/tensor_converter/tensor_converter.h b/gst/nnstreamer/tensor_converter/tensor_converter.h index 832809b..343b2bc 100644 --- a/gst/nnstreamer/tensor_converter/tensor_converter.h +++ b/gst/nnstreamer/tensor_converter/tensor_converter.h @@ -37,6 +37,7 @@ #include #include #include +#include G_BEGIN_DECLS @@ -53,7 +54,6 @@ G_BEGIN_DECLS typedef struct _GstTensorConverter GstTensorConverter; typedef struct _GstTensorConverterClass GstTensorConverterClass; -typedef struct _NNStreamerExternalConverter NNStreamerExternalConverter; /** * @brief Internal data structure for tensor_converter instances. @@ -73,13 +73,14 @@ struct _GstTensorConverter GstAdapter *adapter; /**< adapt incoming media stream */ media_type in_media_type; /**< incoming media type */ + /** ExternalConverter is used if in_media_type == _NNS_MEDIA_PLUGINS */ const NNStreamerExternalConverter *externalConverter; - /**< used if in_media_type == _NNS_MEDIA_PLUGINS */ gsize frame_size; /**< size of one frame */ gboolean remove_padding; /**< If true, zero-padding must be removed */ gboolean tensor_configured; /**< True if already successfully configured tensor metadata */ GstTensorConfig tensor_config; /**< output tensor info */ + GstTensorsConfig tensors_config; /**< output tensors info */ gboolean have_segment; /**< True if received segment */ gboolean need_segment; /**< True to handle seg event */ diff --git a/meson.build b/meson.build index b47c476..7af2ecb 100644 --- a/meson.build +++ b/meson.build @@ -92,6 +92,7 @@ subplugin_install_prefix = join_paths(nnstreamer_prefix, 'lib', 'nnstreamer') filter_subplugin_install_dir = join_paths(subplugin_install_prefix, 'filters') decoder_subplugin_install_dir = join_paths(subplugin_install_prefix, 'decoders') customfilter_install_dir = join_paths(subplugin_install_prefix, 'customfilters') +converter_subplugin_install_dir = join_paths(subplugin_install_prefix, 'converters') unittest_install_dir = join_paths(subplugin_install_prefix, 'unittest') # Set default configuration -- 2.7.4