From ea7195e8b0409b63047765b3c9feeed8bf0b2def Mon Sep 17 00:00:00 2001 From: "jy1210.jung" Date: Fri, 24 Aug 2018 14:01:17 +0900 Subject: [PATCH] [Convert/Decode] support text type (text/x-raw) 1. add code to support string (text/x-raw,format=utf8) 2. update transform buffer size (passes received size) 3. change code to get raw caps from string 4. add simple stream test using tensor sink related issue : #305 Signed-off-by: Jaeyun Jung --- CMakeLists.txt | 1 + common/tensor_common.c | 259 ++++++++++++++++++++++++-------- gst/tensor_converter/tensor_converter.c | 144 ++++++++++++++---- gst/tensor_decoder/tensordec.c | 28 ++-- include/tensor_common.h | 65 +++++++- tests/nnstreamer_sink/unittest_sink.cpp | 207 +++++++++++++++++++------ 6 files changed, 542 insertions(+), 162 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aa93ae3..fd18ddf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,7 @@ SET(PKG_MODULES gstreamer-controller-1.0 gstreamer-video-1.0 gstreamer-audio-1.0 + gstreamer-app-1.0 glib-2.0 ) diff --git a/common/tensor_common.c b/common/tensor_common.c index fb6570a..376b15b 100644 --- a/common/tensor_common.c +++ b/common/tensor_common.c @@ -59,11 +59,12 @@ gst_tensor_media_type_from_structure (const GstStructure * structure) g_return_val_if_fail (name != NULL, _NNS_MEDIA_END); - /** @todo Support other types */ if (g_str_has_prefix (name, "video/")) { return _NNS_VIDEO; } else if (g_str_has_prefix (name, "audio/")) { return _NNS_AUDIO; + } else if (g_str_has_prefix (name, "text/")) { + return _NNS_STRING; } /** unknown, or not-supported type */ @@ -94,6 +95,9 @@ gst_tensor_video_info_init (GstTensorVideoInfo * v_info) { g_return_if_fail (v_info != NULL); + /** + * Refer: https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html + */ v_info->format = GST_VIDEO_FORMAT_UNKNOWN; v_info->w = 0; v_info->h = 0; @@ -110,6 +114,9 @@ gst_tensor_audio_info_init (GstTensorAudioInfo * a_info) { g_return_if_fail (a_info != NULL); + /** + * Refer: https://gstreamer.freedesktop.org/documentation/design/mediatype-audio-raw.html + */ a_info->format = GST_AUDIO_FORMAT_UNKNOWN; a_info->ch = 0; a_info->rate = 0; @@ -117,6 +124,21 @@ gst_tensor_audio_info_init (GstTensorAudioInfo * a_info) } /** + * @brief Initialize the text info structure + * @param t_info text info structure to be initialized + */ +void +gst_tensor_text_info_init (GstTensorTextInfo * t_info) +{ + g_return_if_fail (t_info != NULL); + + /** + * Refer: https://gstreamer.freedesktop.org/documentation/design/mediatype-text-raw.html + */ + t_info->format = 0; +} + +/** * @brief Set video info to configure tensor * @param v_info video info structure to be filled * @param structure caps structure @@ -174,6 +196,30 @@ gst_tensor_audio_info_from_structure (GstTensorAudioInfo * a_info, } /** + * @brief Set text info to configure tensor + * @param t_info text info structure to be filled + * @param structure caps structure + */ +void +gst_tensor_text_info_from_structure (GstTensorTextInfo * t_info, + const GstStructure * structure) +{ + const gchar *format; + + g_return_if_fail (t_info != NULL); + g_return_if_fail (structure != NULL); + + gst_tensor_text_info_init (t_info); + + format = gst_structure_get_string (structure, "format"); + if (format) { + if (g_str_equal (format, "utf8")) { + t_info->format = 1; + } + } +} + +/** * @brief Set the video info structure from tensor config * @param v_info video info structure to be filled * @param config tensor config structure to be interpreted @@ -225,6 +271,28 @@ gst_tensor_audio_info_from_config (GstTensorAudioInfo * a_info, } /** + * @brief Set the text info structure from tensor config + * @param t_info text info structure to be filled + * @param config tensor config structure to be interpreted + * @return TRUE if supported format + */ +gboolean +gst_tensor_text_info_from_config (GstTensorTextInfo * t_info, + const GstTensorConfig * config) +{ + g_return_val_if_fail (config != NULL, FALSE); + g_return_val_if_fail (t_info != NULL, FALSE); + + gst_tensor_text_info_init (t_info); + + g_return_val_if_fail (config->tensor_media_type == _NNS_STRING, FALSE); + + t_info->format = config->tensor_media_format; + + return (t_info->format != 0); +} + +/** * @brief Initialize the tensor config info structure * @param config tensor config structure to be initialized */ @@ -244,7 +312,6 @@ gst_tensor_config_init (GstTensorConfig * config) config->rate_n = -1; config->rate_d = -1; - config->frame_size = 0; config->tensor_media_type = _NNS_MEDIA_END; config->tensor_media_format = 0; } @@ -255,7 +322,7 @@ gst_tensor_config_init (GstTensorConfig * config) * @return TRUE if configured */ gboolean -gst_tensor_config_validate (GstTensorConfig * config) +gst_tensor_config_validate (const GstTensorConfig * config) { guint i; @@ -275,18 +342,20 @@ gst_tensor_config_validate (GstTensorConfig * config) return FALSE; } - if (config->frame_size == 0 || config->tensor_media_format == 0) { - return FALSE; - } - switch (config->tensor_media_type) { case _NNS_VIDEO: case _NNS_AUDIO: + case _NNS_STRING: break; default: + /** unsupported type */ return FALSE; } + if (config->tensor_media_format == 0) { + return FALSE; + } + return TRUE; } @@ -312,8 +381,7 @@ gst_tensor_config_is_same (const GstTensorConfig * c1, } if (c1->rank == c2->rank && c1->type == c2->type && - c1->rate_n == c2->rate_n && c1->rate_d == c2->rate_d && - c1->frame_size == c2->frame_size) { + c1->rate_n == c2->rate_n && c1->rate_d == c2->rate_d) { for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) { if (c1->dimension[i] != c2->dimension[i]) { return FALSE; @@ -361,17 +429,20 @@ gst_tensor_config_from_tensor_structure (GstTensorConfig * config, gst_structure_get_fraction (structure, "framerate", &config->rate_n, &config->rate_d); - if (config->type != _NNS_END) { - config->frame_size = - tensor_element_size[config->type] * - get_tensor_element_count (config->dimension); - } - /** @todo we cannot get media type from caps */ - if (config->rank == 2) { - config->tensor_media_type = _NNS_AUDIO; - } else if (config->rank == 3) { - config->tensor_media_type = _NNS_VIDEO; + switch (config->rank) { + case 1: + config->tensor_media_type = _NNS_STRING; + break; + case 2: + config->tensor_media_type = _NNS_AUDIO; + break; + case 3: + config->tensor_media_type = _NNS_VIDEO; + break; + default: + config->tensor_media_type = _NNS_MEDIA_END; + break; } /** @todo we cannot get media format from caps */ @@ -407,6 +478,11 @@ gst_tensor_config_from_tensor_structure (GstTensorConfig * config, config->tensor_media_format = GST_AUDIO_FORMAT_UNKNOWN; break; } + } else if (config->tensor_media_type == _NNS_STRING) { + if (config->type == _NNS_INT8) { + /** utf8 */ + config->tensor_media_format = 1; + } } return TRUE; @@ -431,20 +507,34 @@ gst_tensor_config_from_media_structure (GstTensorConfig * config, m_type = gst_tensor_media_type_from_structure (structure); - if (m_type == _NNS_VIDEO) { - GstTensorVideoInfo v_info; + switch (m_type) { + case _NNS_VIDEO: + { + GstTensorVideoInfo v_info; + + gst_tensor_video_info_from_structure (&v_info, structure); + gst_tensor_config_from_video_info (config, &v_info); + break; + } + case _NNS_AUDIO: + { + GstTensorAudioInfo a_info; - gst_tensor_video_info_from_structure (&v_info, structure); - gst_tensor_config_from_video_info (config, &v_info); - } else if (m_type == _NNS_AUDIO) { - GstTensorAudioInfo a_info; + gst_tensor_audio_info_from_structure (&a_info, structure); + gst_tensor_config_from_audio_info (config, &a_info); + break; + } + case _NNS_STRING: + { + GstTensorTextInfo t_info; - gst_tensor_audio_info_from_structure (&a_info, structure); - gst_tensor_config_from_audio_info (config, &a_info); - } else { - /** @todo Support other types */ - err_print ("Unsupported type %d\n", m_type); - return FALSE; + gst_tensor_text_info_from_structure (&t_info, structure); + gst_tensor_config_from_text_info (config, &t_info); + break; + } + default: + err_print ("Unsupported type %d\n", m_type); + return FALSE; } return TRUE; @@ -474,7 +564,7 @@ gst_tensor_config_from_structure (GstTensorConfig * config, * @brief Set the tensor config structure from video info * @param config tensor config structure to be filled * @param v_info video info structure to be interpreted - * @return TRUE if ok + * @return TRUE if supported format */ gboolean gst_tensor_config_from_video_info (GstTensorConfig * config, @@ -485,8 +575,6 @@ gst_tensor_config_from_video_info (GstTensorConfig * config, * A 4-D uint8 or float32 Tensor of shape [batch_size, height, width, channels] * where channels is 1, 3, or 4. */ - gboolean res = TRUE; - g_return_val_if_fail (config != NULL, FALSE); g_return_val_if_fail (v_info != NULL, FALSE); @@ -507,7 +595,6 @@ gst_tensor_config_from_video_info (GstTensorConfig * config, default: /** unsupported format */ err_print ("Unsupported format = %d\n", v_info->format); - res = FALSE; break; } @@ -518,22 +605,16 @@ gst_tensor_config_from_video_info (GstTensorConfig * config, config->rate_n = v_info->fn; config->rate_d = v_info->fd; - if (config->type != _NNS_END) { - config->frame_size = - tensor_element_size[config->type] * - get_tensor_element_count (config->dimension); - } - config->tensor_media_type = _NNS_VIDEO; config->tensor_media_format = v_info->format; - return res; + return (config->type != _NNS_END); } /** * @brief Set the tensor config structure from audio info * @param config tensor config structure to be filled * @param a_info audio info structure to be interpreted - * @return TRUE if ok + * @return TRUE if supported format */ gboolean gst_tensor_config_from_audio_info (GstTensorConfig * config, @@ -544,8 +625,6 @@ gst_tensor_config_from_audio_info (GstTensorConfig * config, * A 3-D float32 Tensor of shape [batch_size, frames, channels] * or a 2-D float32 Tensor of shape [batch_size, frames]. */ - gboolean res = TRUE; - g_return_val_if_fail (config != NULL, FALSE); g_return_val_if_fail (a_info != NULL, FALSE); @@ -570,7 +649,6 @@ gst_tensor_config_from_audio_info (GstTensorConfig * config, default: /** unsupported format */ err_print ("Unsupported format = %d\n", a_info->format); - res = FALSE; break; } @@ -584,15 +662,54 @@ gst_tensor_config_from_audio_info (GstTensorConfig * config, config->rate_d = 1; } - if (config->type != _NNS_END) { - config->frame_size = - tensor_element_size[config->type] * - get_tensor_element_count (config->dimension); - } - config->tensor_media_type = _NNS_AUDIO; config->tensor_media_format = a_info->format; - return res; + return (config->type != _NNS_END); +} + +/** + * @brief Set the tensor config structure from text info + * @param config tensor config structure to be filled + * @param t_info text info structure to be interpreted + * @return TRUE if supported format + */ +gboolean +gst_tensor_config_from_text_info (GstTensorConfig * config, + const GstTensorTextInfo * t_info) +{ + /** + * Refer: https://www.tensorflow.org/api_docs/python/tf/summary/text + * A string-type Tensor + */ + g_return_val_if_fail (config != NULL, FALSE); + g_return_val_if_fail (t_info != NULL, FALSE); + + gst_tensor_config_init (config); + + config->rank = 1; + + switch (t_info->format) { + case 1: + /** utf8 */ + config->type = _NNS_INT8; + break; + default: + /** unsupported format */ + err_print ("Unsupported format = %d\n", t_info->format); + break; + } + + config->dimension[0] = 1; + config->dimension[1] = 1; + config->dimension[2] = 1; + config->dimension[3] = 1; + + /** cannot get the framerate for text type */ + config->rate_n = config->rate_d = 1; + + config->tensor_media_type = _NNS_STRING; + config->tensor_media_format = t_info->format; + return (config->type != _NNS_END); } /** @@ -604,12 +721,10 @@ GstCaps * gst_tensor_caps_from_config (const GstTensorConfig * config) { GstCaps *caps; - GstStaticCaps raw_caps = GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT); g_return_val_if_fail (config != NULL, NULL); - caps = gst_static_caps_get (&raw_caps); - caps = gst_caps_make_writable (caps); + caps = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT); if (config->rank > 0) { gst_caps_set_simple (caps, "rank", G_TYPE_INT, config->rank, NULL); @@ -654,12 +769,10 @@ gst_tensor_video_caps_from_config (const GstTensorConfig * config) { GstTensorVideoInfo v_info; GstCaps *caps; - GstStaticCaps raw_caps = GST_STATIC_CAPS (GST_TENSOR_VIDEO_CAPS_STR); g_return_val_if_fail (config != NULL, NULL); - caps = gst_static_caps_get (&raw_caps); - caps = gst_caps_make_writable (caps); + caps = gst_caps_from_string (GST_TENSOR_VIDEO_CAPS_STR); gst_tensor_video_info_from_config (&v_info, config); @@ -694,12 +807,10 @@ gst_tensor_audio_caps_from_config (const GstTensorConfig * config) { GstTensorAudioInfo a_info; GstCaps *caps; - GstStaticCaps raw_caps = GST_STATIC_CAPS (GST_TENSOR_AUDIO_CAPS_STR); g_return_val_if_fail (config != NULL, NULL); - caps = gst_static_caps_get (&raw_caps); - caps = gst_caps_make_writable (caps); + caps = gst_caps_from_string (GST_TENSOR_AUDIO_CAPS_STR); gst_tensor_audio_info_from_config (&a_info, config); @@ -720,6 +831,26 @@ gst_tensor_audio_caps_from_config (const GstTensorConfig * config) } /** + * @brief Get text caps from tensor config + * @param config tensor config info + * @return caps for given config + */ +static GstCaps * +gst_tensor_text_caps_from_config (const GstTensorConfig * config) +{ + GstTensorTextInfo t_info; + + g_return_val_if_fail (config != NULL, NULL); + + gst_tensor_text_info_from_config (&t_info, config); + + /** utf8 */ + g_return_val_if_fail (t_info.format != 1, NULL); + + return gst_caps_from_string (GST_TENSOR_TEXT_CAPS_STR); +} + +/** * @brief Get media caps from tensor config * @param config tensor config info * @return caps for given config @@ -738,8 +869,10 @@ gst_tensor_media_caps_from_config (const GstTensorConfig * config) case _NNS_AUDIO: caps = gst_tensor_audio_caps_from_config (config); break; + case _NNS_STRING: + caps = gst_tensor_text_caps_from_config (config); + break; default: - /** @todo Support other types */ err_print ("Unsupported type %d\n", config->tensor_media_type); break; } diff --git a/gst/tensor_converter/tensor_converter.c b/gst/tensor_converter/tensor_converter.c index 0a64881..c1605f7 100644 --- a/gst/tensor_converter/tensor_converter.c +++ b/gst/tensor_converter/tensor_converter.c @@ -120,19 +120,13 @@ enum */ #define DEFAULT_FRAMES_PER_BUFFER 0 -#define SINK_CAPS \ - GST_STATIC_CAPS (GST_TENSOR_VIDEO_CAPS_STR "; " GST_TENSOR_AUDIO_CAPS_STR) - -#define SRC_CAPS \ - GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT) - /** * @brief The capabilities of the inputs */ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - SINK_CAPS); + GST_STATIC_CAPS (GST_TENSOR_MEDIA_CAPS_STR)); /** * @brief The capabilities of the outputs @@ -140,7 +134,7 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - SRC_CAPS); + GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT)); #define gst_tensor_converter_parent_class parent_class G_DEFINE_TYPE (GstTensorConverter, gst_tensor_converter, @@ -188,6 +182,75 @@ gst_tensor_converter_check_consistency (GstTensorConverter * self, } /** + * @brief Validate newly configured tensor metadata + * @param self "this" pointer + * @param config newly configured tensor metadata + */ +static gboolean +gst_tensor_converter_validate_config (GstTensorConverter * self, + const GstTensorConfig * config) +{ + g_return_val_if_fail (self != NULL, FALSE); + g_return_val_if_fail (config != NULL, FALSE); + + if (self->tensor_configured && + !gst_tensor_converter_check_consistency (self, config)) { + /** mismatched to old metadata */ + return FALSE; + } + + if (!gst_tensor_config_validate (config)) { + /** not fully configured */ + return FALSE; + } + + return TRUE; +} + +/** + * @brief Get output frame size + * @param self "this" pointer + * @param in_size received buffer size + */ +static gsize +gst_tensor_converter_get_frame_size (GstTensorConverter * self, gsize in_size) +{ + GstTensorConfig *config; + gsize out_size = 0; + + g_assert (self->tensor_configured); + config = &self->tensor_config; + + /** + * @todo Do we need to aggregate the buffers to meet tensor dimension? + * video : supposed 1 frame per buffer. + * audio : we can get samples in buffer, but it would not be equal to tensor dimension. + * text : just passes the size of bytearray. + */ + switch (config->tensor_media_type) { + case _NNS_VIDEO: + out_size = + tensor_element_size[config->type] * + get_tensor_element_count (config->dimension); + break; + case _NNS_AUDIO: + /** + * samples in buffer = size / GST_AUDIO_INFO_BPF (&self->in_info.audio) + */ + out_size = in_size; + break; + case _NNS_STRING: + out_size = in_size; + break; + default: + /** unsupported type */ + break; + } + + return out_size; +} + +/** * @brief Parse structure and return tensor caps * @param self "this" pointer * @param structure structure to be interpreted @@ -250,9 +313,7 @@ gst_tensor_converter_configure_for_video (GstTensorConverter * self, return FALSE; } - if (self->tensor_configured && - !gst_tensor_converter_check_consistency (self, &config)) { - /** mismatched to old metadata */ + if (!gst_tensor_converter_validate_config (self, &config)) { return FALSE; } @@ -303,9 +364,7 @@ gst_tensor_converter_configure_for_audio (GstTensorConverter * self, return FALSE; } - if (self->tensor_configured && - !gst_tensor_converter_check_consistency (self, &config)) { - /** mismatched to old metadata */ + if (!gst_tensor_converter_validate_config (self, &config)) { return FALSE; } @@ -315,6 +374,36 @@ gst_tensor_converter_configure_for_audio (GstTensorConverter * self, } /** + * @brief Configure tensor metadata for text (internal static function) + * @param self "this" pointer to be configured. + * @param caps the sink cap. + * @return FALSE if error. TRUE if ok. + */ +static gboolean +gst_tensor_converter_configure_for_text (GstTensorConverter * self, + const GstCaps * caps) +{ + GstStructure *structure; + GstTensorConfig config; + GstTensorTextInfo t_info; + + structure = gst_caps_get_structure (caps, 0); + gst_tensor_text_info_from_structure (&t_info, structure); + + if (!gst_tensor_config_from_text_info (&config, &t_info)) { + /** unsupported format */ + return FALSE; + } + + if (!gst_tensor_converter_validate_config (self, &config)) { + return FALSE; + } + + self->tensor_config = config; + return TRUE; +} + +/** * @brief Configure tensor metadata from sink caps (internal static function) * @param self "this" pointer to be configured. * @param caps the sink cap. @@ -328,20 +417,22 @@ gst_tensor_converter_configure_tensor (GstTensorConverter * self, m_type = gst_tensor_media_type_from_caps (caps); - /** @todo Support other types */ switch (m_type) { case _NNS_VIDEO: if (!gst_tensor_converter_configure_for_video (self, caps)) { return FALSE; } break; - case _NNS_AUDIO: if (!gst_tensor_converter_configure_for_audio (self, caps)) { return FALSE; } break; - + case _NNS_STRING: + if (!gst_tensor_converter_configure_for_text (self, caps)) { + return FALSE; + } + break; default: err_print ("Unsupported type %d\n", m_type); return FALSE; @@ -500,12 +591,13 @@ gst_tensor_converter_copy_buffer (GstTensorConverter * self, g_assert (self->tensor_configured); config = &self->tensor_config; - block_size = config->frame_size; - g_assert (gst_buffer_get_size (outbuf) >= block_size); g_assert (gst_buffer_map (inbuf, &src_info, GST_MAP_READ)); g_assert (gst_buffer_map (outbuf, &dest_info, GST_MAP_WRITE)); + block_size = gst_tensor_converter_get_frame_size (self, src_info.size); + g_assert (gst_buffer_get_size (outbuf) >= block_size); + srcptr = src_info.data; destptr = dest_info.data; @@ -573,10 +665,9 @@ gst_tensor_converter_transform (GstBaseTransform * trans, switch (config->tensor_media_type) { case _NNS_VIDEO: case _NNS_AUDIO: + case _NNS_STRING: res = gst_tensor_converter_copy_buffer (self, inbuf, outbuf); break; - - case _NNS_STRING: default: err_print ("Unsupported Media Type (%d)\n", config->tensor_media_type); goto unknown_type; @@ -653,9 +744,9 @@ gst_tensor_converter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) break; case _NNS_AUDIO: + case _NNS_STRING: break; - case _NNS_STRING: default: err_print ("Unsupported Media Type (%d)\n", config->tensor_media_type); goto unknown_type; @@ -694,6 +785,7 @@ gst_tensor_converter_transform_caps (GstBaseTransform * trans, self = GST_TENSOR_CONVERTER_CAST (trans); + silent_debug ("Direction = %d\n", direction); silent_debug_caps (caps, "from"); silent_debug_caps (filter, "filter"); @@ -701,10 +793,8 @@ gst_tensor_converter_transform_caps (GstBaseTransform * trans, GstStructure *s = gst_caps_get_structure (caps, 0); result = gst_tensor_converter_caps_from_structure (self, s); } else if (direction == GST_PAD_SRC) { - GstStaticCaps raw_sink_caps = SINK_CAPS; - result = gst_static_caps_get (&raw_sink_caps); + result = gst_caps_from_string (GST_TENSOR_MEDIA_CAPS_STR); } else { - silent_debug ("Direction = %d\n", direction); g_assert (0); return NULL; } @@ -838,9 +928,9 @@ gst_tensor_converter_transform_size (GstBaseTransform * trans, self = GST_TENSOR_CONVERTER_CAST (trans); g_assert (self->tensor_configured); - *othersize = self->tensor_config.frame_size; + *othersize = gst_tensor_converter_get_frame_size (self, size); - return TRUE; + return (*othersize > 0); } /** diff --git a/gst/tensor_decoder/tensordec.c b/gst/tensor_decoder/tensordec.c index 76e2dc7..baa672f 100644 --- a/gst/tensor_decoder/tensordec.c +++ b/gst/tensor_decoder/tensordec.c @@ -93,8 +93,7 @@ enum static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT) - ); + GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT)); /** * @brief The capabilities of the outputs @@ -102,8 +101,7 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS ("ANY") - ); + GST_STATIC_CAPS ("ANY")); #define gst_tensordec_parent_class parent_class G_DEFINE_TYPE (GstTensorDec, gst_tensordec, GST_TYPE_BASE_TRANSFORM); @@ -146,13 +144,8 @@ gst_tensordec_media_caps_from_structure (GstTensorDec * self, } if (result == NULL) { - /** raw caps for video, audio */ - GstStaticCaps caps_video = GST_STATIC_CAPS (GST_TENSOR_VIDEO_CAPS_STR); - GstStaticCaps caps_audio = GST_STATIC_CAPS (GST_TENSOR_AUDIO_CAPS_STR); - - result = gst_caps_merge (gst_static_caps_get (&caps_video), - gst_static_caps_get (&caps_audio)); - result = gst_caps_simplify (result); + /** raw caps for supported media types */ + result = gst_caps_from_string (GST_TENSOR_MEDIA_CAPS_STR); } return result; @@ -322,6 +315,7 @@ gst_tensordec_configure (GstTensorDec * self, const GstCaps * caps) break; } case _NNS_AUDIO: + case _NNS_STRING: break; default: err_print ("Unsupported type %d\n", config.tensor_media_type); @@ -414,9 +408,9 @@ gst_tensordec_transform (GstBaseTransform * trans, switch (config->tensor_media_type) { case _NNS_VIDEO: case _NNS_AUDIO: + case _NNS_STRING: res = gst_tensordec_copy_buffer (self, inbuf, outbuf); break; - case _NNS_STRING: default: err_print ("Unsupported Media Type (%d)\n", config->tensor_media_type); goto unknown_type; @@ -464,8 +458,8 @@ gst_tensordec_transform_ip (GstBaseTransform * trans, GstBuffer * buf) } break; case _NNS_AUDIO: - break; case _NNS_STRING: + break; default: err_print ("Unsupported Media Type (%d)\n", config->tensor_media_type); goto unknown_type; @@ -504,6 +498,7 @@ gst_tensordec_transform_caps (GstBaseTransform * trans, self = GST_TENSORDEC_CAST (trans); + silent_debug ("Direction = %d\n", direction); silent_debug_caps (caps, "from"); silent_debug_caps (filter, "filter"); @@ -512,14 +507,13 @@ gst_tensordec_transform_caps (GstBaseTransform * trans, * Currently video/x-raw and audio/x-raw supported. */ if (direction == GST_PAD_SINK) { + /** caps from media */ GstStructure *s = gst_caps_get_structure (caps, 0); result = gst_tensordec_media_caps_from_structure (self, s); } else if (direction == GST_PAD_SRC) { /** caps from tensor */ - GstStaticCaps raw_caps = GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT); - result = gst_static_caps_get (&raw_caps); + result = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT); } else { - silent_debug ("Direction = %d\n", direction); g_assert (0); return NULL; } @@ -652,7 +646,7 @@ gst_tensordec_transform_size (GstBaseTransform * trans, *othersize = offset * config->dimension[2] * config->dimension[3]; } else { - *othersize = self->tensor_config.frame_size; + *othersize = size; } return TRUE; diff --git a/include/tensor_common.h b/include/tensor_common.h index e5d3c72..da52ead 100644 --- a/include/tensor_common.h +++ b/include/tensor_common.h @@ -43,6 +43,18 @@ G_BEGIN_DECLS GST_AUDIO_CAPS_MAKE ("{ S8, U8, S16LE, S16BE, U16LE, U16BE }") \ ", layout = (string) interleaved" +#define GST_TENSOR_TEXT_CAPS_STR \ + "text/x-raw, format = (string) utf8" + +/** + * @brief Caps string for supported types + * @todo Support other types + */ +#define GST_TENSOR_MEDIA_CAPS_STR \ + GST_TENSOR_VIDEO_CAPS_STR "; " \ + GST_TENSOR_AUDIO_CAPS_STR "; " \ + GST_TENSOR_TEXT_CAPS_STR + /** @todo I'm not sure if the range is to be 1, 65535 or larger */ #define GST_TENSOR_RANK_RANGE "(int) [ 1, 4 ]" #define GST_TENSOR_DIM_RANGE "(int) [ 1, 65535 ]" @@ -91,7 +103,7 @@ typedef enum _nns_media_type { _NNS_VIDEO = 0, /**< supposedly video/x-raw */ _NNS_AUDIO, /**< supposedly audio/x-raw */ - _NNS_STRING, /**< Not Supported Yet */ + _NNS_STRING, /**< supposedly text/x-raw */ _NNS_MEDIA_END, /**< End Marker */ } media_type; @@ -120,6 +132,14 @@ typedef struct } GstTensorAudioInfo; /** + * @brief Internal data structure for text info to configure tensor. + */ +typedef struct +{ + gint format; /**< text format (0:unknown, 1:utf8) */ +} GstTensorTextInfo; + +/** * @brief Internal data structure for configured tensor info. */ typedef struct @@ -129,7 +149,6 @@ typedef struct tensor_dim dimension; /**< Dimensions. We support up to 4th ranks. */ gint rate_n; /**< framerate is in fraction, which is numerator/denominator */ gint rate_d; /**< framerate is in fraction, which is numerator/denominator */ - gsize frame_size; /**< Size of a single tensor frame in # bytes */ media_type tensor_media_type; /**< Denotes the input media stream type */ gint tensor_media_format; /**< Denotes the input media stream format */ } GstTensorConfig; @@ -170,6 +189,13 @@ extern void gst_tensor_audio_info_init (GstTensorAudioInfo * a_info); /** + * @brief Initialize the text info structure + * @param t_info text info structure to be initialized + */ +extern void +gst_tensor_text_info_init (GstTensorTextInfo * t_info); + +/** * @brief Set video info to configure tensor * @param v_info video info structure to be filled * @param structure caps structure @@ -188,6 +214,15 @@ gst_tensor_audio_info_from_structure (GstTensorAudioInfo * a_info, const GstStructure * structure); /** + * @brief Set text info to configure tensor + * @param t_info text info structure to be filled + * @param structure caps structure + */ +extern void +gst_tensor_text_info_from_structure (GstTensorTextInfo * t_info, + const GstStructure * structure); + +/** * @brief Set the video info structure from tensor config * @param v_info video info structure to be filled * @param config tensor config structure to be interpreted @@ -208,6 +243,16 @@ gst_tensor_audio_info_from_config (GstTensorAudioInfo * a_info, const GstTensorConfig * config); /** + * @brief Set the text info structure from tensor config + * @param t_info text info structure to be filled + * @param config tensor config structure to be interpreted + * @return TRUE if supported format + */ +extern gboolean +gst_tensor_text_info_from_config (GstTensorTextInfo * t_info, + const GstTensorConfig * config); + +/** * @brief Initialize the tensor config info structure * @param config tensor config structure to be initialized */ @@ -220,7 +265,7 @@ gst_tensor_config_init (GstTensorConfig * config); * @return TRUE if configured */ extern gboolean -gst_tensor_config_validate (GstTensorConfig * config); +gst_tensor_config_validate (const GstTensorConfig * config); /** * @brief Compare tensor config info @@ -244,7 +289,7 @@ gst_tensor_config_from_structure (GstTensorConfig * config, * @brief Set the tensor config structure from video info * @param config tensor config structure to be filled * @param v_info video info structure to be interpreted - * @return TRUE if ok + * @return TRUE if supported format */ extern gboolean gst_tensor_config_from_video_info (GstTensorConfig * config, @@ -254,13 +299,23 @@ gst_tensor_config_from_video_info (GstTensorConfig * config, * @brief Set the tensor config structure from audio info * @param config tensor config structure to be filled * @param a_info audio info structure to be interpreted - * @return TRUE if ok + * @return TRUE if supported format */ extern gboolean gst_tensor_config_from_audio_info (GstTensorConfig * config, const GstTensorAudioInfo * a_info); /** + * @brief Set the tensor config structure from text info + * @param config tensor config structure to be filled + * @param t_info text info structure to be interpreted + * @return TRUE if supported format + */ +extern gboolean +gst_tensor_config_from_text_info (GstTensorConfig * config, + const GstTensorTextInfo * t_info); + +/** * @brief Get tensor caps from tensor config * @param config tensor config info * @return caps for given config diff --git a/tests/nnstreamer_sink/unittest_sink.cpp b/tests/nnstreamer_sink/unittest_sink.cpp index a0e4418..0aa83a2 100644 --- a/tests/nnstreamer_sink/unittest_sink.cpp +++ b/tests/nnstreamer_sink/unittest_sink.cpp @@ -8,8 +8,10 @@ * @bug No known bugs. */ +#include #include #include +#include /** * @brief Macro for debug mode. @@ -45,13 +47,24 @@ typedef enum } TestStatus; /** + * @brief Test type. + */ +typedef enum +{ + TEST_TYPE_VIDEO, /**< pipeline for video */ + TEST_TYPE_AUDIO, /**< pipeline for audio */ + TEST_TYPE_TEXT, /**< pipeline for text */ + TEST_TYPE_TENSORS, /**< pipeline for tensors with tensormux */ + TEST_TYPE_NEGO_FAILED, /**< pipeline to test caps negotiation */ +} TestType; + +/** * @brief Test options. */ typedef struct { guint num_buffers; /**< count of buffers */ - gboolean convert_tensor; /**< true to set tensor converter */ - gboolean caps_tensors; /**< true to test caps other/tensors */ + TestType test_type; /**< test pipeline */ } TestOption; /** @@ -194,27 +207,53 @@ _setup_pipeline (TestOption & option) g_test_data.end = FALSE; g_test_data.caps_name = NULL; - _print_log ("option num_buffers[%d] convert_tensor[%d] caps_tensors[%d]", - option.num_buffers, option.convert_tensor, option.caps_tensors); + _print_log ("option num_buffers[%d] test_type[%d]", + option.num_buffers, option.test_type); g_test_data.loop = g_main_loop_new (NULL, FALSE); _check_cond_err (g_test_data.loop != NULL); - if (option.caps_tensors) { - /** other/tensors with tensormux */ - str_pipeline = - g_strdup_printf - ("tensormux name=mux ! tensor_sink name=test_sink " - "videotestsrc num-buffers=%d ! video/x-raw,width=160,height=120,format=RGB,framerate=(fraction)30/1 ! tensor_converter ! mux.sink_0 " - "videotestsrc num-buffers=%d ! video/x-raw,width=160,height=120,format=RGB,framerate=(fraction)30/1 ! tensor_converter ! mux.sink_1 ", - option.num_buffers, option.num_buffers); - } else { - str_pipeline = - g_strdup_printf - ("videotestsrc num-buffers=%d ! video/x-raw,width=160,height=120,format=RGB,framerate=(fraction)30/1 ! " - "%s ! tensor_sink name=test_sink", option.num_buffers, - option.convert_tensor ? "tensor_converter" : "videoconvert"); + switch (option.test_type) { + case TEST_TYPE_VIDEO: + /** video 160x120 */ + str_pipeline = + g_strdup_printf + ("videotestsrc num-buffers=%d ! video/x-raw,width=160,height=120,format=RGB,framerate=(fraction)30/1 ! " + "tensor_converter ! tensor_sink name=test_sink", option.num_buffers); + break; + case TEST_TYPE_AUDIO: + /** audio sample rate 16000 (16 bits, signed, little endian) */ + str_pipeline = + g_strdup_printf + ("audiotestsrc num-buffers=%d ! audio/x-raw,format=S16LE,rate=16000 ! " + "tensor_converter ! tensor_sink name=test_sink", option.num_buffers); + break; + case TEST_TYPE_TEXT: + str_pipeline = + g_strdup_printf + ("appsrc name=appsrc caps=text/x-raw,format=utf8 ! " + "tensor_converter ! tensor_sink name=test_sink"); + break; + case TEST_TYPE_TENSORS: + /** other/tensors with tensormux */ + str_pipeline = + g_strdup_printf + ("tensormux name=mux ! tensor_sink name=test_sink " + "videotestsrc num-buffers=%d ! video/x-raw,width=160,height=120,format=RGB,framerate=(fraction)30/1 ! tensor_converter ! mux.sink_0 " + "videotestsrc num-buffers=%d ! video/x-raw,width=160,height=120,format=RGB,framerate=(fraction)30/1 ! tensor_converter ! mux.sink_1 ", + option.num_buffers, option.num_buffers); + break; + case TEST_TYPE_NEGO_FAILED: + /** caps negotiation failed */ + str_pipeline = + g_strdup_printf + ("videotestsrc num-buffers=%d ! video/x-raw,width=160,height=120,format=RGB,framerate=(fraction)30/1 ! " + "videoconvert ! tensor_sink name=test_sink", option.num_buffers); + break; + default: + goto error; } + g_test_data.pipeline = gst_parse_launch (str_pipeline, NULL); g_free (str_pipeline); _check_cond_err (g_test_data.pipeline != NULL); @@ -231,6 +270,11 @@ _setup_pipeline (TestOption & option) gst_bin_get_by_name (GST_BIN (g_test_data.pipeline), "test_sink"); _check_cond_err (g_test_data.sink != NULL); + if (DBG) { + /** print logs */ + g_object_set (g_test_data.sink, "silent", (gboolean) FALSE, NULL); + } + g_test_data.status = TEST_INIT; return TRUE; @@ -250,7 +294,7 @@ TEST (tensor_sink_test, properties) gboolean emit, res_emit; gboolean sync, res_sync; gboolean qos, res_qos; - TestOption option = { 1, TRUE, FALSE }; + TestOption option = { 1, TEST_TYPE_VIDEO }; ASSERT_TRUE (_setup_pipeline (option)); @@ -272,7 +316,7 @@ TEST (tensor_sink_test, properties) /** default silent is TRUE */ g_object_get (g_test_data.sink, "silent", &silent, NULL); - EXPECT_EQ (silent, TRUE); + EXPECT_EQ (silent, (DBG) ? FALSE : TRUE); g_object_set (g_test_data.sink, "silent", !silent, NULL); g_object_get (g_test_data.sink, "silent", &res_silent, NULL); @@ -313,15 +357,10 @@ TEST (tensor_sink_test, signals) { const guint num_buffers = 10; gulong handle_id; - TestOption option = { num_buffers, TRUE, FALSE }; + TestOption option = { num_buffers, TEST_TYPE_VIDEO }; ASSERT_TRUE (_setup_pipeline (option)); - if (DBG) { - /** print logs */ - g_object_set (g_test_data.sink, "silent", (gboolean) FALSE, NULL); - } - /** tensor sink signals */ handle_id = g_signal_connect (g_test_data.sink, "new-data", (GCallback) _new_data_cb, NULL); @@ -361,15 +400,10 @@ TEST (tensor_sink_test, signal_rate) { const guint num_buffers = 10; gulong handle_id; - TestOption option = { num_buffers, TRUE, FALSE }; + TestOption option = { num_buffers, TEST_TYPE_VIDEO }; ASSERT_TRUE (_setup_pipeline (option)); - if (DBG) { - /** print logs */ - g_object_set (g_test_data.sink, "silent", (gboolean) FALSE, NULL); - } - /** set signal-rate */ g_object_set (g_test_data.sink, "signal-rate", (guint) 15, NULL); @@ -403,15 +437,10 @@ TEST (tensor_sink_test, unknown_case) const guint num_buffers = 5; gulong handle_id; gint unknown = -1; - TestOption option = { num_buffers, TRUE, FALSE }; + TestOption option = { num_buffers, TEST_TYPE_VIDEO }; ASSERT_TRUE (_setup_pipeline (option)); - if (DBG) { - /** print logs */ - g_object_set (g_test_data.sink, "silent", (gboolean) FALSE, NULL); - } - /** try to set/get unknown property */ g_object_set (g_test_data.sink, "unknown-prop", 1, NULL); g_object_get (g_test_data.sink, "unknown-prop", &unknown, NULL); @@ -443,16 +472,11 @@ TEST (tensor_sink_test, caps_error) { const guint num_buffers = 5; gulong handle_id; - TestOption option = { num_buffers, FALSE, FALSE }; + TestOption option = { num_buffers, TEST_TYPE_NEGO_FAILED }; /** failed : cannot link videoconvert and tensor_sink */ ASSERT_TRUE (_setup_pipeline (option)); - if (DBG) { - /** print logs */ - g_object_set (g_test_data.sink, "silent", (gboolean) FALSE, NULL); - } - /** signal for new data */ handle_id = g_signal_connect (g_test_data.sink, "new-data", (GCallback) _new_data_cb, NULL); @@ -479,15 +503,10 @@ TEST (tensor_sink_test, caps_tensors) { const guint num_buffers = 5; gulong handle_id; - TestOption option = { num_buffers, TRUE, TRUE }; + TestOption option = { num_buffers, TEST_TYPE_TENSORS }; ASSERT_TRUE (_setup_pipeline (option)); - if (DBG) { - /** print logs */ - g_object_set (g_test_data.sink, "silent", (gboolean) FALSE, NULL); - } - /** signal for new data */ handle_id = g_signal_connect (g_test_data.sink, "new-data", (GCallback) _new_data_cb, NULL); @@ -511,6 +530,94 @@ TEST (tensor_sink_test, caps_tensors) } /** + * @brief Test for audio stream. + */ +TEST (tensor_sink_test, audio_stream) +{ + const guint num_buffers = 10; + gulong handle_id; + TestOption option = { num_buffers, TEST_TYPE_AUDIO }; + + ASSERT_TRUE (_setup_pipeline (option)); + + /** signal for new data */ + handle_id = g_signal_connect (g_test_data.sink, "new-data", + (GCallback) _new_data_cb, NULL); + EXPECT_TRUE (handle_id > 0); + + _print_log ("start pipeline to test audio stream"); + gst_element_set_state (g_test_data.pipeline, GST_STATE_PLAYING); + g_main_loop_run (g_test_data.loop); + gst_element_set_state (g_test_data.pipeline, GST_STATE_NULL); + + /** check eos message */ + EXPECT_EQ (g_test_data.status, TEST_EOS); + + /** check received buffers */ + EXPECT_EQ (g_test_data.received, num_buffers); + + /** check caps name */ + EXPECT_TRUE (g_str_equal (g_test_data.caps_name, "other/tensor")); + + _free_test_data (); +} + +/** + * @brief Test for text stream. + */ +TEST (tensor_sink_test, text_stream) +{ + const guint num_buffers = 10; + gulong handle_id; + guint i; + GstElement *appsrc; + TestOption option = { num_buffers, TEST_TYPE_TEXT }; + + ASSERT_TRUE (_setup_pipeline (option)); + + appsrc = gst_bin_get_by_name (GST_BIN (g_test_data.pipeline), "appsrc"); + + /** signal for new data */ + handle_id = g_signal_connect (g_test_data.sink, "new-data", + (GCallback) _new_data_cb, NULL); + EXPECT_TRUE (handle_id > 0); + + _print_log ("start pipeline to test text stream"); + gst_element_set_state (g_test_data.pipeline, GST_STATE_PLAYING); + + for (i = 0; i < num_buffers; i++) { + GstBuffer *buf = gst_buffer_new_allocate (NULL, 5, NULL); + GstMapInfo info; + + gst_buffer_map (buf, &info, GST_MAP_WRITE); + strcpy ((gchar *) info.data, "test"); + gst_buffer_unmap (buf, &info); + + GST_BUFFER_PTS (buf) = (i + 1) * 20 * GST_MSECOND; + GST_BUFFER_DTS (buf) = GST_BUFFER_PTS (buf); + + EXPECT_EQ (gst_app_src_push_buffer (GST_APP_SRC (appsrc), buf), + GST_FLOW_OK); + } + + EXPECT_EQ (gst_app_src_end_of_stream (GST_APP_SRC (appsrc)), GST_FLOW_OK); + + g_main_loop_run (g_test_data.loop); + gst_element_set_state (g_test_data.pipeline, GST_STATE_NULL); + + /** check eos message */ + EXPECT_EQ (g_test_data.status, TEST_EOS); + + /** check received buffers */ + EXPECT_EQ (g_test_data.received, num_buffers); + + /** check caps name */ + EXPECT_TRUE (g_str_equal (g_test_data.caps_name, "other/tensor")); + + _free_test_data (); +} + +/** * @brief Main function for unit test. */ int -- 2.7.4