From fcc4868a231aa8d1ba4c32bfd25974aaa072ff64 Mon Sep 17 00:00:00 2001 From: Jaeyun Date: Tue, 12 Mar 2019 16:58:33 +0900 Subject: [PATCH] [Converter] add option about supported media type 1. add meson option to disable video support. tensor-converter will not get the video if video-support disabled. 2. add macros for media format and info to disable audio/video stream. Signed-off-by: Jaeyun Jung --- .../tensor_converter/converter-media-info.h | 134 +++++++++++++++ gst/nnstreamer/tensor_converter/tensor_converter.c | 188 +++++++++------------ meson.build | 9 + meson_options.txt | 1 + 4 files changed, 228 insertions(+), 104 deletions(-) create mode 100644 gst/nnstreamer/tensor_converter/converter-media-info.h diff --git a/gst/nnstreamer/tensor_converter/converter-media-info.h b/gst/nnstreamer/tensor_converter/converter-media-info.h new file mode 100644 index 0000000..69bbb15 --- /dev/null +++ b/gst/nnstreamer/tensor_converter/converter-media-info.h @@ -0,0 +1,134 @@ +/** + * NNStreamer media type definition for tensor-converter + * Copyright (C) 2019 Jijoong Moon + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + */ + +/** + * @file converter-media-info.h + * @date 26 Mar 2019 + * @brief Define collection of media type and functions to parse media info for the case of no audio/video support + * @see https://github.com/nnsuite/nnstreamer + * @author Jijoong Moon + * @bug No known bugs except for NYI items + */ + +#ifndef __CONVERTER_MEDIA_INFO_H__ +#define __CONVERTER_MEDIA_INFO_H__ + +#ifndef NO_VIDEO +#include + +/** + * @brief Caps string for supported video format + */ +#define VIDEO_CAPS_STR \ + GST_VIDEO_CAPS_MAKE ("{ RGB, BGR, RGBx, BGRx, xRGB, xBGR, RGBA, BGRA, ARGB, ABGR, GRAY8 }") \ + ", views = (int) 1, interlace-mode = (string) progressive" + +#define append_video_caps_template(caps) \ + gst_caps_append (caps, gst_caps_from_string (VIDEO_CAPS_STR)) + +#define is_video_supported(...) TRUE +#else +#define append_video_caps_template(caps) +#define is_video_supported(...) FALSE + +#define GstVideoInfo gsize + +typedef enum { + GST_VIDEO_FORMAT_UNKNOWN, + GST_VIDEO_FORMAT_GRAY8, + GST_VIDEO_FORMAT_RGB, + GST_VIDEO_FORMAT_BGR, + GST_VIDEO_FORMAT_RGBx, + GST_VIDEO_FORMAT_BGRx, + GST_VIDEO_FORMAT_xRGB, + GST_VIDEO_FORMAT_xBGR, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_FORMAT_BGRA, + GST_VIDEO_FORMAT_ARGB, + GST_VIDEO_FORMAT_ABGR, + GST_VIDEO_FORMAT_I420 +} GstVideoFormat; + +#define gst_video_info_init(i) memset (i, 0, sizeof (GstVideoInfo)) +#define gst_video_info_from_caps(...) FALSE +#define gst_video_format_to_string(...) "Unknown" + +#define GST_VIDEO_INFO_FORMAT(...) GST_VIDEO_FORMAT_UNKNOWN +#define GST_VIDEO_INFO_WIDTH(...) 0 +#define GST_VIDEO_INFO_HEIGHT(...) 0 +#define GST_VIDEO_INFO_SIZE(...) 0 +#define GST_VIDEO_INFO_FPS_N(...) 0 +#define GST_VIDEO_INFO_FPS_D(...) 1 +#endif /* NO_VIDEO */ + +#ifndef NO_AUDIO +#include + +/** + * @brief Caps string for supported audio format + */ +#define AUDIO_CAPS_STR \ + GST_AUDIO_CAPS_MAKE ("{ S8, U8, S16LE, S16BE, U16LE, U16BE, S32LE, S32BE, U32LE, U32BE, F32LE, F32BE, F64LE, F64BE }") \ + ", layout = (string) interleaved" + +#define append_audio_caps_template(caps) \ + gst_caps_append (caps, gst_caps_from_string (AUDIO_CAPS_STR)) + +#define is_audio_supported(...) TRUE +#else +#define append_audio_caps_template(caps) +#define is_audio_supported(...) FALSE + +#define GstAudioInfo gsize + +typedef enum { + GST_AUDIO_FORMAT_UNKNOWN, + GST_AUDIO_FORMAT_S8, + GST_AUDIO_FORMAT_U8, + GST_AUDIO_FORMAT_S16, + GST_AUDIO_FORMAT_U16, + GST_AUDIO_FORMAT_S32, + GST_AUDIO_FORMAT_U32, + GST_AUDIO_FORMAT_F32, + GST_AUDIO_FORMAT_F64 +} GstAudioFormat; + +#define gst_audio_info_init(i) memset (i, 0, sizeof (GstAudioInfo)) +#define gst_audio_info_from_caps(...) FALSE +#define gst_audio_format_to_string(...) "Unknown" + +#define GST_AUDIO_INFO_FORMAT(...) GST_AUDIO_FORMAT_UNKNOWN +#define GST_AUDIO_INFO_CHANNELS(...) 0 +#define GST_AUDIO_INFO_RATE(...) 0 +#define GST_AUDIO_INFO_BPF(...) 0 +#endif /* NO_AUDIO */ + +/** + * @brief Caps string for text input + */ +#define TEXT_CAPS_STR "text/x-raw, format = (string) utf8" + +#define append_text_caps_template(caps) \ + gst_caps_append (caps, gst_caps_from_string (TEXT_CAPS_STR)) + +/** + * @brief Caps string for binary stream + */ +#define OCTET_CAPS_STR "application/octet-stream" + +#define append_octet_caps_template(caps) \ + gst_caps_append (caps, gst_caps_from_string (OCTET_CAPS_STR)) + +#endif /* __CONVERTER_MEDIA_INFO_H__ */ diff --git a/gst/nnstreamer/tensor_converter/tensor_converter.c b/gst/nnstreamer/tensor_converter/tensor_converter.c index b30fcdf..450e7dd 100644 --- a/gst/nnstreamer/tensor_converter/tensor_converter.c +++ b/gst/nnstreamer/tensor_converter/tensor_converter.c @@ -44,10 +44,7 @@ #include #include "tensor_converter.h" -#include -#ifndef NO_AUDIO -#include -#endif +#include "converter-media-info.h" /** * @brief Macro for debug mode. @@ -121,21 +118,6 @@ enum */ #define DEFAULT_FRAMES_PER_TENSOR 1 -/** - * @brief Caps string for supported types - * @todo Support other types - */ -#define VIDEO_CAPS_STR \ - GST_VIDEO_CAPS_MAKE ("{ RGB, BGR, RGBx, BGRx, xRGB, xBGR, RGBA, BGRA, ARGB, ABGR, GRAY8 }") \ - ", views = (int) 1, interlace-mode = (string) progressive" -#ifndef NO_AUDIO -#define AUDIO_CAPS_STR \ - GST_AUDIO_CAPS_MAKE ("{ S8, U8, S16LE, S16BE, U16LE, U16BE, S32LE, S32BE, U32LE, U32BE, F32LE, F32BE, F64LE, F64BE }") \ - ", layout = (string) interleaved" -#endif -#define TEXT_CAPS_STR "text/x-raw, format = (string) utf8" -#define OCTET_CAPS_STR "application/octet-stream" - #define gst_tensor_converter_parent_class parent_class G_DEFINE_TYPE (GstTensorConverter, gst_tensor_converter, GST_TYPE_ELEMENT); @@ -250,16 +232,12 @@ gst_tensor_converter_class_init (GstTensorConverterClass * klass) /* set sink pad template */ pad_caps = gst_caps_new_empty (); - /* video */ - gst_caps_append (pad_caps, gst_caps_from_string (VIDEO_CAPS_STR)); -#ifndef NO_AUDIO - /* audio */ - gst_caps_append (pad_caps, gst_caps_from_string (AUDIO_CAPS_STR)); -#endif - /* text */ - gst_caps_append (pad_caps, gst_caps_from_string (TEXT_CAPS_STR)); - /* octet stream */ - gst_caps_append (pad_caps, gst_caps_from_string (OCTET_CAPS_STR)); + + /* append caps string for all media types */ + append_video_caps_template (pad_caps); + append_audio_caps_template (pad_caps); + append_text_caps_template (pad_caps); + append_octet_caps_template (pad_caps); pad_template = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, pad_caps); @@ -667,13 +645,11 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) } break; } -#ifndef NO_AUDIO case _NNS_AUDIO: /* number of bytes for one frame */ frame_size = self->frame_size; frames_in = buf_size / frame_size; break; -#endif case _NNS_TEXT: /* supposed 1 frame in buffer */ frame_size = self->frame_size; @@ -898,6 +874,29 @@ gst_tensor_converter_reset (GstTensorConverter * self) } /** + * @brief Get supported format list. + */ +static void +gst_tensor_converter_get_format_list (GValue * list, ...) +{ + GValue item = G_VALUE_INIT; + gchar *str; + va_list args; + + g_value_init (list, GST_TYPE_LIST); + + va_start (args, list); + while ((str = va_arg (args, gchar *))) { + g_value_init (&item, G_TYPE_STRING); + g_value_set_string (&item, str); + + gst_value_list_append_value (list, &item); + g_value_unset (&item); + } + va_end (args); +} + +/** * @brief Determine if we need zero-padding * @return TRUE if we need to add (or remove) stride per row from the stream data. */ @@ -994,7 +993,6 @@ gst_tensor_converter_parse_video (GstTensorConverter * self, return (config->info.type != _NNS_END); } -#ifndef NO_AUDIO /** * @brief Set the tensor config structure from audio info (internal static function) * @param self this pointer to GstTensorConverter @@ -1067,7 +1065,6 @@ gst_tensor_converter_parse_audio (GstTensorConverter * self, return (config->info.type != _NNS_END); } -#endif /** * @brief Set the tensor config structure from text info (internal static function) @@ -1095,7 +1092,7 @@ gst_tensor_converter_parse_text (GstTensorConverter * self, format_string = gst_structure_get_string (structure, "format"); if (format_string) { - if (g_str_equal (format_string, "utf8")) { + if (g_ascii_strcasecmp (format_string, "utf8") == 0) { config->info.type = _NNS_UINT8; } else { /* unsupported format */ @@ -1161,29 +1158,6 @@ gst_tensor_converter_parse_octet (GstTensorConverter * self, } /** - * @brief Get supported format list. - */ -static void -gst_tensor_converter_get_format_list (GValue * list, ...) -{ - GValue item = G_VALUE_INIT; - gchar *str; - va_list args; - - g_value_init (list, GST_TYPE_LIST); - - va_start (args, list); - while ((str = va_arg (args, gchar *))) { - g_value_init (&item, G_TYPE_STRING); - g_value_set_string (&item, str); - - gst_value_list_append_value (list, &item); - g_value_unset (&item); - } - va_end (args); -} - -/** * @brief Get pad caps for caps negotiation. */ static GstCaps * @@ -1229,7 +1203,7 @@ gst_tensor_converter_query_caps (GstTensorConverter * self, GstPad * pad, switch (type) { case _NNS_VIDEO: /* video caps from tensor info */ - if (config.info.type == _NNS_UINT8) { + if (is_video_supported (self) && config.info.type == _NNS_UINT8) { GValue supported_formats = G_VALUE_INIT; gint colorspace, width, height; @@ -1272,10 +1246,9 @@ gst_tensor_converter_query_caps (GstTensorConverter * self, GstPad * pad, } } break; -#ifndef NO_AUDIO case _NNS_AUDIO: /* audio caps from tensor info */ - if (config.info.type != _NNS_END) { + if (is_audio_supported (self) && config.info.type != _NNS_END) { gint channels, samplerate; GstAudioFormat aformat; @@ -1326,7 +1299,6 @@ gst_tensor_converter_query_caps (GstTensorConverter * self, GstPad * pad, } } break; -#endif default: /* do nothing for text and octet stream */ break; @@ -1383,60 +1355,68 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self, switch (in_type) { case _NNS_VIDEO: - { - GstVideoInfo info; + if (is_video_supported (self)) { + GstVideoInfo info; - gst_video_info_init (&info); - if (!gst_video_info_from_caps (&info, caps)) { - GST_ERROR_OBJECT (self, "Failed to get video info from caps.\n"); - return FALSE; - } + gst_video_info_init (&info); + if (!gst_video_info_from_caps (&info, caps)) { + GST_ERROR_OBJECT (self, "Failed to get video info from caps.\n"); + return FALSE; + } - if (!gst_tensor_converter_parse_video (self, &config, &info)) { - GST_ERROR_OBJECT (self, "Failed to configure tensor from video info."); - return FALSE; - } + if (!gst_tensor_converter_parse_video (self, &config, &info)) { + GST_ERROR_OBJECT (self, + "Failed to configure tensor from video info."); + return FALSE; + } - /** - * Emit Warning if RSTRIDE = RU4 (3BPP) && Width % 4 > 0 - * @todo Add more conditions! - */ - if (gst_tensor_converter_video_stride (GST_VIDEO_INFO_FORMAT (&info), - GST_VIDEO_INFO_WIDTH (&info))) { - self->remove_padding = TRUE; - silent_debug ("Set flag to remove padding, width = %d", - GST_VIDEO_INFO_WIDTH (&info)); - - GST_WARNING_OBJECT (self, - "\nYOUR STREAM CONFIGURATION INCURS PERFORMANCE DETERIORATION!\n" - "Please use 4 x n as image width for inputs.\n"); - } + /** + * Emit Warning if RSTRIDE = RU4 (3BPP) && Width % 4 > 0 + * @todo Add more conditions! + */ + if (gst_tensor_converter_video_stride (GST_VIDEO_INFO_FORMAT (&info), + GST_VIDEO_INFO_WIDTH (&info))) { + self->remove_padding = TRUE; + silent_debug ("Set flag to remove padding, width = %d", + GST_VIDEO_INFO_WIDTH (&info)); + + GST_WARNING_OBJECT (self, + "\nYOUR STREAM CONFIGURATION INCURS PERFORMANCE DETERIORATION!\n" + "Please use 4 x n as image width for inputs.\n"); + } - frames_dim = 3; - self->frame_size = GST_VIDEO_INFO_SIZE (&info); + frames_dim = 3; + self->frame_size = GST_VIDEO_INFO_SIZE (&info); + } else { + g_critical + ("\n This binary does not support video type. Please build NNStreamer with disable-video-support : false\n"); + return FALSE; + } break; - } -#ifndef NO_AUDIO case _NNS_AUDIO: - { - GstAudioInfo info; + if (is_audio_supported (self)) { + GstAudioInfo info; - gst_audio_info_init (&info); - if (!gst_audio_info_from_caps (&info, caps)) { - GST_ERROR_OBJECT (self, "Failed to get audio info from caps.\n"); - return FALSE; - } + gst_audio_info_init (&info); + if (!gst_audio_info_from_caps (&info, caps)) { + GST_ERROR_OBJECT (self, "Failed to get audio info from caps.\n"); + return FALSE; + } - if (!gst_tensor_converter_parse_audio (self, &config, &info)) { - GST_ERROR_OBJECT (self, "Failed to configure tensor from audio info."); + if (!gst_tensor_converter_parse_audio (self, &config, &info)) { + GST_ERROR_OBJECT (self, + "Failed to configure tensor from audio info."); + return FALSE; + } + + frames_dim = 1; + self->frame_size = GST_AUDIO_INFO_BPF (&info); + } else { + g_critical + ("\n This binary does not support audio type. Please build NNStreamer with disable-audio-support : false\n"); return FALSE; } - - frames_dim = 1; - self->frame_size = GST_AUDIO_INFO_BPF (&info); break; - } -#endif case _NNS_TEXT: { if (!gst_tensor_converter_parse_text (self, &config, structure)) { diff --git a/meson.build b/meson.build index c8beb23..5b11601 100644 --- a/meson.build +++ b/meson.build @@ -77,6 +77,15 @@ else add_project_arguments('-DDISABLE_ORC=1', language: ['c', 'cpp']) endif +# NO Video support +disable_video = false + +if get_option('disable-video-support') + disable_audio = true + add_project_arguments('-DNO_VIDEO=1', language: ['c', 'cpp']) + message('Disable Video Type Support') +endif + # NO Audio support disable_audio = false diff --git a/meson_options.txt b/meson_options.txt index 15d4704..878e319 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -5,5 +5,6 @@ option('enable-tensorflow-lite', type: 'boolean', value: true) option('enable-tensorflow', type: 'boolean', value: true) option('enable-tensorflow-mem-optmz', type: 'boolean', value: true) option('install-example', type: 'boolean', value: false) +option('disable-video-support', type: 'boolean', value: false) option('disable-audio-support', type: 'boolean', value: false) option('enable-tizen-capi', type: 'boolean', value: false) -- 2.7.4