From 85400c813fe513aa6a91484c4e33caef8e700794 Mon Sep 17 00:00:00 2001 From: Gichan Jang Date: Fri, 19 Mar 2021 13:56:25 +0900 Subject: [PATCH] [Decoder] Add decoder cusotm operation It is similar to cusom-easy of the tensor filter. Get an input tensors and make output media stream. It can be used to create cusom media stream from tensors such as flexbuffer. Output capability is application/octet-stream. Signed-off-by: Gichan Jang --- debian/nnstreamer-dev.install | 1 + ext/nnstreamer/tensor_decoder/tensordec-flexbuf.cc | 21 ++++ gst/nnstreamer/include/meson.build | 1 + gst/nnstreamer/include/tensor_decoder_custom.h | 54 ++++++++++ gst/nnstreamer/nnstreamer_subplugin.c | 2 + gst/nnstreamer/nnstreamer_subplugin.h | 1 + gst/nnstreamer/tensor_decoder/tensordec.c | 112 ++++++++++++++++++--- gst/nnstreamer/tensor_decoder/tensordec.h | 10 ++ packaging/nnstreamer.spec | 1 + 9 files changed, 189 insertions(+), 14 deletions(-) create mode 100644 gst/nnstreamer/include/tensor_decoder_custom.h diff --git a/debian/nnstreamer-dev.install b/debian/nnstreamer-dev.install index 0f95eee..15dce14 100644 --- a/debian/nnstreamer-dev.install +++ b/debian/nnstreamer-dev.install @@ -8,6 +8,7 @@ /usr/include/nnstreamer/tensor_filter_custom_easy.h /usr/include/nnstreamer/tensor_converter_custom.h /usr/include/nnstreamer/tensor_typedef.h +/usr/include/nnstreamer/tensor_decoder_custom.h /usr/include/nnstreamer/tensor_filter_cpp.hh /usr/include/nnstreamer/nnstreamer_cppplugin_api_filter.hh /usr/lib/*/pkgconfig/nnstreamer.pc diff --git a/ext/nnstreamer/tensor_decoder/tensordec-flexbuf.cc b/ext/nnstreamer/tensor_decoder/tensordec-flexbuf.cc index 20642d5..938bff7 100644 --- a/ext/nnstreamer/tensor_decoder/tensordec-flexbuf.cc +++ b/ext/nnstreamer/tensor_decoder/tensordec-flexbuf.cc @@ -33,6 +33,27 @@ * Blob | * } * } + * + * If you want to convert tensors to your own binary format of the flexbuffers, + * You can use custom mode of the tensor decoder. + * This is an example of a callback type custom mode. + * @code + * // Define custom callback function + * int * tensor_decoder_custom_cb (const GstTensorMemory *input, + * const GstTensorsConfig *config, void * data, GstBuffer * out_buf) { + * // Write a code to convert tensors to flexbuffers. + * } + * + * ... + * // Register custom callback function + * nnstreamer_decoder_custom_register ("tdec", tensor_converter_custom_cb, NULL); + * ... + * // Use the custom tensor converter in a pipeline. + * // E.g., Pipeline of " ... (tensors) ! tensor_decoder mode=custom-code option1=tdec ! (flexbuffers)... " + * ... + * // After everything is done. + * nnstreamer_decoder_custom_unregister ("tdec"); + * @endcode */ diff --git a/gst/nnstreamer/include/meson.build b/gst/nnstreamer/include/meson.build index 8f40a1d..90f4b5c 100644 --- a/gst/nnstreamer/include/meson.build +++ b/gst/nnstreamer/include/meson.build @@ -5,6 +5,7 @@ nnst_common_headers = [ 'tensor_filter_custom.h', 'tensor_filter_custom_easy.h', 'tensor_converter_custom.h', + 'tensor_decoder_custom.h', 'nnstreamer_plugin_api_filter.h', 'nnstreamer_plugin_api_decoder.h', 'nnstreamer_plugin_api_converter.h', diff --git a/gst/nnstreamer/include/tensor_decoder_custom.h b/gst/nnstreamer/include/tensor_decoder_custom.h new file mode 100644 index 0000000..e265620 --- /dev/null +++ b/gst/nnstreamer/include/tensor_decoder_custom.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: LGPL-2.1-only */ +/** + * GStreamer/NNStreamer Tensor-Decoder + * Copyright (C) 2021 Gichan Jang + */ +/** + * @file tensor_decoder_custom.h + * @date 22 Mar 2021 + * @brief NNStreamer APIs for tensor_decoder custom condition + * @see https://github.com/nnstreamer/nnstreamer + * @author Gichan Jang + * @bug No known bugs except for NYI items + * + */ + +#ifndef __NNS_TENSOR_DECODER_CUSTOM_H__ +#define __NNS_TENSOR_DECODER_CUSTOM_H__ + +#include +#include +#include "tensor_typedef.h" + +G_BEGIN_DECLS +/** + * @brief Decode from tensors to media as customized operation + * @param[in] input the input memory containg tensors + * @param[in] config input tensors config + * @param[in] data private data for the callback + * @param[out] output buffer filled by user + * @return 0 if success. -ERRNO if error. + */ +typedef int (* tensor_decoder_custom) (const GstTensorMemory *input, + const GstTensorsConfig *config, void *data, GstBuffer * out_buf); + +/** + * @brief Register the custom callback function. + * @param[in] name The name of tensor_decoder custom callback function. + * @param[in] func The custom condition function body + * @param[in/out] data The internal data for the function + * @return 0 if success. -ERRNO if error. + */ +extern int +nnstreamer_decoder_custom_register (const gchar *name, tensor_decoder_custom func, void *data); + +/** + * @brief Unregister the custom callback function. + * @param[in] name The registered name of tensor_decoder custom callback function. + * @return 0 if success. -ERRNO if error. + */ +extern int +nnstreamer_decoder_custom_unregister (const gchar *name); + +G_END_DECLS +#endif /*__NNS_TENSOR_DECODER_CUSTOM_H__*/ diff --git a/gst/nnstreamer/nnstreamer_subplugin.c b/gst/nnstreamer/nnstreamer_subplugin.c index 5563f79..0937d57 100644 --- a/gst/nnstreamer/nnstreamer_subplugin.c +++ b/gst/nnstreamer/nnstreamer_subplugin.c @@ -73,6 +73,7 @@ static subpluginSearchLogic searchAlgorithm[] = { [NNS_EASY_CUSTOM_FILTER] = NNS_SEARCH_FILENAME, [NNS_SUBPLUGIN_CONVERTER] = NNS_SEARCH_GETALL, [NNS_CUSTOM_CONVERTER] = NNS_SEARCH_NO_OP, + [NNS_CUSTOM_DECODER] = NNS_SEARCH_NO_OP, [NNS_IF_CUSTOM] = NNS_SEARCH_NO_OP, [NNS_SUBPLUGIN_END] = NNS_SEARCH_NO_OP, }; @@ -182,6 +183,7 @@ register_subplugin (subpluginType type, const char *name, const void *data) case NNS_SUBPLUGIN_DECODER: case NNS_EASY_CUSTOM_FILTER: case NNS_SUBPLUGIN_CONVERTER: + case NNS_CUSTOM_DECODER: case NNS_IF_CUSTOM: case NNS_CUSTOM_CONVERTER: break; diff --git a/gst/nnstreamer/nnstreamer_subplugin.h b/gst/nnstreamer/nnstreamer_subplugin.h index d22662c..d3eb853 100644 --- a/gst/nnstreamer/nnstreamer_subplugin.h +++ b/gst/nnstreamer/nnstreamer_subplugin.h @@ -41,6 +41,7 @@ typedef enum { NNS_EASY_CUSTOM_FILTER = NNSCONF_PATH_EASY_CUSTOM_FILTERS, NNS_SUBPLUGIN_CONVERTER = NNSCONF_PATH_CONVERTERS, NNS_CUSTOM_CONVERTER, + NNS_CUSTOM_DECODER, NNS_IF_CUSTOM, NNS_SUBPLUGIN_END, diff --git a/gst/nnstreamer/tensor_decoder/tensordec.c b/gst/nnstreamer/tensor_decoder/tensordec.c index 1f564d8..1f9db51 100644 --- a/gst/nnstreamer/tensor_decoder/tensordec.c +++ b/gst/nnstreamer/tensor_decoder/tensordec.c @@ -242,6 +242,14 @@ gst_tensordec_media_caps_from_tensor (GstTensorDec * self, g_return_val_if_fail (config != NULL, NULL); if (self->decoder == NULL) { + if (self->is_custom) { + GstCaps *caps; + caps = gst_caps_from_string ("application/octet-stream"); + if (config->rate_n >= 0 && config->rate_d > 0) + gst_caps_set_simple (caps, "framerate", + GST_TYPE_FRACTION, config->rate_n, config->rate_d, NULL); + return caps; + } GST_ERROR_OBJECT (self, "Decoder plugin is not yet configured."); return NULL; } @@ -418,7 +426,9 @@ gst_tensordec_init (GstTensorDec * self) self->negotiated = FALSE; self->decoder = NULL; self->plugin_data = NULL; - + self->is_custom = FALSE; + self->custom.func = NULL; + self->custom.data = NULL; for (i = 0; i < TensorDecMaxOpNum; i++) self->option[i] = NULL; @@ -478,6 +488,11 @@ gst_tensordec_set_property (GObject * object, guint prop_id, guint i; mode_string = g_value_get_string (value); + if (g_ascii_strcasecmp (mode_string, "custom-code") == 0) { + self->is_custom = TRUE; + break; + } + decoder = nnstreamer_decoder_find (mode_string); /* See if we are using "plugin" */ @@ -512,7 +527,6 @@ gst_tensordec_set_property (GObject * object, guint prop_id, gst_tensor_decoder_clean_plugin (self); self->decoder = NULL; } - break; } PROP_MODE_OPTION (1); @@ -524,6 +538,7 @@ gst_tensordec_set_property (GObject * object, guint prop_id, PROP_MODE_OPTION (7); PROP_MODE_OPTION (8); PROP_MODE_OPTION (9); + default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -555,7 +570,9 @@ gst_tensordec_get_property (GObject * object, guint prop_id, g_value_set_boolean (value, self->silent); break; case PROP_MODE: - if (self->decoder) + if (self->is_custom) + g_value_set_string (value, "custom"); + else if (self->decoder) g_value_set_string (value, self->decoder->modename); else g_value_set_string (value, ""); @@ -599,6 +616,8 @@ gst_tensordec_class_finalize (GObject * object) for (i = 0; i < TensorDecMaxOpNum; ++i) { g_free (self->option[i]); } + self->custom.func = NULL; + self->custom.data = NULL; G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -626,7 +645,7 @@ gst_tensordec_configure (GstTensorDec * self, const GstCaps * in_caps, return FALSE; } - if (self->decoder == NULL) { + if (self->decoder == NULL && !self->is_custom) { GST_ERROR_OBJECT (self, "Decoder plugin is not yet configured."); return FALSE; } @@ -675,7 +694,7 @@ gst_tensordec_transform (GstBaseTransform * trans, if (G_UNLIKELY (!self->configured)) goto unknown_format; - if (self->decoder) { + if (self->decoder || self->is_custom) { GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT]; GstMapInfo in_info[NNS_TENSOR_SIZE_LIMIT]; GstTensorMemory input[NNS_TENSOR_SIZE_LIMIT]; @@ -699,9 +718,16 @@ gst_tensordec_transform (GstBaseTransform * trans, input[i].data = in_info[i].data; input[i].size = in_info[i].size; } - - res = self->decoder->decode (&self->plugin_data, &self->tensor_config, - input, outbuf); + if (!self->is_custom) { + res = self->decoder->decode (&self->plugin_data, &self->tensor_config, + input, outbuf); + } else if (self->custom.func != NULL) { + res = self->custom.func (input, &self->tensor_config, self->custom.data, + outbuf); + } else { + GST_ERROR_OBJECT (self, "Custom decoder callback is not registered."); + res = GST_FLOW_ERROR; + } for (i = 0; i < num_tensors; i++) gst_memory_unmap (in_mem[i], &in_info[i]); @@ -745,9 +771,25 @@ gst_tensordec_transform_caps (GstBaseTransform * trans, self = GST_TENSOR_DECODER_CAST (trans); /* Not ready */ - if (self->decoder == NULL) + if (self->decoder == NULL && !self->is_custom) return NULL; + if (self->is_custom) { + const decoder_custom_cb_s *ptr = NULL; + if (self->option[0] == NULL) { + nns_logw ("Tensor decoder custom option is not given."); + return NULL; + } + self->custom.func = NULL; + ptr = get_subplugin (NNS_CUSTOM_DECODER, self->option[0]); + if (!ptr) { + nns_logw ("Failed to find custom subplugin of the tensor_decoder"); + return NULL; + } + self->custom.func = ptr->func; + self->custom.data = ptr->data; + } + silent_debug ("Direction = %d\n", direction); silent_debug_caps (caps, "from"); silent_debug_caps (filter, "filter"); @@ -850,9 +892,7 @@ static gboolean gst_tensordec_set_caps (GstBaseTransform * trans, GstCaps * incaps, GstCaps * outcaps) { - GstTensorDec *self; - - self = GST_TENSOR_DECODER_CAST (trans); + GstTensorDec *self = GST_TENSOR_DECODER_CAST (trans); silent_debug_caps (incaps, "from incaps"); silent_debug_caps (outcaps, "from outcaps"); @@ -893,9 +933,8 @@ gst_tensordec_transform_size (GstBaseTransform * trans, self = GST_TENSOR_DECODER_CAST (trans); g_assert (self->configured); - g_assert (self->decoder); - if (self->decoder->getTransformSize) + if (!self->is_custom && self->decoder->getTransformSize) *othersize = self->decoder->getTransformSize (&self->plugin_data, &self->tensor_config, caps, size, othercaps, direction); else @@ -903,3 +942,48 @@ gst_tensordec_transform_size (GstBaseTransform * trans, return TRUE; } + +/** + * @brief Registers a callback for tensor_decoder custom condition + * @return 0 if success. -ERRNO if error. + */ +int +nnstreamer_decoder_custom_register (const gchar * name, + tensor_decoder_custom func, void *data) +{ + decoder_custom_cb_s *ptr; + + g_return_val_if_fail (name && strlen (name), -EINVAL); + g_return_val_if_fail (func, -EINVAL); + + if (!(ptr = g_try_new0 (decoder_custom_cb_s, 1))) + return -ENOMEM; + + ptr->func = func; + ptr->data = data; + + if (register_subplugin (NNS_CUSTOM_DECODER, name, ptr) == TRUE) + return 0; + + g_free (ptr); + return -EINVAL; +} + +/** + * @brief Unregisters a callback for tensor_decoder custom condition + * @return 0 if success. -ERRNO if error. + */ +int +nnstreamer_decoder_custom_unregister (const gchar * name) +{ + decoder_custom_cb_s *ptr; + + ptr = (decoder_custom_cb_s *) get_subplugin (NNS_CUSTOM_DECODER, name); + if (!unregister_subplugin (NNS_CUSTOM_DECODER, name)) { + ml_loge ("Failed to unregister custom callback %s.", name); + return -EINVAL; + } + g_free (ptr); + + return 0; +} diff --git a/gst/nnstreamer/tensor_decoder/tensordec.h b/gst/nnstreamer/tensor_decoder/tensordec.h index 03ba833..3934687 100644 --- a/gst/nnstreamer/tensor_decoder/tensordec.h +++ b/gst/nnstreamer/tensor_decoder/tensordec.h @@ -35,6 +35,7 @@ #include "tensor_common.h" #include "nnstreamer_subplugin.h" #include "nnstreamer_plugin_api_decoder.h" +#include "tensor_decoder_custom.h" G_BEGIN_DECLS @@ -52,6 +53,11 @@ G_BEGIN_DECLS typedef struct _GstTensorDec GstTensorDec; typedef struct _GstTensorDecClass GstTensorDecClass; +typedef struct +{ + tensor_decoder_custom func; + void * data; +} decoder_custom_cb_s; #define TensorDecMaxOpNum (9) @@ -71,6 +77,10 @@ struct _GstTensorDec gboolean configured; /**< TRUE if already successfully configured tensor metadata */ GstTensorsConfig tensor_config; /**< configured tensor info @todo support tensors in the future */ + /** For tensor decoder custom */ + gboolean is_custom; + decoder_custom_cb_s custom; + const GstTensorDecoderDef *decoder; /**< Plugin object */ void *plugin_data; }; diff --git a/packaging/nnstreamer.spec b/packaging/nnstreamer.spec index 3334151..59f5ec3 100644 --- a/packaging/nnstreamer.spec +++ b/packaging/nnstreamer.spec @@ -860,6 +860,7 @@ cp -r result %{buildroot}%{_datadir}/nnstreamer/unittest/ %{_includedir}/nnstreamer/tensor_filter_custom.h %{_includedir}/nnstreamer/tensor_filter_custom_easy.h %{_includedir}/nnstreamer/tensor_converter_custom.h +%{_includedir}/nnstreamer/tensor_decoder_custom.h %{_includedir}/nnstreamer/nnstreamer_plugin_api_filter.h %{_includedir}/nnstreamer/nnstreamer_plugin_api_decoder.h %{_includedir}/nnstreamer/nnstreamer_plugin_api_converter.h -- 2.7.4