[Decoder] Add decoder cusotm operation
authorGichan Jang <gichan2.jang@samsung.com>
Fri, 19 Mar 2021 04:56:25 +0000 (13:56 +0900)
committerMyungJoo Ham <myungjoo.ham@samsung.com>
Mon, 29 Mar 2021 06:36:26 +0000 (15:36 +0900)
It is similar to cusom-easy of the tensor filter.
Get an input tensors and make output media stream.
It can be used to create cusom media stream from tensors such as flexbuffer.
Output capability is application/octet-stream.

Signed-off-by: Gichan Jang <gichan2.jang@samsung.com>
debian/nnstreamer-dev.install
ext/nnstreamer/tensor_decoder/tensordec-flexbuf.cc
gst/nnstreamer/include/meson.build
gst/nnstreamer/include/tensor_decoder_custom.h [new file with mode: 0644]
gst/nnstreamer/nnstreamer_subplugin.c
gst/nnstreamer/nnstreamer_subplugin.h
gst/nnstreamer/tensor_decoder/tensordec.c
gst/nnstreamer/tensor_decoder/tensordec.h
packaging/nnstreamer.spec

index 0f95eee..15dce14 100644 (file)
@@ -8,6 +8,7 @@
 /usr/include/nnstreamer/tensor_filter_custom_easy.h
 /usr/include/nnstreamer/tensor_converter_custom.h
 /usr/include/nnstreamer/tensor_typedef.h
+/usr/include/nnstreamer/tensor_decoder_custom.h
 /usr/include/nnstreamer/tensor_filter_cpp.hh
 /usr/include/nnstreamer/nnstreamer_cppplugin_api_filter.hh
 /usr/lib/*/pkgconfig/nnstreamer.pc
index 20642d5..938bff7 100644 (file)
  *                          Blob | <tensor data>
  *                         }
  * }
+ *
+ * If you want to convert tensors to your own binary format of the flexbuffers,
+ * You can use custom mode of the tensor decoder.
+ * This is an example of a callback type custom mode.
+ * @code
+ * // Define custom callback function
+ * int * tensor_decoder_custom_cb (const GstTensorMemory *input,
+ *   const GstTensorsConfig *config, void * data, GstBuffer * out_buf) {
+ *   // Write a code to convert tensors to flexbuffers.
+ * }
+ *
+ * ...
+ * // Register custom callback function
+ * nnstreamer_decoder_custom_register ("tdec", tensor_converter_custom_cb, NULL);
+ * ...
+ * // Use the custom tensor converter in a pipeline.
+ * // E.g., Pipeline of " ... (tensors) ! tensor_decoder mode=custom-code option1=tdec ! (flexbuffers)... "
+ * ...
+ * // After everything is done.
+ * nnstreamer_decoder_custom_unregister ("tdec");
+ * @endcode
  */
 
 
index 8f40a1d..90f4b5c 100644 (file)
@@ -5,6 +5,7 @@ nnst_common_headers = [
   'tensor_filter_custom.h',
   'tensor_filter_custom_easy.h',
   'tensor_converter_custom.h',
+  'tensor_decoder_custom.h',
   'nnstreamer_plugin_api_filter.h',
   'nnstreamer_plugin_api_decoder.h',
   'nnstreamer_plugin_api_converter.h',
diff --git a/gst/nnstreamer/include/tensor_decoder_custom.h b/gst/nnstreamer/include/tensor_decoder_custom.h
new file mode 100644 (file)
index 0000000..e265620
--- /dev/null
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: LGPL-2.1-only */\r
+/**\r
+ * GStreamer/NNStreamer Tensor-Decoder\r
+ * Copyright (C) 2021 Gichan Jang <gichan2.jang@samsung.com>\r
+ */\r
+/**\r
+ * @file       tensor_decoder_custom.h\r
+ * @date       22 Mar 2021\r
+ * @brief      NNStreamer APIs for tensor_decoder custom condition\r
+ * @see                https://github.com/nnstreamer/nnstreamer\r
+ * @author     Gichan Jang <gichan2.jang@samsung.com>\r
+ * @bug                No known bugs except for NYI items\r
+ *\r
+ */\r
+\r
+#ifndef __NNS_TENSOR_DECODER_CUSTOM_H__\r
+#define __NNS_TENSOR_DECODER_CUSTOM_H__\r
+\r
+#include <glib.h>\r
+#include <gst/gst.h>\r
+#include "tensor_typedef.h"\r
+\r
+G_BEGIN_DECLS\r
+/**\r
+ * @brief Decode from tensors to media as customized operation\r
+ * @param[in] input the input memory containg tensors\r
+ * @param[in] config input tensors config\r
+ * @param[in] data private data for the callback\r
+ * @param[out] output buffer filled by user\r
+ * @return 0 if success. -ERRNO if error.\r
+ */\r
+typedef int (* tensor_decoder_custom) (const GstTensorMemory *input,\r
+    const GstTensorsConfig *config, void *data, GstBuffer * out_buf);\r
+\r
+/**\r
+ * @brief Register the custom callback function.\r
+ * @param[in] name The name of tensor_decoder custom callback function.\r
+ * @param[in] func The custom condition function body\r
+ * @param[in/out] data The internal data for the function\r
+ * @return 0 if success. -ERRNO if error.\r
+ */\r
+extern int\r
+nnstreamer_decoder_custom_register (const gchar *name, tensor_decoder_custom func, void *data);\r
+\r
+/**\r
+ * @brief Unregister the custom callback function.\r
+ * @param[in] name The registered name of tensor_decoder custom callback function.\r
+ * @return 0 if success. -ERRNO if error.\r
+ */\r
+extern int\r
+nnstreamer_decoder_custom_unregister (const gchar *name);\r
+\r
+G_END_DECLS\r
+#endif /*__NNS_TENSOR_DECODER_CUSTOM_H__*/\r
index 5563f79..0937d57 100644 (file)
@@ -73,6 +73,7 @@ static subpluginSearchLogic searchAlgorithm[] = {
   [NNS_EASY_CUSTOM_FILTER] = NNS_SEARCH_FILENAME,
   [NNS_SUBPLUGIN_CONVERTER] = NNS_SEARCH_GETALL,
   [NNS_CUSTOM_CONVERTER] = NNS_SEARCH_NO_OP,
+  [NNS_CUSTOM_DECODER] = NNS_SEARCH_NO_OP,
   [NNS_IF_CUSTOM] = NNS_SEARCH_NO_OP,
   [NNS_SUBPLUGIN_END] = NNS_SEARCH_NO_OP,
 };
@@ -182,6 +183,7 @@ register_subplugin (subpluginType type, const char *name, const void *data)
     case NNS_SUBPLUGIN_DECODER:
     case NNS_EASY_CUSTOM_FILTER:
     case NNS_SUBPLUGIN_CONVERTER:
+    case NNS_CUSTOM_DECODER:
     case NNS_IF_CUSTOM:
     case NNS_CUSTOM_CONVERTER:
       break;
index d22662c..d3eb853 100644 (file)
@@ -41,6 +41,7 @@ typedef enum {
   NNS_EASY_CUSTOM_FILTER = NNSCONF_PATH_EASY_CUSTOM_FILTERS,
   NNS_SUBPLUGIN_CONVERTER = NNSCONF_PATH_CONVERTERS,
   NNS_CUSTOM_CONVERTER,
+  NNS_CUSTOM_DECODER,
   NNS_IF_CUSTOM,
 
   NNS_SUBPLUGIN_END,
index 1f564d8..1f9db51 100644 (file)
@@ -242,6 +242,14 @@ gst_tensordec_media_caps_from_tensor (GstTensorDec * self,
   g_return_val_if_fail (config != NULL, NULL);
 
   if (self->decoder == NULL) {
+    if (self->is_custom) {
+      GstCaps *caps;
+      caps = gst_caps_from_string ("application/octet-stream");
+      if (config->rate_n >= 0 && config->rate_d > 0)
+        gst_caps_set_simple (caps, "framerate",
+            GST_TYPE_FRACTION, config->rate_n, config->rate_d, NULL);
+      return caps;
+    }
     GST_ERROR_OBJECT (self, "Decoder plugin is not yet configured.");
     return NULL;
   }
@@ -418,7 +426,9 @@ gst_tensordec_init (GstTensorDec * self)
   self->negotiated = FALSE;
   self->decoder = NULL;
   self->plugin_data = NULL;
-
+  self->is_custom = FALSE;
+  self->custom.func = NULL;
+  self->custom.data = NULL;
   for (i = 0; i < TensorDecMaxOpNum; i++)
     self->option[i] = NULL;
 
@@ -478,6 +488,11 @@ gst_tensordec_set_property (GObject * object, guint prop_id,
       guint i;
 
       mode_string = g_value_get_string (value);
+      if (g_ascii_strcasecmp (mode_string, "custom-code") == 0) {
+        self->is_custom = TRUE;
+        break;
+      }
+
       decoder = nnstreamer_decoder_find (mode_string);
 
       /* See if we are using "plugin" */
@@ -512,7 +527,6 @@ gst_tensordec_set_property (GObject * object, guint prop_id,
         gst_tensor_decoder_clean_plugin (self);
         self->decoder = NULL;
       }
-
       break;
     }
       PROP_MODE_OPTION (1);
@@ -524,6 +538,7 @@ gst_tensordec_set_property (GObject * object, guint prop_id,
       PROP_MODE_OPTION (7);
       PROP_MODE_OPTION (8);
       PROP_MODE_OPTION (9);
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -555,7 +570,9 @@ gst_tensordec_get_property (GObject * object, guint prop_id,
       g_value_set_boolean (value, self->silent);
       break;
     case PROP_MODE:
-      if (self->decoder)
+      if (self->is_custom)
+        g_value_set_string (value, "custom");
+      else if (self->decoder)
         g_value_set_string (value, self->decoder->modename);
       else
         g_value_set_string (value, "");
@@ -599,6 +616,8 @@ gst_tensordec_class_finalize (GObject * object)
   for (i = 0; i < TensorDecMaxOpNum; ++i) {
     g_free (self->option[i]);
   }
+  self->custom.func = NULL;
+  self->custom.data = NULL;
 
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
@@ -626,7 +645,7 @@ gst_tensordec_configure (GstTensorDec * self, const GstCaps * in_caps,
     return FALSE;
   }
 
-  if (self->decoder == NULL) {
+  if (self->decoder == NULL && !self->is_custom) {
     GST_ERROR_OBJECT (self, "Decoder plugin is not yet configured.");
     return FALSE;
   }
@@ -675,7 +694,7 @@ gst_tensordec_transform (GstBaseTransform * trans,
   if (G_UNLIKELY (!self->configured))
     goto unknown_format;
 
-  if (self->decoder) {
+  if (self->decoder || self->is_custom) {
     GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT];
     GstMapInfo in_info[NNS_TENSOR_SIZE_LIMIT];
     GstTensorMemory input[NNS_TENSOR_SIZE_LIMIT];
@@ -699,9 +718,16 @@ gst_tensordec_transform (GstBaseTransform * trans,
       input[i].data = in_info[i].data;
       input[i].size = in_info[i].size;
     }
-
-    res = self->decoder->decode (&self->plugin_data, &self->tensor_config,
-        input, outbuf);
+    if (!self->is_custom) {
+      res = self->decoder->decode (&self->plugin_data, &self->tensor_config,
+          input, outbuf);
+    } else if (self->custom.func != NULL) {
+      res = self->custom.func (input, &self->tensor_config, self->custom.data,
+          outbuf);
+    } else {
+      GST_ERROR_OBJECT (self, "Custom decoder callback is not registered.");
+      res = GST_FLOW_ERROR;
+    }
 
     for (i = 0; i < num_tensors; i++)
       gst_memory_unmap (in_mem[i], &in_info[i]);
@@ -745,9 +771,25 @@ gst_tensordec_transform_caps (GstBaseTransform * trans,
   self = GST_TENSOR_DECODER_CAST (trans);
 
   /* Not ready */
-  if (self->decoder == NULL)
+  if (self->decoder == NULL && !self->is_custom)
     return NULL;
 
+  if (self->is_custom) {
+    const decoder_custom_cb_s *ptr = NULL;
+    if (self->option[0] == NULL) {
+      nns_logw ("Tensor decoder custom option is not given.");
+      return NULL;
+    }
+    self->custom.func = NULL;
+    ptr = get_subplugin (NNS_CUSTOM_DECODER, self->option[0]);
+    if (!ptr) {
+      nns_logw ("Failed to find custom subplugin of the tensor_decoder");
+      return NULL;
+    }
+    self->custom.func = ptr->func;
+    self->custom.data = ptr->data;
+  }
+
   silent_debug ("Direction = %d\n", direction);
   silent_debug_caps (caps, "from");
   silent_debug_caps (filter, "filter");
@@ -850,9 +892,7 @@ static gboolean
 gst_tensordec_set_caps (GstBaseTransform * trans,
     GstCaps * incaps, GstCaps * outcaps)
 {
-  GstTensorDec *self;
-
-  self = GST_TENSOR_DECODER_CAST (trans);
+  GstTensorDec *self = GST_TENSOR_DECODER_CAST (trans);
 
   silent_debug_caps (incaps, "from incaps");
   silent_debug_caps (outcaps, "from outcaps");
@@ -893,9 +933,8 @@ gst_tensordec_transform_size (GstBaseTransform * trans,
   self = GST_TENSOR_DECODER_CAST (trans);
 
   g_assert (self->configured);
-  g_assert (self->decoder);
 
-  if (self->decoder->getTransformSize)
+  if (!self->is_custom && self->decoder->getTransformSize)
     *othersize = self->decoder->getTransformSize (&self->plugin_data,
         &self->tensor_config, caps, size, othercaps, direction);
   else
@@ -903,3 +942,48 @@ gst_tensordec_transform_size (GstBaseTransform * trans,
 
   return TRUE;
 }
+
+/**
+ * @brief Registers a callback for tensor_decoder custom condition
+ * @return 0 if success. -ERRNO if error.
+ */
+int
+nnstreamer_decoder_custom_register (const gchar * name,
+    tensor_decoder_custom func, void *data)
+{
+  decoder_custom_cb_s *ptr;
+
+  g_return_val_if_fail (name && strlen (name), -EINVAL);
+  g_return_val_if_fail (func, -EINVAL);
+
+  if (!(ptr = g_try_new0 (decoder_custom_cb_s, 1)))
+    return -ENOMEM;
+
+  ptr->func = func;
+  ptr->data = data;
+
+  if (register_subplugin (NNS_CUSTOM_DECODER, name, ptr) == TRUE)
+    return 0;
+
+  g_free (ptr);
+  return -EINVAL;
+}
+
+/**
+ * @brief Unregisters a callback for tensor_decoder custom condition
+ * @return 0 if success. -ERRNO if error.
+ */
+int
+nnstreamer_decoder_custom_unregister (const gchar * name)
+{
+  decoder_custom_cb_s *ptr;
+
+  ptr = (decoder_custom_cb_s *) get_subplugin (NNS_CUSTOM_DECODER, name);
+  if (!unregister_subplugin (NNS_CUSTOM_DECODER, name)) {
+    ml_loge ("Failed to unregister custom callback %s.", name);
+    return -EINVAL;
+  }
+  g_free (ptr);
+
+  return 0;
+}
index 03ba833..3934687 100644 (file)
@@ -35,6 +35,7 @@
 #include "tensor_common.h"
 #include "nnstreamer_subplugin.h"
 #include "nnstreamer_plugin_api_decoder.h"
+#include "tensor_decoder_custom.h"
 
 G_BEGIN_DECLS
 
@@ -52,6 +53,11 @@ G_BEGIN_DECLS
 
 typedef struct _GstTensorDec GstTensorDec;
 typedef struct _GstTensorDecClass GstTensorDecClass;
+typedef struct
+{
+  tensor_decoder_custom func;
+  void * data;
+} decoder_custom_cb_s;
 
 #define TensorDecMaxOpNum (9)
 
@@ -71,6 +77,10 @@ struct _GstTensorDec
   gboolean configured; /**< TRUE if already successfully configured tensor metadata */
   GstTensorsConfig tensor_config; /**< configured tensor info @todo support tensors in the future */
 
+  /** For tensor decoder custom */
+  gboolean is_custom;
+  decoder_custom_cb_s custom;
+
   const GstTensorDecoderDef *decoder; /**< Plugin object */
   void *plugin_data;
 };
index 3334151..59f5ec3 100644 (file)
@@ -860,6 +860,7 @@ cp -r result %{buildroot}%{_datadir}/nnstreamer/unittest/
 %{_includedir}/nnstreamer/tensor_filter_custom.h
 %{_includedir}/nnstreamer/tensor_filter_custom_easy.h
 %{_includedir}/nnstreamer/tensor_converter_custom.h
+%{_includedir}/nnstreamer/tensor_decoder_custom.h
 %{_includedir}/nnstreamer/nnstreamer_plugin_api_filter.h
 %{_includedir}/nnstreamer/nnstreamer_plugin_api_decoder.h
 %{_includedir}/nnstreamer/nnstreamer_plugin_api_converter.h