Support Multi-Tensor for Input

Now, decoder plugins may have multi-tensors as their input stream.

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/gst/tensor_decoder/tensordec-directvideo.c b/gst/tensor_decoder/tensordec-directvideo.c
index 443724f..05b0da9 100644 (file)
--- a/gst/tensor_decoder/tensordec-directvideo.c
+++ b/gst/tensor_decoder/tensordec-directvideo.c
@@ -57,7 +57,7 @@ dv_setOption (GstTensorDec * self, int opNum, const gchar * param)
 
 /** @brief tensordec-plugin's TensorDecDef callback */
 static GstCaps *
-dv_getOutputDim (GstTensorDec * self, const GstTensorConfig * config)
+dv_getOutputDim (GstTensorDec * self, const GstTensorsConfig * config)
 {
   /* Old gst_tensordec_video_caps_from_config () had this */
   GstVideoFormat format;
@@ -65,10 +65,13 @@ dv_getOutputDim (GstTensorDec * self, const GstTensorConfig * config)
   GstCaps *caps;
 
   g_return_val_if_fail (config != NULL, NULL);
+  GST_ERROR ("Num Tensors = %d", config->info.num_tensors);
+  g_return_val_if_fail (config->info.num_tensors >= 1, NULL);
 
   caps = gst_caps_from_string (GST_TENSOR_VIDEO_CAPS_STR);
 
-  switch (config->info.dimension[0]) {
+  /* Direct video uses the first tensor only even if it's multi-tensor */
+  switch (config->info.info[0].dimension[0]) {
     case 1:
       format = GST_VIDEO_FORMAT_GRAY8;
       break;
@@ -83,9 +86,9 @@ dv_getOutputDim (GstTensorDec * self, const GstTensorConfig * config)
       break;
   }
 
-  width = config->info.dimension[1];
-  height = config->info.dimension[2];
-  fn = config->rate_n;
+  width = config->info.info[0].dimension[1];
+  height = config->info.info[0].dimension[2];
+  fn = config->rate_n; /** @todo Verify if this rate is ok */
   fd = config->rate_d;
 
   if (format != GST_VIDEO_FORMAT_UNKNOWN) {
@@ -121,8 +124,9 @@ static gsize
 dv_getTransformSize (GstTensorDec * self, GstCaps * caps,
     gsize size, GstCaps * othercaps, GstPadDirection direction)
 {
-  GstTensorConfig *config = &self->tensor_config;
-  uint32_t *dim = &(config->info.dimension[0]);
+  GstTensorsConfig *config = &self->tensor_config;
+  /* Direct video uses the first tensor only even if it's multi-tensor */
+  uint32_t *dim = &(config->info.info[0].dimension[0]);
 
   if (direction == GST_PAD_SINK)
     return _get_video_xraw_bufsize (dim);
@@ -137,15 +141,16 @@ dv_decode (GstTensorDec * self, const GstTensorMemory * input,
 {
   GstMapInfo out_info;
   GstMemory *out_mem;
-  GstTensorConfig *config = &self->tensor_config;
-  uint32_t *dim = &(config->info.dimension[0]);
+  GstTensorsConfig *config = &self->tensor_config;
+  /* Direct video uses the first tensor only even if it's multi-tensor */
+  uint32_t *dim = &(config->info.info[0].dimension[0]);
   size_t size = _get_video_xraw_bufsize (dim);
 
   g_assert (outbuf);
   if (gst_buffer_get_size (outbuf) > 0 && gst_buffer_get_size (outbuf) != size) {
     gst_buffer_set_size (outbuf, size);
   }
-  g_assert (config->info.type == _NNS_UINT8);
+  g_assert (config->info.info[0].type == _NNS_UINT8);
 
   if (gst_buffer_get_size (outbuf) == size) {
     /* Don't reallocate. Reuse what's already given */

diff --git a/gst/tensor_decoder/tensordec-imagelabel.c b/gst/tensor_decoder/tensordec-imagelabel.c
index 332da59..cd16eb8 100644 (file)
--- a/gst/tensor_decoder/tensordec-imagelabel.c
+++ b/gst/tensor_decoder/tensordec-imagelabel.c
@@ -169,14 +169,16 @@ _setOption (GstTensorDec * self, int opNum, const gchar * param)
 
 /** @brief tensordec-plugin's TensorDecDef callback */
 static GstCaps *
-_getOutputDim (GstTensorDec * self, const GstTensorConfig * config)
+_getOutputDim (GstTensorDec * self, const GstTensorsConfig * config)
 {
   const uint32_t *dim;
   int i;
 
   g_return_val_if_fail (config != NULL, NULL);
+  g_return_val_if_fail (config->info.num_tensors >= 1, NULL);
 
-  dim = config->info.dimension;
+  /* Even if it's multi-tensor, we use the first tensor only in image labeling */
+  dim = config->info.info[0].dimension;
   /* This allows N:1:1:1 only! */
   for (i = 1; i < NNS_TENSOR_RANK_LIMIT; i++)
     if (dim[i] != 1) {
@@ -226,7 +228,7 @@ _decode (GstTensorDec * self, const GstTensorMemory * input, GstBuffer * outbuf)
   GstMapInfo out_info;
   GstMemory *out_mem;
 
-  gsize bpe = tensor_element_size[self->tensor_config.info.type];
+  gsize bpe = tensor_element_size[self->tensor_config.info.info[0].type];
   tensor_element max_val;
   guint max_index = 0;
   gsize num_data;               /* Size / bpe */
@@ -239,9 +241,9 @@ _decode (GstTensorDec * self, const GstTensorMemory * input, GstBuffer * outbuf)
   g_assert (outbuf);
 
   input_data = input->data;
-  num_data = gst_tensor_info_get_size (&self->tensor_config.info) / bpe;
+  num_data = gst_tensor_info_get_size (&self->tensor_config.info.info[0]) / bpe;
 
-  switch (self->tensor_config.info.type) {
+  switch (self->tensor_config.info.info[0].type) {
       search_max_case (int32_t, _NNS_INT32);
       search_max_case (uint32_t, _NNS_UINT32);
       search_max_case (int16_t, _NNS_INT16);

diff --git a/gst/tensor_decoder/tensordec.c b/gst/tensor_decoder/tensordec.c
index 50d8ba3..72aec9e 100644 (file)
--- a/gst/tensor_decoder/tensordec.c
+++ b/gst/tensor_decoder/tensordec.c
@@ -103,12 +103,17 @@ enum
 #define DEFAULT_SILENT TRUE
 
 /**
+ * @brief Support multi-tensor along with single-tensor as the input
+ */
+#define CAPS_STRING GST_TENSOR_CAP_DEFAULT "; " GST_TENSORS_CAP_DEFAULT
+
+/**
  * @brief The capabilities of the inputs
  */
 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
     GST_PAD_SINK,
     GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT));
+    GST_STATIC_CAPS (CAPS_STRING));
 
 /**
  * @brief The capabilities of the outputs
@@ -148,7 +153,7 @@ static gboolean gst_tensordec_transform_size (GstBaseTransform * trans,
  */
 static GstCaps *
 gst_tensordec_media_caps_from_tensor (GstTensorDec * self,
-    const GstTensorConfig * config)
+    const GstTensorsConfig * config)
 {
   g_return_val_if_fail (config != NULL, NULL);
 
@@ -170,10 +175,10 @@ static GstCaps *
 gst_tensordec_media_caps_from_structure (GstTensorDec * self,
     const GstStructure * structure)
 {
-  GstTensorConfig config;
+  GstTensorsConfig config;
   GstCaps *result = NULL;
 
-  if (gst_tensor_config_from_structure (&config, structure)) {
+  if (gst_tensors_config_from_structure (&config, structure)) {
     result = gst_tensordec_media_caps_from_tensor (self, &config);
   }
 
@@ -191,13 +196,13 @@ gst_tensordec_media_caps_from_structure (GstTensorDec * self,
  * @param t_info newly configured tensor metadata
  */
 static gboolean
-gst_tensordec_check_consistency (GstTensorDec * self, GstTensorConfig * config)
+gst_tensordec_check_consistency (GstTensorDec * self, GstTensorsConfig * config)
 {
   g_return_val_if_fail (self != NULL, FALSE);
   g_return_val_if_fail (config != NULL, FALSE);
 
   if (self->configured) {
-    return gst_tensor_config_is_equal (&self->tensor_config, config);
+    return gst_tensors_config_is_equal (&self->tensor_config, config);
   }
 
   /** not configured yet */
@@ -301,7 +306,7 @@ gst_tensordec_init (GstTensorDec * self)
   self->option[0] = NULL;
   self->option[1] = NULL;
   self->decoder = NULL;
-  gst_tensor_config_init (&self->tensor_config);
+  gst_tensors_config_init (&self->tensor_config);
 }
 
 /**
@@ -453,17 +458,17 @@ static gboolean
 gst_tensordec_configure (GstTensorDec * self, const GstCaps * caps)
 {
   GstStructure *structure;
-  GstTensorConfig config;
+  GstTensorsConfig config;
 
   /** This caps is coming from tensor */
   structure = gst_caps_get_structure (caps, 0);
 
-  if (!gst_tensor_config_from_structure (&config, structure)) {
+  if (!gst_tensors_config_from_structure (&config, structure)) {
     err_print ("Cannot configure tensor from structure");
     return FALSE;
   }
 
-  if (!gst_tensor_config_validate (&config)) {
+  if (!gst_tensors_config_validate (&config)) {
     err_print ("Not configured yet");
     return FALSE;
   }
@@ -510,21 +515,27 @@ gst_tensordec_transform (GstBaseTransform * trans,
     goto unknown_format;
 
   if (self->mode == DECODE_MODE_PLUGIN) {
-    /** @todo Supporting multi-tensor will require significant changes */
-    GstMemory *in_mem;
-    GstMapInfo in_info;
-    GstTensorMemory input;
+    int num_tensors = self->tensor_config.info.num_tensors;
+    int i;
+    GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT];
+    GstMapInfo in_info[NNS_TENSOR_SIZE_LIMIT];
+    GstTensorMemory input[NNS_TENSOR_SIZE_LIMIT];
 
-    in_mem = gst_buffer_peek_memory (inbuf, 0);   /** @todo support multi-tensor! */
-    g_assert (gst_memory_map (in_mem, &in_info, GST_MAP_READ));
+    g_assert (gst_buffer_n_memory (inbuf) == num_tensors);
 
-    input.data = in_info.data;
-    input.size = in_info.size;
-    input.type = self->tensor_config.info.type;
+    for (i = 0; i < num_tensors; i++) {
+      in_mem[i] = gst_buffer_peek_memory (inbuf, i);
+      g_assert (gst_memory_map (in_mem[i], &in_info[i], GST_MAP_READ));
 
-    res = self->decoder->decode (self, &input, outbuf);
+      input[i].data = in_info[i].data;
+      input[i].size = in_info[i].size;
+      input[i].type = self->tensor_config.info.info[i].type;
+    }
+
+    res = self->decoder->decode (self, input, outbuf);
 
-    gst_memory_unmap (in_mem, &in_info);
+    for (i = 0; i < num_tensors; i++)
+      gst_memory_unmap (in_mem[i], &in_info[i]);
   } else {
     GST_ERROR ("Decoder plugin not yet configured.");
     goto unknown_type;
@@ -670,10 +681,22 @@ gst_tensordec_set_caps (GstBaseTransform * trans,
   silent_debug_caps (incaps, "from incaps");
   silent_debug_caps (outcaps, "from outcaps");
 
-  /** @todo Check if outcaps == getOutputDim (incaps) */
+  if (gst_tensordec_configure (self, incaps)) {
+    GstCaps *supposed = gst_tensordec_media_caps_from_tensor (self,
+        &self->tensor_config);
 
-  return TRUE;
+    /** Check if outcaps ==equivalent== supposed */
+    if (!gst_caps_is_always_compatible (outcaps, supposed)) {
+      GST_ERROR ("This is not compatible with the supposed output pad cap");
+      gst_caps_unref (supposed);
+      return FALSE;
+    }
+    gst_caps_unref (supposed);
+  } else {
+    return FALSE;
+  }
 
+  return TRUE;
 }
 
 /**

diff --git a/gst/tensor_decoder/tensordec.h b/gst/tensor_decoder/tensordec.h
index 14862ef..ad84847 100644 (file)
--- a/gst/tensor_decoder/tensordec.h
+++ b/gst/tensor_decoder/tensordec.h
@@ -69,7 +69,7 @@ struct _GstTensorDec
   gboolean configured; /**< TRUE if already successfully configured tensor metadata */
   void *plugin_data;
   void (*cleanup_plugin_data)(GstTensorDec *self); /**< exit() of subplugin is registered here. If it's null, gfree(plugin_data) is used. */
-  GstTensorConfig tensor_config; /**< configured tensor info @todo support tensors in the future */
+  GstTensorsConfig tensor_config; /**< configured tensor info @todo support tensors in the future */
 
   TensorDecDef *decoder; /**< Plugin object */
 };
@@ -141,7 +141,7 @@ struct _TensorDecDef
       /**< Object destruction for the decoder */
   gboolean (*setOption) (GstTensorDec *self, int opNum, const gchar *param);
       /**< Process with the given options. It can be called repeatedly */
-  GstCaps *(*getOutputDim) (GstTensorDec *self, const GstTensorConfig *config);
+  GstCaps *(*getOutputDim) (GstTensorDec *self, const GstTensorsConfig *config);
       /**< The caller should unref the returned GstCaps
         * Current implementation supports single-tensor only.
         * @todo WIP: support multi-tensor for input!!!