[Converter] support byte-stream

add new properties input-dim and input-type to get tensor info from byte-stream (application/octet-stream)
add testcase for byte-stream

Signed-off-by: Jaeyun Jung <jy1210.jung@samsung.com>

diff --git a/gst/tensor_converter/tensor_converter.c b/gst/tensor_converter/tensor_converter.c
index 5e9d926..4cb29a2 100644 (file)
--- a/gst/tensor_converter/tensor_converter.c
+++ b/gst/tensor_converter/tensor_converter.c
@@ -104,6 +104,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_tensor_converter_debug);
 enum
 {
   PROP_0,
+  PROP_INPUT_DIMENSION,
+  PROP_INPUT_TYPE,
   PROP_FRAMES_PER_TENSOR,
   PROP_SILENT
 };
@@ -178,6 +180,32 @@ gst_tensor_converter_class_init (GstTensorConverterClass * klass)
   object_class->finalize = gst_tensor_converter_finalize;
 
   /**
+   * GstTensorConverter::input-dim:
+   *
+   * Input tensor dimension from inner array.
+   * Generally this property is used to set tensor configuration for byte-stream (application/octet-stream).
+   * When setting this property and input media type is video or audio stream, GstTensorConverter will compare the media info with this.
+   * (If it is different, it will be failed.)
+   */
+  g_object_class_install_property (object_class, PROP_INPUT_DIMENSION,
+      g_param_spec_string ("input-dim", "Input tensor dimension",
+          "Input tensor dimension from inner array", "",
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  /**
+   * GstTensorConverter::input-type:
+   *
+   * Type of each element of the input tensor.
+   * Generally this property is used to set tensor configuration for byte-stream (application/octet-stream).
+   * When setting this property and input media type is video or audio stream, GstTensorConverter will compare the media info with this.
+   * (If it is different, it will be failed.)
+   */
+  g_object_class_install_property (object_class, PROP_INPUT_TYPE,
+      g_param_spec_string ("input-type", "Input tensor type",
+          "Type of each element of the input tensor", "",
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  /**
    * GstTensorConverter::frames-per-tensor:
    *
    * The number of frames in outgoing buffer. (buffer is a sinle tensor instance)
@@ -241,6 +269,7 @@ gst_tensor_converter_init (GstTensorConverter * self)
   self->frames_per_tensor = DEFAULT_FRAMES_PER_TENSOR;
   self->in_media_type = _NNS_MEDIA_END;
   self->remove_padding = FALSE;
+  gst_tensor_info_init (&self->tensor_info);
 
   self->adapter = gst_adapter_new ();
   gst_tensor_converter_reset (self);
@@ -278,6 +307,14 @@ gst_tensor_converter_set_property (GObject * object, guint prop_id,
   self = GST_TENSOR_CONVERTER (object);
 
   switch (prop_id) {
+    case PROP_INPUT_DIMENSION:
+      g_assert (get_tensor_dimension (g_value_get_string (value),
+              self->tensor_info.dimension) > 0);
+      break;
+    case PROP_INPUT_TYPE:
+      self->tensor_info.type = get_tensor_type (g_value_get_string (value));
+      g_assert (self->tensor_info.type != _NNS_END);
+      break;
     case PROP_FRAMES_PER_TENSOR:
       self->frames_per_tensor = g_value_get_uint (value);
       silent_debug ("Set frames in output = %d", self->frames_per_tensor);
@@ -304,6 +341,19 @@ gst_tensor_converter_get_property (GObject * object, guint prop_id,
   self = GST_TENSOR_CONVERTER (object);
 
   switch (prop_id) {
+    case PROP_INPUT_DIMENSION:
+    {
+      gchar *str_dim;
+
+      str_dim = get_tensor_dimension_string (self->tensor_info.dimension);
+      g_value_set_string (value, str_dim);
+      g_free (str_dim);
+      break;
+    }
+    case PROP_INPUT_TYPE:
+      g_value_set_string (value,
+          tensor_element_typename[self->tensor_info.type]);
+      break;
     case PROP_FRAMES_PER_TENSOR:
       g_value_set_uint (value, self->frames_per_tensor);
       break;
@@ -557,6 +607,13 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
       }
       break;
 
+    case _NNS_OCTET:
+      /** get frame size from the properties */
+      frame_size = gst_tensor_info_get_size (&config->info);
+      g_assert ((buf_size % frame_size) == 0);
+      frames_in = buf_size / frame_size;
+      break;
+
     default:
       err_print ("Unsupported type %d\n", self->in_media_type);
       g_assert (0);
@@ -774,6 +831,17 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self,
     case _NNS_STRING:
       frames_dim = 1;
       break;
+    case _NNS_OCTET:
+      /**
+       * update tensor info from properties
+       */
+      if (!gst_tensor_info_validate (&self->tensor_info)) {
+        err_print ("Failed to get tensor info, update dimension and type.\n");
+        return FALSE;
+      }
+
+      config.info = self->tensor_info;
+      break;
     default:
       err_print ("Unsupported type %d\n", in_type);
       return FALSE;
@@ -790,9 +858,18 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self,
     return FALSE;
   }
 
+  if (gst_tensor_info_validate (&self->tensor_info)) {
+    /** compare tensor info */
+    if (!gst_tensor_info_is_equal (&self->tensor_info, &config.info)) {
+      err_print ("Failed, mismatched tensor info.\n");
+      return FALSE;
+    }
+  }
+
   self->in_media_type = in_type;
   self->tensor_configured = TRUE;
   self->tensor_config = config;
+  self->tensor_info = config.info;
   return TRUE;
 }
 

diff --git a/gst/tensor_converter/tensor_converter.h b/gst/tensor_converter/tensor_converter.h
index 7739a92..9eec49f 100644 (file)
--- a/gst/tensor_converter/tensor_converter.h
+++ b/gst/tensor_converter/tensor_converter.h
@@ -68,6 +68,7 @@ struct _GstTensorConverter
 
   gboolean silent; /**< true to print minimized log */
   guint frames_per_tensor; /**< number of frames in output tensor */
+  GstTensorInfo tensor_info; /**< data structure to get/set tensor info */
 
   GstAdapter *adapter; /**< adapt incoming media stream */
 

diff --git a/tests/nnstreamer_sink/unittest_sink.cpp b/tests/nnstreamer_sink/unittest_sink.cpp
index 44caca2..fcab6ff 100644 (file)
--- a/tests/nnstreamer_sink/unittest_sink.cpp
+++ b/tests/nnstreamer_sink/unittest_sink.cpp
@@ -70,6 +70,8 @@ typedef enum
   TEST_TYPE_AUDIO_F64, /**< pipeline for audio (F64) */
   TEST_TYPE_TEXT, /**< pipeline for text */
   TEST_TYPE_TEXT_3F, /**< pipeline for text 3 frames */
+  TEST_TYPE_OCTET_1, /**< pipeline for octet stream */
+  TEST_TYPE_OCTET_2, /**< pipeline for octet stream */
   TEST_TYPE_TENSORS, /**< pipeline for tensors with tensor_mux */
   TEST_TYPE_TENSORS_MIX, /**< pipeline for tensors with tensor_mux, tensor_demux */
   TEST_TYPE_CUSTOM_TENSOR, /**< pipeline for single tensor with passthrough custom filter */
@@ -197,6 +199,7 @@ _new_data_cb (GstElement * element, GstBuffer * buffer, gpointer user_data)
 {
   gsize buf_size = gst_buffer_get_size (buffer);
   guint mem_blocks = gst_buffer_n_memory (buffer);
+  GstClockTime timestamp = GST_BUFFER_DTS_OR_PTS (buffer);
 
   if (g_test_data.received > 0) {
     if (g_test_data.mem_blocks != mem_blocks) {
@@ -212,13 +215,6 @@ _new_data_cb (GstElement * element, GstBuffer * buffer, gpointer user_data)
     }
   }
 
-  g_test_data.received++;
-  g_test_data.received_size = buf_size;
-  g_test_data.mem_blocks = mem_blocks;
-
-  _print_log ("new data callback [%d] size [%zd]",
-      g_test_data.received, g_test_data.received_size);
-
   if (DBG) {
     GstClockTime pts, dts;
 
@@ -230,10 +226,17 @@ _new_data_cb (GstElement * element, GstBuffer * buffer, gpointer user_data)
   }
 
   /** check timestamp */
-  if (!GST_CLOCK_TIME_IS_VALID (GST_BUFFER_DTS_OR_PTS (buffer))) {
+  if (!GST_CLOCK_TIME_IS_VALID (timestamp)) {
     g_test_data.invalid_timestamp = TRUE;
   }
 
+  g_test_data.received++;
+  g_test_data.received_size = buf_size;
+  g_test_data.mem_blocks = mem_blocks;
+
+  _print_log ("new data callback [%d] size [%zd]",
+      g_test_data.received, g_test_data.received_size);
+
   if (g_test_data.caps_name == NULL) {
     GstPad *sink_pad;
     GstCaps *caps;
@@ -493,9 +496,23 @@ _setup_pipeline (TestOption & option)
       /** text stream 3 frames */
       str_pipeline =
           g_strdup_printf
-          ("appsrc name=appsrc caps=text/x-raw,format=utf8 ! "
+          ("appsrc name=appsrc caps=text/x-raw,format=utf8,framerate=(fraction)100/1 ! "
           "tensor_converter frames-per-tensor=3 ! tensor_sink name=test_sink");
       break;
+    case TEST_TYPE_OCTET_1:
+      /** byte stream */
+      str_pipeline =
+          g_strdup_printf
+          ("appsrc name=appsrc caps=application/octet-stream ! "
+          "tensor_converter input-dim=1:10 input-type=uint8 ! tensor_sink name=test_sink");
+      break;
+    case TEST_TYPE_OCTET_2:
+      /** byte stream */
+      str_pipeline =
+          g_strdup_printf
+          ("appsrc name=appsrc caps=application/octet-stream,framerate=(fraction)100/1 ! "
+          "tensor_converter input-dim=1:10 input-type=uint8 ! tensor_sink name=test_sink");
+      break;
     case TEST_TYPE_TENSORS:
       /** other/tensors with tensor_mux */
       str_pipeline =
@@ -1712,6 +1729,51 @@ TEST (tensor_stream_test, text_utf8_3f)
   EXPECT_EQ (g_test_data.tensor_config.info.dimension[1], 3);
   EXPECT_EQ (g_test_data.tensor_config.info.dimension[2], 1);
   EXPECT_EQ (g_test_data.tensor_config.info.dimension[3], 1);
+  EXPECT_EQ (g_test_data.tensor_config.rate_n, 100);
+  EXPECT_EQ (g_test_data.tensor_config.rate_d, 1);
+
+  EXPECT_FALSE (g_test_data.test_failed);
+  _free_test_data ();
+}
+
+/**
+ * @brief Test for octet stream.
+ */
+TEST (tensor_stream_test, octet_1)
+{
+  const guint num_buffers = 10;
+  TestOption option = { num_buffers, TEST_TYPE_OCTET_1 };
+
+  ASSERT_TRUE (_setup_pipeline (option));
+
+  gst_element_set_state (g_test_data.pipeline, GST_STATE_PLAYING);
+
+  _push_text_data (num_buffers);
+
+  g_main_loop_run (g_test_data.loop);
+  gst_element_set_state (g_test_data.pipeline, GST_STATE_NULL);
+
+  /** check eos message */
+  EXPECT_EQ (g_test_data.status, TEST_EOS);
+
+  /** check received buffers */
+  EXPECT_EQ (g_test_data.received, num_buffers);
+  EXPECT_EQ (g_test_data.mem_blocks, 1);
+  EXPECT_EQ (g_test_data.received_size, 10);
+
+  /** check caps name */
+  EXPECT_TRUE (g_str_equal (g_test_data.caps_name, "other/tensor"));
+
+  /** check timestamp */
+  EXPECT_FALSE (g_test_data.invalid_timestamp);
+
+  /** check tensor config for text */
+  EXPECT_TRUE (gst_tensor_config_validate (&g_test_data.tensor_config));
+  EXPECT_EQ (g_test_data.tensor_config.info.type, _NNS_UINT8);
+  EXPECT_EQ (g_test_data.tensor_config.info.dimension[0], 1);
+  EXPECT_EQ (g_test_data.tensor_config.info.dimension[1], 10);
+  EXPECT_EQ (g_test_data.tensor_config.info.dimension[2], 1);
+  EXPECT_EQ (g_test_data.tensor_config.info.dimension[3], 1);
   EXPECT_EQ (g_test_data.tensor_config.rate_n, 0);
   EXPECT_EQ (g_test_data.tensor_config.rate_d, 1);
 
@@ -1720,6 +1782,51 @@ TEST (tensor_stream_test, text_utf8_3f)
 }
 
 /**
+ * @brief Test for octet stream.
+ */
+TEST (tensor_stream_test, octet_2)
+{
+  const guint num_buffers = 10;
+  TestOption option = { num_buffers, TEST_TYPE_OCTET_2 };
+
+  ASSERT_TRUE (_setup_pipeline (option));
+
+  gst_element_set_state (g_test_data.pipeline, GST_STATE_PLAYING);
+
+  _push_text_data (num_buffers);
+
+  g_main_loop_run (g_test_data.loop);
+  gst_element_set_state (g_test_data.pipeline, GST_STATE_NULL);
+
+  /** check eos message */
+  EXPECT_EQ (g_test_data.status, TEST_EOS);
+
+  /** check received buffers */
+  EXPECT_EQ (g_test_data.received, num_buffers);
+  EXPECT_EQ (g_test_data.mem_blocks, 1);
+  EXPECT_EQ (g_test_data.received_size, 10);
+
+  /** check caps name */
+  EXPECT_TRUE (g_str_equal (g_test_data.caps_name, "other/tensor"));
+
+  /** check timestamp */
+  EXPECT_FALSE (g_test_data.invalid_timestamp);
+
+  /** check tensor config for text */
+  EXPECT_TRUE (gst_tensor_config_validate (&g_test_data.tensor_config));
+  EXPECT_EQ (g_test_data.tensor_config.info.type, _NNS_UINT8);
+  EXPECT_EQ (g_test_data.tensor_config.info.dimension[0], 1);
+  EXPECT_EQ (g_test_data.tensor_config.info.dimension[1], 10);
+  EXPECT_EQ (g_test_data.tensor_config.info.dimension[2], 1);
+  EXPECT_EQ (g_test_data.tensor_config.info.dimension[3], 1);
+  EXPECT_EQ (g_test_data.tensor_config.rate_n, 100);
+  EXPECT_EQ (g_test_data.tensor_config.rate_d, 1);
+
+  EXPECT_FALSE (g_test_data.test_failed);
+  _free_test_data ();
+}
+
+/**
  * @brief Test for other/tensor, passthrough custom filter.
  */
 TEST (tensor_stream_test, custom_filter_tensor)