[Common] util function to handle tensors

With gstreamer plugins, nnstreamer element may receive gst-buffer with invalid memory chunks.
nnstreamer internally checks the number of tensors using the mem blocks in gst-buffer.

To handle multi tensors, add util function to separate memories in gst-buffer.

Signed-off-by: Jaeyun <jy1210.jung@samsung.com>

diff --git a/gst/nnstreamer/tensor_common.h b/gst/nnstreamer/tensor_common.h
index f6ff34f..e405d96 100644 (file)
--- a/gst/nnstreamer/tensor_common.h
+++ b/gst/nnstreamer/tensor_common.h
@@ -165,6 +165,18 @@ extern gboolean
 gst_tensor_time_sync_buffer_from_collectpad (GstCollectPads * collect, tensor_time_sync_data * sync, GstClockTime current_time, GstBuffer * tensors_buf, GstTensorsConfig * configs, gboolean * is_eos);
 
 /**
+ * @brief Configure gst-buffer with tensors information.
+ * NNStreamer handles single memory chunk as single tensor.
+ * If incoming buffer has invalid memories, separate it and generate new gst-buffer using tensors information.
+ * Note that this function always takes the ownership of input buffer.
+ * @param in input buffer
+ * @param config tensors config structure
+ * @return Newly allocated buffer. Null if failed. Caller should unref the buffer using gst_buffer_unref().
+ */
+extern GstBuffer *
+gst_tensor_buffer_from_config (GstBuffer * in, GstTensorsConfig * config);
+
+/**
  * @brief Get pad caps from tensors config and caps of the peer connected to the pad.
  * @param pad GstPad to get possible caps
  * @param config tensors config structure

diff --git a/gst/nnstreamer/tensor_common_pipeline.c b/gst/nnstreamer/tensor_common_pipeline.c
index 02a1f89..f3f8023 100644 (file)
--- a/gst/nnstreamer/tensor_common_pipeline.c
+++ b/gst/nnstreamer/tensor_common_pipeline.c
@@ -446,6 +446,121 @@ gst_tensor_time_sync_buffer_from_collectpad (GstCollectPads * collect,
 }
 
 /**
+ * @brief Configure gst-buffer with tensors information.
+ * NNStreamer handles single memory chunk as single tensor.
+ * If incoming buffer has invalid memories, separate it and generate new gst-buffer using tensors information.
+ * Note that this function always takes the ownership of input buffer.
+ * @param in input buffer
+ * @param config tensors config structure
+ * @return Newly allocated buffer. Null if failed. Caller should unref the buffer using gst_buffer_unref().
+ */
+GstBuffer *
+gst_tensor_buffer_from_config (GstBuffer * in, GstTensorsConfig * config)
+{
+  GstBuffer *out = NULL;
+  GstMemory *all = NULL;
+  GstMapInfo map;
+  guint i, num;
+  gsize total, offset;
+  gsize mem_size[NNS_TENSOR_SIZE_LIMIT];
+  gboolean configured = FALSE;
+
+  if (!GST_IS_BUFFER (in)) {
+    nns_loge ("Failed to get tensor buffer, invalid input buffer.");
+    return NULL;
+  }
+
+  if (!gst_tensors_config_validate (config)) {
+    nns_loge ("Failed to get tensor buffer, invalid tensor configuration.");
+    goto error;
+  }
+
+  num = gst_buffer_n_memory (in);
+  total = gst_buffer_get_size (in);
+
+  /* get memory size */
+  if (gst_tensors_config_is_static (config)) {
+    if (num == config->info.num_tensors) {
+      /* Do nothing, pass input buffer. */
+      out = gst_buffer_ref (in);
+      goto done;
+    }
+
+    num = config->info.num_tensors;
+    for (i = 0; i < num; i++)
+      mem_size[i] = gst_tensors_info_get_size (&config->info, i);
+  } else {
+    if (num > 1) {
+      /* Suppose it is already configured. */
+      out = gst_buffer_ref (in);
+      goto done;
+    }
+
+    if (!gst_buffer_map (in, &map, GST_MAP_READ)) {
+      nns_loge ("Failed to get tensor buffer, cannot get the memory info.");
+      goto error;
+    }
+
+    num = 0;
+    offset = 0;
+    while (offset < total) {
+      GstTensorMetaInfo meta;
+      gpointer h = map.data + offset;
+
+      gst_tensor_meta_info_parse_header (&meta, h);
+      mem_size[num] = gst_tensor_meta_info_get_header_size (&meta);
+      mem_size[num] += gst_tensor_meta_info_get_data_size (&meta);
+
+      offset += mem_size[num];
+      num++;
+    }
+
+    gst_buffer_unmap (in, &map);
+
+    if (num == 1) {
+      /* Do nothing, pass input buffer. */
+      out = gst_buffer_ref (in);
+      goto done;
+    }
+  }
+
+  /* configure output buffer */
+  out = gst_buffer_new ();
+  all = gst_buffer_get_all_memory (in);
+  offset = 0;
+
+  for (i = 0; i < num; i++) {
+    /* invalid memory size */
+    if (offset + mem_size[i] > total) {
+      nns_loge ("Failed to get tensor buffer, data size is mismatched.");
+      goto error;
+    }
+
+    gst_buffer_append_memory (out, gst_memory_share (all, offset, mem_size[i]));
+    offset += mem_size[i];
+  }
+
+  gst_buffer_copy_into (out, in, GST_BUFFER_COPY_METADATA, 0, -1);
+
+done:
+  configured = TRUE;
+error:
+  gst_buffer_unref (in);
+
+  if (all)
+    gst_memory_unref (all);
+
+  if (!configured) {
+    if (out) {
+      gst_buffer_unref (out);
+      out = NULL;
+    }
+  }
+
+  return out;
+}
+
+/**
  * @brief Internal struct to handle aggregation data in hash table.
  */
 typedef struct

diff --git a/tests/common/unittest_common.cc b/tests/common/unittest_common.cc
index 1d8a332..91e8128 100644 (file)
--- a/tests/common/unittest_common.cc
+++ b/tests/common/unittest_common.cc
@@ -1758,6 +1758,185 @@ TEST (versionControl, getVer01)
 }
 
 /**
+ * @brief Test tensor buffer util (create static tensor buffer)
+ */
+TEST (commonUtil, createStaticTensorBuffer)
+{
+  GstTensorsConfig config;
+  GstBuffer *in, *out;
+  GstMemory *mem;
+  GstMapInfo map;
+  guint i, j, num, expected;
+  guint *input, *output;
+  gsize data_size;
+
+  gst_tensors_config_init (&config);
+  config.format = _NNS_TENSOR_FORMAT_STATIC;
+  config.rate_n = config.rate_d = 1;
+  config.info.num_tensors = 3U;
+  gst_tensors_info_parse_dimensions_string (&config.info, "20,40,50");
+  gst_tensors_info_parse_types_string (&config.info, "uint32,uint32,uint32");
+
+  data_size = gst_tensors_info_get_size (&config.info, -1);
+  input = (guint *) g_malloc (data_size);
+
+  for (i = 0; i < data_size / sizeof (guint); i++)
+    input[i] = i;
+
+  in = gst_buffer_new_wrapped (input, data_size);
+  out = gst_tensor_buffer_from_config (in, &config);
+  ASSERT_TRUE (out != NULL);
+
+  num = gst_buffer_n_memory (out);
+  EXPECT_EQ (num, config.info.num_tensors);
+
+  expected = 0U;
+  for (i = 0; i < num; i++) {
+    mem = gst_buffer_peek_memory (out, i);
+    ASSERT_TRUE (gst_memory_map (mem, &map, GST_MAP_READ));
+    EXPECT_EQ (map.size, gst_tensors_info_get_size (&config.info, i));
+
+    output = (guint *) map.data;
+    for (j = 0; j < map.size / sizeof (guint); j++)
+      EXPECT_EQ (output[j], expected++);
+
+    gst_memory_unmap (mem, &map);
+  }
+
+  gst_buffer_unref (out);
+}
+
+/**
+ * @brief Test tensor buffer util (create flexible tensor buffer)
+ */
+TEST (commonUtil, createFlexTensorBuffer)
+{
+  GstTensorsConfig config;
+  GstBuffer *in, *out;
+  GstMemory *mem;
+  GstMapInfo map;
+  guint i, j, num;
+  guint *input, *output;
+  guint8 *data;
+  GstTensorMetaInfo meta[3];
+  gsize data_size, offset, hsize[3], dsize[3];
+
+  gst_tensors_config_init (&config);
+  config.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
+  config.rate_n = config.rate_d = 1;
+  config.info.num_tensors = 3U;
+  gst_tensors_info_parse_dimensions_string (&config.info, "20,30,40");
+  gst_tensors_info_parse_types_string (&config.info, "uint32,uint32,uint32");
+
+  data_size = 0;
+  for (i = 0; i < 3U; i++) {
+    gst_tensor_info_convert_to_meta (&config.info.info[i], &meta[i]);
+    hsize[i] = gst_tensor_meta_info_get_header_size (&meta[i]);
+    dsize[i] = gst_tensor_meta_info_get_data_size (&meta[i]);
+    data_size += (hsize[i] + dsize[i]);
+  }
+
+  data = (guint8 *) g_malloc (data_size);
+
+  offset = 0;
+  for (i = 0; i < 3U; i++) {
+    gst_tensor_meta_info_update_header (&meta[i], data + offset);
+
+    input = (guint *) (data + offset + hsize[i]);
+    for (j = 0; j < dsize[i] / sizeof (guint); j++)
+      input[j] = i * 10 + j;
+
+    offset += (hsize[i] + dsize[i]);
+  }
+
+  in = gst_buffer_new_wrapped (data, data_size);
+  out = gst_tensor_buffer_from_config (in, &config);
+  ASSERT_TRUE (out != NULL);
+
+  num = gst_buffer_n_memory (out);
+  EXPECT_EQ (num, config.info.num_tensors);
+
+  for (i = 0; i < num; i++) {
+    mem = gst_buffer_peek_memory (out, i);
+    ASSERT_TRUE (gst_memory_map (mem, &map, GST_MAP_READ));
+    EXPECT_EQ (map.size, hsize[i] + dsize[i]);
+
+    output = (guint *) (map.data + hsize[i]);
+    for (j = 0; j < dsize[i] / sizeof (guint); j++)
+      EXPECT_EQ (output[j], i * 10 + j);
+
+    gst_memory_unmap (mem, &map);
+  }
+
+  gst_buffer_unref (out);
+}
+
+/**
+ * @brief Test tensor buffer util (invalid config)
+ */
+TEST (commonUtil, createTensorBufferInvalidConfig_n)
+{
+  GstTensorsConfig config;
+  GstBuffer *in, *out;
+
+  gst_tensors_config_init (&config);
+
+  in = gst_buffer_new_allocate (NULL, 200, NULL);
+  out = gst_tensor_buffer_from_config (in, &config);
+  EXPECT_FALSE (out != NULL);
+}
+
+/**
+ * @brief Test tensor buffer util (null param)
+ */
+TEST (commonUtil, createTensorBufferNullParam_n)
+{
+  GstTensorsConfig config;
+  GstBuffer *in, *out;
+  gsize data_size;
+
+  gst_tensors_config_init (&config);
+  config.format = _NNS_TENSOR_FORMAT_STATIC;
+  config.rate_n = config.rate_d = 1;
+  config.info.num_tensors = 3U;
+  gst_tensors_info_parse_dimensions_string (&config.info, "20,40,50");
+  gst_tensors_info_parse_types_string (&config.info, "uint32,uint32,uint32");
+
+  data_size = gst_tensors_info_get_size (&config.info, -1);
+
+  in = gst_buffer_new_allocate (NULL, data_size, NULL);
+  out = gst_tensor_buffer_from_config (NULL, &config);
+  EXPECT_FALSE (out != NULL);
+
+  in = gst_buffer_new_allocate (NULL, data_size, NULL);
+  out = gst_tensor_buffer_from_config (in, NULL);
+  EXPECT_FALSE (out != NULL);
+}
+
+/**
+ * @brief Test tensor buffer util (invalid buffer size)
+ */
+TEST (commonUtil, createTensorBufferInvalidSize_n)
+{
+  GstTensorsConfig config;
+  GstBuffer *in, *out;
+  gsize data_size;
+
+  gst_tensors_config_init (&config);
+  config.format = _NNS_TENSOR_FORMAT_STATIC;
+  config.rate_n = config.rate_d = 1;
+  config.info.num_tensors = 3U;
+  gst_tensors_info_parse_dimensions_string (&config.info, "20,40,50");
+  gst_tensors_info_parse_types_string (&config.info, "uint32,uint32,uint32");
+
+  data_size = gst_tensors_info_get_size (&config.info, -1) / 2;
+
+  in = gst_buffer_new_allocate (NULL, data_size, NULL);
+  out = gst_tensor_buffer_from_config (in, &config);
+  EXPECT_FALSE (out != NULL);
+}
+
+/**
  * @brief Main function for unit test.
  */
 int